From 97dade0bc4987cd532b009d717f80fd60e301b5d Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 28 Jul 2025 20:53:05 +0300 Subject: [PATCH 01/47] WIP: Use SyncA to generate [a]sync APIs Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 116 ++- avro/Cargo.toml | 6 + avro/src/bigdecimal.rs | 42 +- avro/src/decode.rs | 1406 ++++++++++++++++--------------- avro/src/encode.rs | 5 +- avro/src/lib.rs | 39 +- avro/src/reader.rs | 1788 +++++++++++++++++++++------------------- avro/src/schema.rs | 21 +- avro/src/types.rs | 412 ++++----- avro/src/util.rs | 408 ++++----- 10 files changed, 2264 insertions(+), 1979 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac864b45..8466e416 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,7 @@ dependencies = [ "crc32fast", "criterion", "digest", + "futures", "hex-literal", "log", "md-5", @@ -76,7 +77,9 @@ dependencies = [ "snap", "strum", "strum_macros", + "synca", "thiserror", + "tokio", "uuid", "xz2", "zstd", @@ -204,6 +207,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "bzip2" version = "0.6.0" @@ -542,6 +551,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -563,6 +583,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -589,7 +610,7 @@ checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.13.3+wasi-0.2.2", "windows-targets", ] @@ -644,6 +665,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "itertools" version = "0.13.0" @@ -692,9 +724,9 @@ checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" [[package]] name = "libc" -version = "0.2.168" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libm" @@ -764,6 +796,17 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1191,6 +1234,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.9" @@ -1212,6 +1264,16 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1247,6 +1309,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synca" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dfc1a8bd0488fee2f2c457925766a8471564a6fbf7cb4211bd8a0e5f68c754a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thiserror" version = "2.0.12" @@ -1277,6 +1350,37 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio" +version = "1.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typenum" version = "1.17.0" @@ -1322,6 +1426,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.13.3+wasi-0.2.2" diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 48337e91..b16e04f3 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -29,11 +29,14 @@ categories.workspace = true documentation.workspace = true [features] +default = [ "tokio"] bzip = ["dep:bzip2"] derive = ["dep:apache-avro-derive"] snappy = ["dep:crc32fast", "dep:snap"] xz = ["dep:xz2"] zstandard = ["dep:zstd"] +sync = [ ] +tokio = [ "dep:tokio", "dep:futures" ] [lib] # disable benchmarks to allow passing criterion arguments to `cargo bench` @@ -70,9 +73,12 @@ snap = { default-features = false, version = "1.1.0", optional = true } strum = { default-features = false, version = "0.27.2" } strum_macros = { default-features = false, version = "0.27.2" } thiserror = { default-features = false, version = "2.0.12" } +tokio = { version = "1.47.0", features = [ "full" ], optional = true } uuid = { default-features = false, version = "1.17.0", features = ["serde", "std"] } xz2 = { default-features = false, version = "0.1.7", optional = true } zstd = { default-features = false, version = "0.13.3", optional = true } +synca = "0.5.3" +futures = { version = "0.3.31", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 0f4e1647..563bbe82 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -15,9 +15,23 @@ // specific language governing permissions and limitations // under the License. +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + decode::tokio => decode::sync, + #[tokio::test] => #[test] + ); + } +)] +mod bigdecimal {} use crate::{ AvroResult, - decode::{decode_len, decode_long}, + decode::tokio::{decode_len, decode_long}, encode::{encode_bytes, encode_long}, error::Details, types::Value, @@ -47,9 +61,9 @@ pub(crate) fn serialize_big_decimal(decimal: &BigDecimal) -> AvroResult> Ok(final_buffer) } -pub(crate) fn deserialize_big_decimal(bytes: &Vec) -> AvroResult { +pub(crate) async fn deserialize_big_decimal(bytes: &Vec) -> AvroResult { let mut bytes: &[u8] = bytes.as_slice(); - let mut big_decimal_buffer = match decode_len(&mut bytes) { + let mut big_decimal_buffer = match decode_len(&mut bytes).await { Ok(size) => vec![0u8; size], Err(err) => return Err(Details::BigDecimalLen(Box::new(err)).into()), }; @@ -58,7 +72,7 @@ pub(crate) fn deserialize_big_decimal(bytes: &Vec) -> AvroResult .read_exact(&mut big_decimal_buffer[..]) .map_err(Details::ReadDouble)?; - match decode_long(&mut bytes) { + match decode_long(&mut bytes).await { Ok(Value::Long(scale_value)) => { let big_int: BigInt = BigInt::from_signed_bytes_be(&big_decimal_buffer); let decimal = BigDecimal::new(big_int, scale_value); @@ -82,8 +96,8 @@ mod tests { str::FromStr, }; - #[test] - fn test_avro_3779_bigdecimal_serial() -> TestResult { + #[tokio::test] + async fn test_avro_3779_bigdecimal_serial() -> TestResult { let value: BigDecimal = bigdecimal::BigDecimal::from(-1421).div(bigdecimal::BigDecimal::from(2)); let mut current: BigDecimal = BigDecimal::one(); @@ -92,7 +106,7 @@ mod tests { let buffer: Vec = serialize_big_decimal(¤t)?; let mut as_slice = buffer.as_slice(); - decode_long(&mut as_slice)?; + decode_long(&mut as_slice).await?; let mut result: Vec = Vec::new(); result.extend_from_slice(as_slice); @@ -109,7 +123,7 @@ mod tests { let buffer: Vec = serialize_big_decimal(&BigDecimal::zero())?; let mut as_slice = buffer.as_slice(); - decode_long(&mut as_slice)?; + decode_long(&mut as_slice).await?; let mut result: Vec = Vec::new(); result.extend_from_slice(as_slice); @@ -128,8 +142,8 @@ mod tests { Ok(()) } - #[test] - fn test_avro_3779_record_with_bg() -> TestResult { + #[tokio::test] + async fn test_avro_3779_record_with_bg() -> TestResult { let schema_str = r#" { "type": "record", @@ -165,7 +179,7 @@ mod tests { let wrote_data = writer.into_inner()?; let mut reader = Reader::new(&wrote_data[..])?; - let value = reader.next().unwrap()?; + let value = reader.next().await.unwrap()?; // extract field value let big_decimal_value: &Value = match value { @@ -182,13 +196,13 @@ mod tests { Ok(()) } - #[test] - fn test_avro_3779_from_java_file() -> TestResult { + #[tokio::test] + async fn test_avro_3779_from_java_file() -> TestResult { // Open file generated with Java code to ensure compatibility // with Java big decimal logical type. let file: File = File::open("./tests/bigdec.avro")?; let mut reader = Reader::new(BufReader::new(&file))?; - let next_element = reader.next(); + let next_element = reader.next().await; assert!(next_element.is_some()); let value = next_element.unwrap()?; let bg = match value { diff --git a/avro/src/decode.rs b/avro/src/decode.rs index fc700ae4..0313d25c 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -15,475 +15,516 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - AvroResult, Error, - bigdecimal::deserialize_big_decimal, - decimal::Decimal, - duration::Duration, - encode::encode_long, - error::Details, - schema::{ - DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, - Schema, - }, - types::Value, - util::{safe_len, zag_i32, zag_i64}, -}; -use std::{ - borrow::Borrow, - collections::HashMap, - io::{ErrorKind, Read}, - str::FromStr, -}; -use uuid::Uuid; - -#[inline] -pub(crate) fn decode_long(reader: &mut R) -> AvroResult { - zag_i64(reader).map(Value::Long) -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + #[tokio::test] => #[test] + ); + } +)] +mod decode { + #[synca::cfg(tokio)] + use tokio::io::AsyncReadExt; -#[inline] -fn decode_int(reader: &mut R) -> AvroResult { - zag_i32(reader).map(Value::Int) -} + use crate::{ + AvroResult, Error, + bigdecimal::deserialize_big_decimal, + decimal::Decimal, + duration::Duration, + encode::encode_long, + error::Details, + schema::{ + DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, + Schema, + }, + types::Value, + }; + use std::{borrow::Borrow, collections::HashMap, io::ErrorKind, str::FromStr}; + use uuid::Uuid; + use crate::util::safe_len; + #[cfg(feature = "tokio")] + use crate::util::tokio::{zag_i32, zag_i64}; + #[cfg(feature = "sync")] + use crate::util::sync::{zag_i32, zag_i64}; + + #[inline] + pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { + zag_i64(reader).await.map(Value::Long) + } -#[inline] -pub(crate) fn decode_len(reader: &mut R) -> AvroResult { - let len = zag_i64(reader)?; - safe_len(usize::try_from(len).map_err(|e| Details::ConvertI64ToUsize(e, len))?) -} + #[inline] + async fn decode_int(reader: &mut R) -> AvroResult { + zag_i32(reader).await.map(Value::Int) + } -/// Decode the length of a sequence. -/// -/// Maps and arrays are 0-terminated, 0i64 is also encoded as 0 in Avro reading a length of 0 means -/// the end of the map or array. -fn decode_seq_len(reader: &mut R) -> AvroResult { - let raw_len = zag_i64(reader)?; - safe_len( - usize::try_from(match raw_len.cmp(&0) { - std::cmp::Ordering::Equal => return Ok(0), - std::cmp::Ordering::Less => { - let _size = zag_i64(reader)?; - raw_len.checked_neg().ok_or(Details::IntegerOverflow)? - } - std::cmp::Ordering::Greater => raw_len, - }) - .map_err(|e| Details::ConvertI64ToUsize(e, raw_len))?, - ) -} + #[inline] + pub(crate) async fn decode_len(reader: &mut R) -> AvroResult { + let len = zag_i64(reader).await?; + safe_len(usize::try_from(len).map_err(|e| Details::ConvertI64ToUsize(e, len))?) + } -/// Decode a `Value` from avro format given its `Schema`. -pub fn decode(schema: &Schema, reader: &mut R) -> AvroResult { - let rs = ResolvedSchema::try_from(schema)?; - decode_internal(schema, rs.get_names(), &None, reader) -} + /// Decode the length of a sequence. + /// + /// Maps and arrays are 0-terminated, 0i64 is also encoded as 0 in Avro reading a length of 0 means + /// the end of the map or array. + async fn decode_seq_len(reader: &mut R) -> AvroResult { + let raw_len = zag_i64(reader).await?; + safe_len( + usize::try_from(match raw_len.cmp(&0) { + std::cmp::Ordering::Equal => return Ok(0), + std::cmp::Ordering::Less => { + let _size = zag_i64(reader).await?; + raw_len.checked_neg().ok_or(Details::IntegerOverflow)? + } + std::cmp::Ordering::Greater => raw_len, + }) + .map_err(|e| Details::ConvertI64ToUsize(e, raw_len))?, + ) + } -pub(crate) fn decode_internal>( - schema: &Schema, - names: &HashMap, - enclosing_namespace: &Namespace, - reader: &mut R, -) -> AvroResult { - match *schema { - Schema::Null => Ok(Value::Null), - Schema::Boolean => { - let mut buf = [0u8; 1]; - match reader.read_exact(&mut buf[..]) { - Ok(_) => match buf[0] { - 0u8 => Ok(Value::Boolean(false)), - 1u8 => Ok(Value::Boolean(true)), - _ => Err(Details::BoolValue(buf[0]).into()), - }, - Err(io_err) => { - if let ErrorKind::UnexpectedEof = io_err.kind() { - Ok(Value::Null) - } else { - Err(Details::ReadBoolean(io_err).into()) + /// Decode a `Value` from avro format given its `Schema`. + pub async fn decode( + schema: &Schema, + reader: &mut R, + ) -> AvroResult { + let rs = ResolvedSchema::try_from(schema)?; + decode_internal(schema, rs.get_names(), &None, reader).await + } + + pub(crate) async fn decode_internal>( + schema: &Schema, + names: &HashMap, + enclosing_namespace: &Namespace, + reader: &mut R, + ) -> AvroResult { + match *schema { + Schema::Null => Ok(Value::Null), + Schema::Boolean => { + let mut buf = [0u8; 1]; + match reader.read_exact(&mut buf[..]).await { + Ok(_) => match buf[0] { + 0u8 => Ok(Value::Boolean(false)), + 1u8 => Ok(Value::Boolean(true)), + _ => Err(Details::BoolValue(buf[0]).into()), + }, + Err(io_err) => { + if let ErrorKind::UnexpectedEof = io_err.kind() { + Ok(Value::Null) + } else { + Err(Details::ReadBoolean(io_err).into()) + } } } } - } - Schema::Decimal(DecimalSchema { ref inner, .. }) => match &**inner { - Schema::Fixed { .. } => { - match decode_internal(inner, names, enclosing_namespace, reader)? { - Value::Fixed(_, bytes) => Ok(Value::Decimal(Decimal::from(bytes))), - value => Err(Details::FixedValue(value).into()), + Schema::Decimal(DecimalSchema { ref inner, .. }) => match &**inner { + Schema::Fixed { .. } => { + match Box::pin(decode_internal(inner, names, enclosing_namespace, reader)).await? { + Value::Fixed(_, bytes) => Ok(Value::Decimal(Decimal::from(bytes))), + value => Err(Details::FixedValue(value).into()), + } } - } - Schema::Bytes => match decode_internal(inner, names, enclosing_namespace, reader)? { - Value::Bytes(bytes) => Ok(Value::Decimal(Decimal::from(bytes))), - value => Err(Details::BytesValue(value).into()), + Schema::Bytes => { + match Box::pin(decode_internal(inner, names, enclosing_namespace, reader)).await? { + Value::Bytes(bytes) => Ok(Value::Decimal(Decimal::from(bytes))), + value => Err(Details::BytesValue(value).into()), + } + } + schema => Err(Details::ResolveDecimalSchema(schema.into()).into()), }, - schema => Err(Details::ResolveDecimalSchema(schema.into()).into()), - }, - Schema::BigDecimal => { - match decode_internal(&Schema::Bytes, names, enclosing_namespace, reader)? { - Value::Bytes(bytes) => deserialize_big_decimal(&bytes).map(Value::BigDecimal), - value => Err(Details::BytesValue(value).into()), - } - } - Schema::Uuid => { - let len = decode_len(reader)?; - let mut bytes = vec![0u8; len]; - reader - .read_exact(&mut bytes) - .map_err(Details::ReadIntoBuf)?; - - // use a Vec to be able re-read the bytes more than once if needed - let mut reader = Vec::with_capacity(len + 1); - encode_long(len as i64, &mut reader)?; - reader.extend_from_slice(&bytes); - - let decode_from_string = |reader| match decode_internal( - &Schema::String, - names, - enclosing_namespace, - reader, - )? { - Value::String(ref s) => { - Uuid::from_str(s).map_err(|e| Details::ConvertStrToUuid(e).into()) + Schema::BigDecimal => { + match Box::pin(decode_internal(&Schema::Bytes, names, enclosing_namespace, reader)).await? { + Value::Bytes(bytes) => deserialize_big_decimal(&bytes).await.map(Value::BigDecimal), + value => Err(Details::BytesValue(value).into()), } - value => Err(Error::new(Details::GetUuidFromStringValue(value))), - }; - - let uuid: Uuid = if len == 16 { - // most probably a Fixed schema - let fixed_result = decode_internal( - &Schema::Fixed(FixedSchema { - size: 16, - name: "uuid".into(), - aliases: None, - doc: None, - default: None, - attributes: Default::default(), - }), + } + Schema::Uuid => { + let len = decode_len(reader).await?; + let mut bytes = vec![0u8; len]; + reader + .read_exact(&mut bytes) + .await + .map_err(Details::ReadIntoBuf)?; + + // use a Vec to be able re-read the bytes more than once if needed + let mut reader = Vec::with_capacity(len + 1); + encode_long(len as i64, &mut reader)?; + reader.extend_from_slice(&bytes); + + let decode_from_string = async |reader| match decode_internal( + &Schema::String, names, enclosing_namespace, - &mut bytes.as_slice(), - ); - if fixed_result.is_ok() { - match fixed_result? { - Value::Fixed(ref size, ref bytes) => { - if *size != 16 { - return Err(Details::ConvertFixedToUuid(*size).into()); + reader, + ) + .await? + { + Value::String(ref s) => { + Uuid::from_str(s).map_err(|e| Details::ConvertStrToUuid(e).into()) + } + value => Err(Error::new(Details::GetUuidFromStringValue(value))), + }; + + let uuid: Uuid = if len == 16 { + // most probably a Fixed schema + let fixed_result = Box::pin(decode_internal( + &Schema::Fixed(FixedSchema { + size: 16, + name: "uuid".into(), + aliases: None, + doc: None, + default: None, + attributes: Default::default(), + }), + names, + enclosing_namespace, + &mut bytes.as_slice(), + )) + .await; + if fixed_result.is_ok() { + match fixed_result? { + Value::Fixed(ref size, ref bytes) => { + if *size != 16 { + return Err(Details::ConvertFixedToUuid(*size).into()); + } + Uuid::from_slice(bytes).map_err(Details::ConvertSliceToUuid)? } - Uuid::from_slice(bytes).map_err(Details::ConvertSliceToUuid)? + _ => Box::pin(decode_from_string(&mut reader.as_slice())).await?, } - _ => decode_from_string(&mut reader.as_slice())?, + } else { + // try to decode as string + Box::pin(decode_from_string(&mut reader.as_slice())).await? } } else { - // try to decode as string - decode_from_string(&mut reader.as_slice())? - } - } else { - // definitely a string - decode_from_string(&mut reader.as_slice())? - }; - Ok(Value::Uuid(uuid)) - } - Schema::Int => decode_int(reader), - Schema::Date => zag_i32(reader).map(Value::Date), - Schema::TimeMillis => zag_i32(reader).map(Value::TimeMillis), - Schema::Long => decode_long(reader), - Schema::TimeMicros => zag_i64(reader).map(Value::TimeMicros), - Schema::TimestampMillis => zag_i64(reader).map(Value::TimestampMillis), - Schema::TimestampMicros => zag_i64(reader).map(Value::TimestampMicros), - Schema::TimestampNanos => zag_i64(reader).map(Value::TimestampNanos), - Schema::LocalTimestampMillis => zag_i64(reader).map(Value::LocalTimestampMillis), - Schema::LocalTimestampMicros => zag_i64(reader).map(Value::LocalTimestampMicros), - Schema::LocalTimestampNanos => zag_i64(reader).map(Value::LocalTimestampNanos), - Schema::Duration => { - let mut buf = [0u8; 12]; - reader.read_exact(&mut buf).map_err(Details::ReadDuration)?; - Ok(Value::Duration(Duration::from(buf))) - } - Schema::Float => { - let mut buf = [0u8; std::mem::size_of::()]; - reader - .read_exact(&mut buf[..]) - .map_err(Details::ReadFloat)?; - Ok(Value::Float(f32::from_le_bytes(buf))) - } - Schema::Double => { - let mut buf = [0u8; std::mem::size_of::()]; - reader - .read_exact(&mut buf[..]) - .map_err(Details::ReadDouble)?; - Ok(Value::Double(f64::from_le_bytes(buf))) - } - Schema::Bytes => { - let len = decode_len(reader)?; - let mut buf = vec![0u8; len]; - reader.read_exact(&mut buf).map_err(Details::ReadBytes)?; - Ok(Value::Bytes(buf)) - } - Schema::String => { - let len = decode_len(reader)?; - let mut buf = vec![0u8; len]; - match reader.read_exact(&mut buf) { - Ok(_) => Ok(Value::String( - String::from_utf8(buf).map_err(Details::ConvertToUtf8)?, - )), - Err(io_err) => { - if let ErrorKind::UnexpectedEof = io_err.kind() { - Ok(Value::Null) - } else { - Err(Details::ReadString(io_err).into()) + // definitely a string + Box::pin(decode_from_string(&mut reader.as_slice())).await? + }; + Ok(Value::Uuid(uuid)) + } + Schema::Int => decode_int(reader).await, + Schema::Date => zag_i32(reader).await.map(Value::Date), + Schema::TimeMillis => zag_i32(reader).await.map(Value::TimeMillis), + Schema::Long => decode_long(reader).await, + Schema::TimeMicros => zag_i64(reader).await.map(Value::TimeMicros), + Schema::TimestampMillis => zag_i64(reader).await.map(Value::TimestampMillis), + Schema::TimestampMicros => zag_i64(reader).await.map(Value::TimestampMicros), + Schema::TimestampNanos => zag_i64(reader).await.map(Value::TimestampNanos), + Schema::LocalTimestampMillis => zag_i64(reader).await.map(Value::LocalTimestampMillis), + Schema::LocalTimestampMicros => zag_i64(reader).await.map(Value::LocalTimestampMicros), + Schema::LocalTimestampNanos => zag_i64(reader).await.map(Value::LocalTimestampNanos), + Schema::Duration => { + let mut buf = [0u8; 12]; + reader + .read_exact(&mut buf) + .await + .map_err(Details::ReadDuration)?; + Ok(Value::Duration(Duration::from(buf))) + } + Schema::Float => { + let mut buf = [0u8; std::mem::size_of::()]; + reader + .read_exact(&mut buf[..]) + .await + .map_err(Details::ReadFloat)?; + Ok(Value::Float(f32::from_le_bytes(buf))) + } + Schema::Double => { + let mut buf = [0u8; std::mem::size_of::()]; + reader + .read_exact(&mut buf[..]) + .await + .map_err(Details::ReadDouble)?; + Ok(Value::Double(f64::from_le_bytes(buf))) + } + Schema::Bytes => { + let len = decode_len(reader).await?; + let mut buf = vec![0u8; len]; + reader.read_exact(&mut buf).await.map_err(Details::ReadBytes)?; + Ok(Value::Bytes(buf)) + } + Schema::String => { + let len = decode_len(reader).await?; + let mut buf = vec![0u8; len]; + match reader.read_exact(&mut buf).await { + Ok(_) => Ok(Value::String( + String::from_utf8(buf).map_err(Details::ConvertToUtf8)?, + )), + Err(io_err) => { + if let ErrorKind::UnexpectedEof = io_err.kind() { + Ok(Value::Null) + } else { + Err(Details::ReadString(io_err).into()) + } } } } - } - Schema::Fixed(FixedSchema { size, .. }) => { - let mut buf = vec![0u8; size]; - reader - .read_exact(&mut buf) - .map_err(|e| Details::ReadFixed(e, size))?; - Ok(Value::Fixed(size, buf)) - } - Schema::Array(ref inner) => { - let mut items = Vec::new(); + Schema::Fixed(FixedSchema { size, .. }) => { + let mut buf = vec![0u8; size]; + reader + .read_exact(&mut buf) + .await + .map_err(|e| Details::ReadFixed(e, size))?; + Ok(Value::Fixed(size, buf)) + } + Schema::Array(ref inner) => { + let mut items = Vec::new(); - loop { - let len = decode_seq_len(reader)?; - if len == 0 { - break; - } + loop { + let len = decode_seq_len(reader).await?; + if len == 0 { + break; + } - items.reserve(len); - for _ in 0..len { - items.push(decode_internal( - &inner.items, - names, - enclosing_namespace, - reader, - )?); + items.reserve(len); + for _ in 0..len { + items.push( + Box::pin(decode_internal(&inner.items, names, enclosing_namespace, reader)) + .await?, + ); + } } - } - Ok(Value::Array(items)) - } - Schema::Map(ref inner) => { - let mut items = HashMap::new(); + Ok(Value::Array(items)) + } + Schema::Map(ref inner) => { + let mut items = HashMap::new(); - loop { - let len = decode_seq_len(reader)?; - if len == 0 { - break; - } + loop { + let len = decode_seq_len(reader).await?; + if len == 0 { + break; + } - items.reserve(len); - for _ in 0..len { - match decode_internal(&Schema::String, names, enclosing_namespace, reader)? { - Value::String(key) => { - let value = - decode_internal(&inner.types, names, enclosing_namespace, reader)?; - items.insert(key, value); + items.reserve(len); + for _ in 0..len { + match Box::pin(decode_internal(&Schema::String, names, enclosing_namespace, reader)) + .await? + { + Value::String(key) => { + let value = Box::pin(decode_internal( + &inner.types, + names, + enclosing_namespace, + reader, + )) + .await?; + items.insert(key, value); + } + value => return Err(Details::MapKeyType(value.into()).into()), } - value => return Err(Details::MapKeyType(value.into()).into()), } } - } - Ok(Value::Map(items)) - } - Schema::Union(ref inner) => match zag_i64(reader).map_err(Error::into_details) { - Ok(index) => { - let variants = inner.variants(); - let variant = variants - .get(usize::try_from(index).map_err(|e| Details::ConvertI64ToUsize(e, index))?) - .ok_or(Details::GetUnionVariant { - index, - num_variants: variants.len(), - })?; - let value = decode_internal(variant, names, enclosing_namespace, reader)?; - Ok(Value::Union(index as u32, Box::new(value))) + Ok(Value::Map(items)) } - Err(Details::ReadVariableIntegerBytes(io_err)) => { - if let ErrorKind::UnexpectedEof = io_err.kind() { - Ok(Value::Union(0, Box::new(Value::Null))) - } else { - Err(Details::ReadVariableIntegerBytes(io_err).into()) + Schema::Union(ref inner) => match zag_i64(reader).await.map_err(Error::into_details) { + Ok(index) => { + let variants = inner.variants(); + let variant = variants + .get( + usize::try_from(index) + .map_err(|e| Details::ConvertI64ToUsize(e, index))?, + ) + .ok_or(Details::GetUnionVariant { + index, + num_variants: variants.len(), + })?; + let value = + Box::pin(decode_internal(variant, names, enclosing_namespace, reader)).await?; + Ok(Value::Union(index as u32, Box::new(value))) + } + Err(Details::ReadVariableIntegerBytes(io_err)) => { + if let ErrorKind::UnexpectedEof = io_err.kind() { + Ok(Value::Union(0, Box::new(Value::Null))) + } else { + Err(Details::ReadVariableIntegerBytes(io_err).into()) + } + } + Err(io_err) => Err(Error::new(io_err)), + }, + Schema::Record(RecordSchema { + ref name, + ref fields, + .. + }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + // Benchmarks indicate ~10% improvement using this method. + let mut items = Vec::with_capacity(fields.len()); + for field in fields { + // TODO: This clone is also expensive. See if we can do away with it... + items.push(( + field.name.clone(), + Box::pin(decode_internal( + &field.schema, + names, + &fully_qualified_name.namespace, + reader, + )) + .await?, + )); } + Ok(Value::Record(items)) } - Err(io_err) => Err(Error::new(io_err)), - }, - Schema::Record(RecordSchema { - ref name, - ref fields, - .. - }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - // Benchmarks indicate ~10% improvement using this method. - let mut items = Vec::with_capacity(fields.len()); - for field in fields { - // TODO: This clone is also expensive. See if we can do away with it... - items.push(( - field.name.clone(), - decode_internal( - &field.schema, + Schema::Enum(EnumSchema { ref symbols, .. }) => { + Ok(if let Value::Int(raw_index) = decode_int(reader).await? { + let index = usize::try_from(raw_index) + .map_err(|e| Details::ConvertI32ToUsize(e, raw_index))?; + if (0..symbols.len()).contains(&index) { + let symbol = symbols[index].clone(); + Value::Enum(raw_index as u32, symbol) + } else { + return Err(Details::GetEnumValue { + index, + nsymbols: symbols.len(), + } + .into()); + } + } else { + return Err(Details::GetEnumUnknownIndexValue.into()); + }) + } + Schema::Ref { ref name } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if let Some(resolved) = names.get(&fully_qualified_name) { + Box::pin(decode_internal( + resolved.borrow(), names, &fully_qualified_name.namespace, reader, - )?, - )); - } - Ok(Value::Record(items)) - } - Schema::Enum(EnumSchema { ref symbols, .. }) => { - Ok(if let Value::Int(raw_index) = decode_int(reader)? { - let index = usize::try_from(raw_index) - .map_err(|e| Details::ConvertI32ToUsize(e, raw_index))?; - if (0..symbols.len()).contains(&index) { - let symbol = symbols[index].clone(); - Value::Enum(raw_index as u32, symbol) + )) + .await } else { - return Err(Details::GetEnumValue { - index, - nsymbols: symbols.len(), - } - .into()); + Err(Details::SchemaResolutionError(fully_qualified_name).into()) } - } else { - return Err(Details::GetEnumUnknownIndexValue.into()); - }) - } - Schema::Ref { ref name } => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if let Some(resolved) = names.get(&fully_qualified_name) { - decode_internal( - resolved.borrow(), - names, - &fully_qualified_name.namespace, - reader, - ) - } else { - Err(Details::SchemaResolutionError(fully_qualified_name).into()) } } } -} - -#[cfg(test)] -#[allow(clippy::expect_fun_call)] -mod tests { - use crate::{ - Decimal, - decode::decode, - encode::{encode, tests::success}, - schema::{DecimalSchema, FixedSchema, Schema}, - types::{ - Value, - Value::{Array, Int, Map}, - }, - }; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; - use std::collections::HashMap; - use uuid::Uuid; - #[test] - fn test_decode_array_without_size() -> TestResult { - let mut input: &[u8] = &[6, 2, 4, 6, 0]; - let result = decode(&Schema::array(Schema::Int), &mut input); - assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); - - Ok(()) - } + #[cfg(test)] + #[allow(clippy::expect_fun_call)] + mod tests { + use crate::{ + Decimal, + decode::decode, + encode::{encode, tests::success}, + schema::{DecimalSchema, FixedSchema, Schema}, + types::{ + Value, + Value::{Array, Int, Map}, + }, + }; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; + use std::collections::HashMap; + use uuid::Uuid; + + #[tokio::test] + async fn test_decode_array_without_size() -> TestResult { + let mut input: &[u8] = &[6, 2, 4, 6, 0]; + let result = decode(&Schema::array(Schema::Int), &mut input); + assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); + + Ok(()) + } - #[test] - fn test_decode_array_with_size() -> TestResult { - let mut input: &[u8] = &[5, 6, 2, 4, 6, 0]; - let result = decode(&Schema::array(Schema::Int), &mut input); - assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); + #[tokio::test] + async fn test_decode_array_with_size() -> TestResult { + let mut input: &[u8] = &[5, 6, 2, 4, 6, 0]; + let result = decode(&Schema::array(Schema::Int), &mut input); + assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); - Ok(()) - } + Ok(()) + } - #[test] - fn test_decode_map_without_size() -> TestResult { - let mut input: &[u8] = &[0x02, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; - let result = decode(&Schema::map(Schema::Int), &mut input); - let mut expected = HashMap::new(); - expected.insert(String::from("test"), Int(1)); - assert_eq!(Map(expected), result?); + #[tokio::test] + async fn test_decode_map_without_size() -> TestResult { + let mut input: &[u8] = &[0x02, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; + let result = decode(&Schema::map(Schema::Int), &mut input); + let mut expected = HashMap::new(); + expected.insert(String::from("test"), Int(1)); + assert_eq!(Map(expected), result?); - Ok(()) - } + Ok(()) + } - #[test] - fn test_decode_map_with_size() -> TestResult { - let mut input: &[u8] = &[0x01, 0x0C, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; - let result = decode(&Schema::map(Schema::Int), &mut input); - let mut expected = HashMap::new(); - expected.insert(String::from("test"), Int(1)); - assert_eq!(Map(expected), result?); + #[tokio::test] + async fn test_decode_map_with_size() -> TestResult { + let mut input: &[u8] = &[0x01, 0x0C, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; + let result = decode(&Schema::map(Schema::Int), &mut input); + let mut expected = HashMap::new(); + expected.insert(String::from("test"), Int(1)); + assert_eq!(Map(expected), result?); - Ok(()) - } + Ok(()) + } - #[test] - fn test_negative_decimal_value() -> TestResult { - use crate::{encode::encode, schema::Name}; - use num_bigint::ToBigInt; - let inner = Box::new(Schema::Fixed( - FixedSchema::builder() - .name(Name::new("decimal")?) - .size(2) - .build(), - )); - let schema = Schema::Decimal(DecimalSchema { - inner, - precision: 4, - scale: 2, - }); - let bigint = (-423).to_bigint().unwrap(); - let value = Value::Decimal(Decimal::from(bigint.to_signed_bytes_be())); - - let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); - - let mut bytes = &buffer[..]; - let result = decode(&schema, &mut bytes)?; - assert_eq!(result, value); - - Ok(()) - } + #[tokio::test] + async fn test_negative_decimal_value() -> TestResult { + use crate::{encode::encode, schema::Name}; + use num_bigint::ToBigInt; + let inner = Box::new(Schema::Fixed( + FixedSchema::builder() + .name(Name::new("decimal")?) + .size(2) + .build(), + )); + let schema = Schema::Decimal(DecimalSchema { + inner, + precision: 4, + scale: 2, + }); + let bigint = (-423).to_bigint().unwrap(); + let value = Value::Decimal(Decimal::from(bigint.to_signed_bytes_be())); + + let mut buffer = Vec::new(); + encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + + let mut bytes = &buffer[..]; + let result = decode(&schema, &mut bytes)?; + assert_eq!(result, value); + + Ok(()) + } - #[test] - fn test_decode_decimal_with_bigger_than_necessary_size() -> TestResult { - use crate::{encode::encode, schema::Name}; - use num_bigint::ToBigInt; - let inner = Box::new(Schema::Fixed(FixedSchema { - size: 13, - name: Name::new("decimal")?, - aliases: None, - doc: None, - default: None, - attributes: Default::default(), - })); - let schema = Schema::Decimal(DecimalSchema { - inner, - precision: 4, - scale: 2, - }); - let value = Value::Decimal(Decimal::from( - ((-423).to_bigint().unwrap()).to_signed_bytes_be(), - )); - let mut buffer = Vec::::new(); - - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); - let mut bytes: &[u8] = &buffer[..]; - let result = decode(&schema, &mut bytes)?; - assert_eq!(result, value); - - Ok(()) - } + #[tokio::test] + async fn test_decode_decimal_with_bigger_than_necessary_size() -> TestResult { + use crate::{encode::encode, schema::Name}; + use num_bigint::ToBigInt; + let inner = Box::new(Schema::Fixed(FixedSchema { + size: 13, + name: Name::new("decimal")?, + aliases: None, + doc: None, + default: None, + attributes: Default::default(), + })); + let schema = Schema::Decimal(DecimalSchema { + inner, + precision: 4, + scale: 2, + }); + let value = Value::Decimal(Decimal::from( + ((-423).to_bigint().unwrap()).to_signed_bytes_be(), + )); + let mut buffer = Vec::::new(); + + encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + let mut bytes: &[u8] = &buffer[..]; + let result = decode(&schema, &mut bytes)?; + assert_eq!(result, value); + + Ok(()) + } - #[test] - fn test_avro_3448_recursive_definition_decode_union() -> TestResult { - // if encoding fails in this test check the corresponding test in encode - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3448_recursive_definition_decode_union() -> TestResult { + // if encoding fails in this test check the corresponding test in encode + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -505,48 +546,48 @@ mod tests { } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value1 = Value::Record(vec![ - ("a".into(), Value::Union(1, Box::new(inner_value1))), - ("b".into(), inner_value2.clone()), - ]); - let mut buf = Vec::new(); - encode(&outer_value1, &schema, &mut buf).expect(&success(&outer_value1, &schema)); - assert!(!buf.is_empty()); - let mut bytes = &buf[..]; - assert_eq!( - outer_value1, - decode(&schema, &mut bytes).expect(&format!( - "Failed to decode using recursive definitions with schema:\n {:?}\n", - &schema - )) - ); - - let mut buf = Vec::new(); - let outer_value2 = Value::Record(vec![ - ("a".into(), Value::Union(0, Box::new(Value::Null))), - ("b".into(), inner_value2), - ]); - encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value2, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_value2, - decode(&schema, &mut bytes).expect(&format!( - "Failed to decode using recursive definitions with schema:\n {:?}\n", - &schema - )) - ); - - Ok(()) - } + )?; + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value1 = Value::Record(vec![ + ("a".into(), Value::Union(1, Box::new(inner_value1))), + ("b".into(), inner_value2.clone()), + ]); + let mut buf = Vec::new(); + encode(&outer_value1, &schema, &mut buf).expect(&success(&outer_value1, &schema)); + assert!(!buf.is_empty()); + let mut bytes = &buf[..]; + assert_eq!( + outer_value1, + decode(&schema, &mut bytes).expect(&format!( + "Failed to decode using recursive definitions with schema:\n {:?}\n", + &schema + )) + ); + + let mut buf = Vec::new(); + let outer_value2 = Value::Record(vec![ + ("a".into(), Value::Union(0, Box::new(Value::Null))), + ("b".into(), inner_value2), + ]); + encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value2, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_value2, + decode(&schema, &mut bytes).expect(&format!( + "Failed to decode using recursive definitions with schema:\n {:?}\n", + &schema + )) + ); + + Ok(()) + } - #[test] - fn test_avro_3448_recursive_definition_decode_array() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3448_recursive_definition_decode_array() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -571,32 +612,32 @@ mod tests { } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ("a".into(), Value::Array(vec![inner_value1])), - ("b".into(), inner_value2), - ]); - let mut buf = Vec::new(); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_value, - decode(&schema, &mut bytes).expect(&format!( - "Failed to decode using recursive definitions with schema:\n {:?}\n", - &schema - )) - ); - - Ok(()) - } + )?; + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ("a".into(), Value::Array(vec![inner_value1])), + ("b".into(), inner_value2), + ]); + let mut buf = Vec::new(); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_value, + decode(&schema, &mut bytes).expect(&format!( + "Failed to decode using recursive definitions with schema:\n {:?}\n", + &schema + )) + ); + + Ok(()) + } - #[test] - fn test_avro_3448_recursive_definition_decode_map() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3448_recursive_definition_decode_map() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -621,35 +662,35 @@ mod tests { } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ( - "a".into(), - Value::Map(vec![("akey".into(), inner_value1)].into_iter().collect()), - ), - ("b".into(), inner_value2), - ]); - let mut buf = Vec::new(); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_value, - decode(&schema, &mut bytes).expect(&format!( - "Failed to decode using recursive definitions with schema:\n {:?}\n", - &schema - )) - ); - - Ok(()) - } + )?; + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ( + "a".into(), + Value::Map(vec![("akey".into(), inner_value1)].into_iter().collect()), + ), + ("b".into(), inner_value2), + ]); + let mut buf = Vec::new(); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_value, + decode(&schema, &mut bytes).expect(&format!( + "Failed to decode using recursive definitions with schema:\n {:?}\n", + &schema + )) + ); + + Ok(()) + } - #[test] - fn test_avro_3448_proper_multi_level_decoding_middle_namespace() -> TestResult { - // if encoding fails in this test check the corresponding test in encode - let schema = r#" + #[tokio::test] + async fn test_avro_3448_proper_multi_level_decoding_middle_namespace() -> TestResult { + // if encoding fails in this test check the corresponding test in encode + let schema = r#" { "name": "record_name", "namespace": "space", @@ -691,81 +732,81 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema)?; + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) - .expect(&success(&outer_record_variation_1, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_record_variation_1, - decode(&schema, &mut bytes).expect(&format!( - "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", - &schema - )) - ); - - let mut buf = Vec::new(); - encode(&outer_record_variation_2, &schema, &mut buf) - .expect(&success(&outer_record_variation_2, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_record_variation_2, - decode(&schema, &mut bytes).expect(&format!( - "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", - &schema - )) - ); - - let mut buf = Vec::new(); - encode(&outer_record_variation_3, &schema, &mut buf) - .expect(&success(&outer_record_variation_3, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_record_variation_3, - decode(&schema, &mut bytes).expect(&format!( - "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", - &schema - )) - ); - - Ok(()) - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + let mut buf = Vec::new(); + encode(&outer_record_variation_1, &schema, &mut buf) + .expect(&success(&outer_record_variation_1, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_record_variation_1, + decode(&schema, &mut bytes).expect(&format!( + "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", + &schema + )) + ); + + let mut buf = Vec::new(); + encode(&outer_record_variation_2, &schema, &mut buf) + .expect(&success(&outer_record_variation_2, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_record_variation_2, + decode(&schema, &mut bytes).expect(&format!( + "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", + &schema + )) + ); + + let mut buf = Vec::new(); + encode(&outer_record_variation_3, &schema, &mut buf) + .expect(&success(&outer_record_variation_3, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_record_variation_3, + decode(&schema, &mut bytes).expect(&format!( + "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", + &schema + )) + ); + + Ok(()) + } - #[test] - fn test_avro_3448_proper_multi_level_decoding_inner_namespace() -> TestResult { - // if encoding fails in this test check the corresponding test in encode - let schema = r#" + #[tokio::test] + async fn test_avro_3448_proper_multi_level_decoding_inner_namespace() -> TestResult { + // if encoding fails in this test check the corresponding test in encode + let schema = r#" { "name": "record_name", "namespace": "space", @@ -808,113 +849,114 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema)?; + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) - .expect(&success(&outer_record_variation_1, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_record_variation_1, - decode(&schema, &mut bytes).expect(&format!( - "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", - &schema - )) - ); - - let mut buf = Vec::new(); - encode(&outer_record_variation_2, &schema, &mut buf) - .expect(&success(&outer_record_variation_2, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_record_variation_2, - decode(&schema, &mut bytes).expect(&format!( - "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", - &schema - )) - ); - - let mut buf = Vec::new(); - encode(&outer_record_variation_3, &schema, &mut buf) - .expect(&success(&outer_record_variation_3, &schema)); - let mut bytes = &buf[..]; - assert_eq!( - outer_record_variation_3, - decode(&schema, &mut bytes).expect(&format!( - "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", - &schema - )) - ); - - Ok(()) - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + let mut buf = Vec::new(); + encode(&outer_record_variation_1, &schema, &mut buf) + .expect(&success(&outer_record_variation_1, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_record_variation_1, + decode(&schema, &mut bytes).expect(&format!( + "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", + &schema + )) + ); + + let mut buf = Vec::new(); + encode(&outer_record_variation_2, &schema, &mut buf) + .expect(&success(&outer_record_variation_2, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_record_variation_2, + decode(&schema, &mut bytes).expect(&format!( + "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", + &schema + )) + ); + + let mut buf = Vec::new(); + encode(&outer_record_variation_3, &schema, &mut buf) + .expect(&success(&outer_record_variation_3, &schema)); + let mut bytes = &buf[..]; + assert_eq!( + outer_record_variation_3, + decode(&schema, &mut bytes).expect(&format!( + "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", + &schema + )) + ); + + Ok(()) + } - #[test] - fn avro_3926_encode_decode_uuid_to_string() -> TestResult { - use crate::encode::encode; + #[tokio::test] + async fn avro_3926_encode_decode_uuid_to_string() -> TestResult { + use crate::encode::encode; - let schema = Schema::String; - let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); + let schema = Schema::String; + let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); - let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + let mut buffer = Vec::new(); + encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); - let result = decode(&Schema::Uuid, &mut &buffer[..])?; - assert_eq!(result, value); + let result = decode(&Schema::Uuid, &mut &buffer[..])?; + assert_eq!(result, value); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3926_encode_decode_uuid_to_fixed() -> TestResult { - use crate::encode::encode; + #[tokio::test] + async fn avro_3926_encode_decode_uuid_to_fixed() -> TestResult { + use crate::encode::encode; - let schema = Schema::Fixed(FixedSchema { - size: 16, - name: "uuid".into(), - aliases: None, - doc: None, - default: None, - attributes: Default::default(), - }); - let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); + let schema = Schema::Fixed(FixedSchema { + size: 16, + name: "uuid".into(), + aliases: None, + doc: None, + default: None, + attributes: Default::default(), + }); + let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); - let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + let mut buffer = Vec::new(); + encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); - let result = decode(&Schema::Uuid, &mut &buffer[..])?; - assert_eq!(result, value); + let result = decode(&Schema::Uuid, &mut &buffer[..])?; + assert_eq!(result, value); - Ok(()) + Ok(()) + } } } diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 38203f9d..ed41c44a 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -24,8 +24,11 @@ use crate::{ Schema, SchemaKind, UnionSchema, }, types::{Value, ValueKind}, - util::{zig_i32, zig_i64}, }; +#[cfg(feature = "tokio")] +use crate::util::tokio::{zig_i32, zig_i64}; +#[cfg(feature = "sync")] +use crate::util::sync::{zig_i32, zig_i64}; use log::error; use std::{borrow::Borrow, collections::HashMap, io::Write}; diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 36a31e6e..a93d661c 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -900,7 +900,14 @@ pub use de::from_value; pub use decimal::Decimal; pub use duration::{Days, Duration, Millis, Months}; pub use error::Error; -pub use reader::{ + +#[cfg(feature = "sync")] +pub use reader::sync::{ + GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, + from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, +}; +#[cfg(feature = "tokio")] +pub use reader::tokio::{ GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, }; @@ -919,6 +926,18 @@ pub use apache_avro_derive::*; /// A convenience type alias for `Result`s with `Error`s. pub type AvroResult = Result; +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tests_tokio { }, + #[cfg(feature = "sync")] + pub mod tests_sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + #[tokio::test] => #[test] + ); + } +)] #[cfg(test)] mod tests { use crate::{ @@ -928,8 +947,8 @@ mod tests { use pretty_assertions::assert_eq; //TODO: move where it fits better - #[test] - fn test_enum_default() { + #[tokio::test] + async fn test_enum_default() { let writer_raw_schema = r#" { "type": "record", @@ -967,16 +986,18 @@ mod tests { record.put("b", "foo"); writer.append(record).unwrap(); let input = writer.into_inner().unwrap(); - let mut reader = Reader::with_schema(&reader_schema, &input[..]).unwrap(); + let mut reader = Reader::with_schema(&reader_schema, &input[..]) + .await + .unwrap(); assert_eq!( - reader.next().unwrap().unwrap(), + reader.next().await.unwrap().unwrap(), Value::Record(vec![ ("a".to_string(), Value::Long(27)), ("b".to_string(), Value::String("foo".to_string())), ("c".to_string(), Value::Enum(1, "spades".to_string())), ]) ); - assert!(reader.next().is_none()); + assert!(reader.next().await.is_none()); } //TODO: move where it fits better @@ -1022,8 +1043,8 @@ mod tests { } //TODO: move where it fits better - #[test] - fn test_enum_no_reader_schema() { + #[tokio::test] + async fn test_enum_no_reader_schema() { let writer_raw_schema = r#" { "type": "record", @@ -1053,7 +1074,7 @@ mod tests { let input = writer.into_inner().unwrap(); let mut reader = Reader::new(&input[..]).unwrap(); assert_eq!( - reader.next().unwrap().unwrap(), + reader.next().await.unwrap().unwrap(), Value::Record(vec![ ("a".to_string(), Value::Long(27)), ("b".to_string(), Value::String("foo".to_string())), diff --git a/avro/src/reader.rs b/avro/src/reader.rs index e2f7570f..2c8e96b8 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -15,599 +15,653 @@ // specific language governing permissions and limitations // under the License. -//! Logic handling reading from Avro format at user level. -use crate::{ - AvroResult, Codec, Error, - decode::{decode, decode_internal}, - error::Details, - from_value, - headers::{HeaderBuilder, RabinFingerprintHeader}, - schema::{ - AvroSchema, Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, - resolve_names_with_schemata, - }, - types::Value, - util, -}; -use log::warn; -use serde::de::DeserializeOwned; -use serde_json::from_slice; -use std::{ - collections::HashMap, - io::{ErrorKind, Read}, - marker::PhantomData, - str::FromStr, -}; - -/// Internal Block reader. -#[derive(Debug, Clone)] -struct Block<'r, R> { - reader: R, - /// Internal buffering to reduce allocation. - buf: Vec, - buf_idx: usize, - /// Number of elements expected to exist within this block. - message_count: usize, - marker: [u8; 16], - codec: Codec, - writer_schema: Schema, - schemata: Vec<&'r Schema>, - user_metadata: HashMap>, - names_refs: Names, -} - -impl<'r, R: Read> Block<'r, R> { - fn new(reader: R, schemata: Vec<&'r Schema>) -> AvroResult> { - let mut block = Block { - reader, - codec: Codec::Null, - writer_schema: Schema::Null, - schemata, - buf: vec![], - buf_idx: 0, - message_count: 0, - marker: [0; 16], - user_metadata: Default::default(), - names_refs: Default::default(), - }; - - block.read_header()?; - Ok(block) +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + decode::tokio::decode => decode::sync::decode, + decode::tokio::decode_internal => decode::sync::decode_internal, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod reader { + #[synca::cfg(tokio)] + use tokio::io::AsyncReadExt; + + use crate::{ + AvroResult, Codec, Error, + decode::tokio::{decode, decode_internal}, + error::Details, + from_value, + headers::{HeaderBuilder, RabinFingerprintHeader}, + schema::{ + AvroSchema, Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, + resolve_names_with_schemata, + }, + types::Value, + util, + }; + use futures::Stream; + use log::warn; + use serde::de::DeserializeOwned; + use serde_json::from_slice; + use std::pin::Pin; + use std::task::Poll; + use std::{ + collections::HashMap, + io::ErrorKind, + marker::PhantomData, + str::FromStr, + }; + /// Internal Block reader. + + #[derive(Debug, Clone)] + struct Block<'r, R> { + reader: R, + /// Internal buffering to reduce allocation. + buf: Vec, + buf_idx: usize, + /// Number of elements expected to exist within this block. + message_count: usize, + marker: [u8; 16], + codec: Codec, + writer_schema: Schema, + schemata: Vec<&'r Schema>, + user_metadata: HashMap>, + names_refs: Names, } - /// Try to read the header and to set the writer `Schema`, the `Codec` and the marker based on - /// its content. - fn read_header(&mut self) -> AvroResult<()> { - let mut buf = [0u8; 4]; - self.reader - .read_exact(&mut buf) - .map_err(Details::ReadHeader)?; - - if buf != [b'O', b'b', b'j', 1u8] { - return Err(Details::HeaderMagic.into()); + impl<'r, R: tokio::io::AsyncRead + Unpin> Block<'r, R> { + async fn new(reader: R, schemata: Vec<&'r Schema>) -> AvroResult> { + let mut block = Block { + reader, + codec: Codec::Null, + writer_schema: Schema::Null, + schemata, + buf: vec![], + buf_idx: 0, + message_count: 0, + marker: [0; 16], + user_metadata: Default::default(), + names_refs: Default::default(), + }; + + block.read_header().await?; + Ok(block) } - let meta_schema = Schema::map(Schema::Bytes); - match decode(&meta_schema, &mut self.reader)? { - Value::Map(metadata) => { - self.read_writer_schema(&metadata)?; - self.codec = read_codec(&metadata)?; + /// Try to read the header and to set the writer `Schema`, the `Codec` and the marker based on + /// its content. + async fn read_header(&mut self) -> AvroResult<()> { + let mut buf = [0u8; 4]; + self.reader + .read_exact(&mut buf) + .await + .map_err(Details::ReadHeader)?; + + if buf != [b'O', b'b', b'j', 1u8] { + return Err(Details::HeaderMagic.into()); + } - for (key, value) in metadata { - if key == "avro.schema" - || key == "avro.codec" - || key == "avro.codec.compression_level" - { - // already processed - } else if key.starts_with("avro.") { - warn!("Ignoring unknown metadata key: {key}"); - } else { - self.read_user_metadata(key, value); + let meta_schema = Schema::map(Schema::Bytes); + match decode(&meta_schema, &mut self.reader).await? { + Value::Map(metadata) => { + self.read_writer_schema(&metadata)?; + self.codec = read_codec(&metadata)?; + + for (key, value) in metadata { + if key == "avro.schema" + || key == "avro.codec" + || key == "avro.codec.compression_level" + { + // already processed + } else if key.starts_with("avro.") { + warn!("Ignoring unknown metadata key: {key}"); + } else { + self.read_user_metadata(key, value); + } } } + _ => { + return Err(Details::GetHeaderMetadata.into()); + } } - _ => { - return Err(Details::GetHeaderMetadata.into()); - } + + self.reader + .read_exact(&mut self.marker) + .await + .map_err(|e| Details::ReadMarker(e).into()) + .map(|_| ()) } - self.reader - .read_exact(&mut self.marker) - .map_err(|e| Details::ReadMarker(e).into()) - } + async fn fill_buf(&mut self, n: usize) -> AvroResult<()> { + // The buffer needs to contain exactly `n` elements, otherwise codecs will potentially read + // invalid bytes. + // + // The are two cases to handle here: + // + // 1. `n > self.buf.len()`: + // In this case we call `Vec::resize`, which guarantees that `self.buf.len() == n`. + // 2. `n < self.buf.len()`: + // We need to resize to ensure that the buffer len is safe to read `n` elements. + // + // TODO: Figure out a way to avoid having to truncate for the second case. + self.buf.resize(util::safe_len(n)?, 0); + self.reader + .read_exact(&mut self.buf) + .await + .map_err(Details::ReadIntoBuf)?; + self.buf_idx = 0; + Ok(()) + } - fn fill_buf(&mut self, n: usize) -> AvroResult<()> { - // The buffer needs to contain exactly `n` elements, otherwise codecs will potentially read - // invalid bytes. - // - // The are two cases to handle here: - // - // 1. `n > self.buf.len()`: - // In this case we call `Vec::resize`, which guarantees that `self.buf.len() == n`. - // 2. `n < self.buf.len()`: - // We need to resize to ensure that the buffer len is safe to read `n` elements. - // - // TODO: Figure out a way to avoid having to truncate for the second case. - self.buf.resize(util::safe_len(n)?, 0); - self.reader - .read_exact(&mut self.buf) - .map_err(Details::ReadIntoBuf)?; - self.buf_idx = 0; - Ok(()) - } + /// Try to read a data block, also performing schema resolution for the objects contained in + /// the block. The objects are stored in an internal buffer to the `Reader`. + async fn read_block_next(&mut self) -> AvroResult<()> { + assert!(self.is_empty(), "Expected self to be empty!"); + match util::tokio::read_long(&mut self.reader) + .await + .map_err(Error::into_details) + { + Ok(block_len) => { + self.message_count = block_len as usize; + let block_bytes = util::tokio::read_long(&mut self.reader).await?; + self.fill_buf(block_bytes as usize).await?; + let mut marker = [0u8; 16]; + self.reader + .read_exact(&mut marker).await + .map_err(Details::ReadBlockMarker)?; + + if marker != self.marker { + return Err(Details::GetBlockMarker.into()); + } - /// Try to read a data block, also performing schema resolution for the objects contained in - /// the block. The objects are stored in an internal buffer to the `Reader`. - fn read_block_next(&mut self) -> AvroResult<()> { - assert!(self.is_empty(), "Expected self to be empty!"); - match util::read_long(&mut self.reader).map_err(Error::into_details) { - Ok(block_len) => { - self.message_count = block_len as usize; - let block_bytes = util::read_long(&mut self.reader)?; - self.fill_buf(block_bytes as usize)?; - let mut marker = [0u8; 16]; - self.reader - .read_exact(&mut marker) - .map_err(Details::ReadBlockMarker)?; - - if marker != self.marker { - return Err(Details::GetBlockMarker.into()); + // NOTE (JAB): This doesn't fit this Reader pattern very well. + // `self.buf` is a growable buffer that is reused as the reader is iterated. + // For non `Codec::Null` variants, `decompress` will allocate a new `Vec` + // and replace `buf` with the new one, instead of reusing the same buffer. + // We can address this by using some "limited read" type to decode directly + // into the buffer. But this is fine, for now. + self.codec.decompress(&mut self.buf) } - - // NOTE (JAB): This doesn't fit this Reader pattern very well. - // `self.buf` is a growable buffer that is reused as the reader is iterated. - // For non `Codec::Null` variants, `decompress` will allocate a new `Vec` - // and replace `buf` with the new one, instead of reusing the same buffer. - // We can address this by using some "limited read" type to decode directly - // into the buffer. But this is fine, for now. - self.codec.decompress(&mut self.buf) - } - Err(Details::ReadVariableIntegerBytes(io_err)) => { - if let ErrorKind::UnexpectedEof = io_err.kind() { - // to not return any error in case we only finished to read cleanly from the stream - Ok(()) - } else { - Err(Details::ReadVariableIntegerBytes(io_err).into()) + Err(Details::ReadVariableIntegerBytes(io_err)) => { + if let ErrorKind::UnexpectedEof = io_err.kind() { + // to not return any error in case we only finished to read cleanly from the stream + Ok(()) + } else { + Err(Details::ReadVariableIntegerBytes(io_err).into()) + } } + Err(e) => Err(Error::new(e)), } - Err(e) => Err(Error::new(e)), } - } - fn len(&self) -> usize { - self.message_count - } + fn len(&self) -> usize { + self.message_count + } - fn is_empty(&self) -> bool { - self.len() == 0 - } + fn is_empty(&self) -> bool { + self.len() == 0 + } - fn read_next(&mut self, read_schema: Option<&Schema>) -> AvroResult> { - if self.is_empty() { - self.read_block_next()?; + async fn read_next(&mut self, read_schema: Option<&Schema>) -> AvroResult> { if self.is_empty() { - return Ok(None); + self.read_block_next().await?; + if self.is_empty() { + return Ok(None); + } } - } - let mut block_bytes = &self.buf[self.buf_idx..]; - let b_original = block_bytes.len(); - - let item = decode_internal( - &self.writer_schema, - &self.names_refs, - &None, - &mut block_bytes, - )?; - let item = match read_schema { - Some(schema) => item.resolve(schema)?, - None => item, - }; - - if b_original != 0 && b_original == block_bytes.len() { - // from_avro_datum did not consume any bytes, so return an error to avoid an infinite loop - return Err(Details::ReadBlock.into()); + let mut block_bytes = &self.buf[self.buf_idx..]; + let b_original = block_bytes.len(); + + let item = decode_internal( + &self.writer_schema, + &self.names_refs, + &None, + &mut block_bytes, + ) + .await?; + let item = match read_schema { + Some(schema) => item.resolve(schema).await?, + None => item, + }; + + if b_original != 0 && b_original == block_bytes.len() { + // from_avro_datum did not consume any bytes, so return an error to avoid an infinite loop + return Err(Details::ReadBlock.into()); + } + self.buf_idx += b_original - block_bytes.len(); + self.message_count -= 1; + Ok(Some(item)) } - self.buf_idx += b_original - block_bytes.len(); - self.message_count -= 1; - Ok(Some(item)) - } - fn read_writer_schema(&mut self, metadata: &HashMap) -> AvroResult<()> { - let json: serde_json::Value = metadata - .get("avro.schema") - .and_then(|bytes| { - if let Value::Bytes(ref bytes) = *bytes { - from_slice(bytes.as_ref()).ok() - } else { - None - } - }) - .ok_or(Details::GetAvroSchemaFromMap)?; - if !self.schemata.is_empty() { - let rs = ResolvedSchema::try_from(self.schemata.clone())?; - let names: Names = rs - .get_names() - .iter() - .map(|(name, schema)| (name.clone(), (*schema).clone())) - .collect(); - self.writer_schema = Schema::parse_with_names(&json, names)?; - resolve_names_with_schemata(&self.schemata, &mut self.names_refs, &None)?; - } else { - self.writer_schema = Schema::parse(&json)?; - resolve_names(&self.writer_schema, &mut self.names_refs, &None)?; + fn read_writer_schema(&mut self, metadata: &HashMap) -> AvroResult<()> { + let json: serde_json::Value = metadata + .get("avro.schema") + .and_then(|bytes| { + if let Value::Bytes(ref bytes) = *bytes { + from_slice(bytes.as_ref()).ok() + } else { + None + } + }) + .ok_or(Details::GetAvroSchemaFromMap)?; + if !self.schemata.is_empty() { + let rs = ResolvedSchema::try_from(self.schemata.clone())?; + let names: Names = rs + .get_names() + .iter() + .map(|(name, schema)| (name.clone(), (*schema).clone())) + .collect(); + self.writer_schema = Schema::parse_with_names(&json, names)?; + resolve_names_with_schemata(&self.schemata, &mut self.names_refs, &None)?; + } else { + self.writer_schema = Schema::parse(&json)?; + resolve_names(&self.writer_schema, &mut self.names_refs, &None)?; + } + Ok(()) } - Ok(()) - } - fn read_user_metadata(&mut self, key: String, value: Value) { - match value { - Value::Bytes(ref vec) => { - self.user_metadata.insert(key, vec.clone()); - } - wrong => { - warn!("User metadata values must be Value::Bytes, found {wrong:?}"); + fn read_user_metadata(&mut self, key: String, value: Value) { + match value { + Value::Bytes(ref vec) => { + self.user_metadata.insert(key, vec.clone()); + } + wrong => { + warn!("User metadata values must be Value::Bytes, found {wrong:?}"); + } } } } -} -fn read_codec(metadata: &HashMap) -> AvroResult { - let result = metadata - .get("avro.codec") - .map(|codec| { - if let Value::Bytes(ref bytes) = *codec { - match std::str::from_utf8(bytes.as_ref()) { - Ok(utf8) => Ok(utf8), - Err(utf8_error) => Err(Details::ConvertToUtf8Error(utf8_error).into()), + fn read_codec(metadata: &HashMap) -> AvroResult { + let result = metadata + .get("avro.codec") + .map(|codec| { + if let Value::Bytes(ref bytes) = *codec { + match std::str::from_utf8(bytes.as_ref()) { + Ok(utf8) => Ok(utf8), + Err(utf8_error) => Err(Details::ConvertToUtf8Error(utf8_error).into()), + } + } else { + Err(Details::BadCodecMetadata.into()) } - } else { - Err(Details::BadCodecMetadata.into()) - } - }) - .map(|codec_res| match codec_res { - Ok(codec) => match Codec::from_str(codec) { - Ok(codec) => match codec { - #[cfg(feature = "bzip")] - Codec::Bzip2(_) => { - use crate::Bzip2Settings; - if let Some(Value::Bytes(bytes)) = - metadata.get("avro.codec.compression_level") - { - Ok(Codec::Bzip2(Bzip2Settings::new(bytes[0]))) - } else { - Ok(codec) + }) + .map(|codec_res| match codec_res { + Ok(codec) => match Codec::from_str(codec) { + Ok(codec) => match codec { + #[cfg(feature = "bzip")] + Codec::Bzip2(_) => { + use crate::Bzip2Settings; + if let Some(Value::Bytes(bytes)) = + metadata.get("avro.codec.compression_level") + { + Ok(Codec::Bzip2(Bzip2Settings::new(bytes[0]))) + } else { + Ok(codec) + } } - } - #[cfg(feature = "xz")] - Codec::Xz(_) => { - use crate::XzSettings; - if let Some(Value::Bytes(bytes)) = - metadata.get("avro.codec.compression_level") - { - Ok(Codec::Xz(XzSettings::new(bytes[0]))) - } else { - Ok(codec) + #[cfg(feature = "xz")] + Codec::Xz(_) => { + use crate::XzSettings; + if let Some(Value::Bytes(bytes)) = + metadata.get("avro.codec.compression_level") + { + Ok(Codec::Xz(XzSettings::new(bytes[0]))) + } else { + Ok(codec) + } } - } - #[cfg(feature = "zstandard")] - Codec::Zstandard(_) => { - use crate::ZstandardSettings; - if let Some(Value::Bytes(bytes)) = - metadata.get("avro.codec.compression_level") - { - Ok(Codec::Zstandard(ZstandardSettings::new(bytes[0]))) - } else { - Ok(codec) + #[cfg(feature = "zstandard")] + Codec::Zstandard(_) => { + use crate::ZstandardSettings; + if let Some(Value::Bytes(bytes)) = + metadata.get("avro.codec.compression_level") + { + Ok(Codec::Zstandard(ZstandardSettings::new(bytes[0]))) + } else { + Ok(codec) + } } - } - _ => Ok(codec), + _ => Ok(codec), + }, + Err(_) => Err(Details::CodecNotSupported(codec.to_owned()).into()), }, - Err(_) => Err(Details::CodecNotSupported(codec.to_owned()).into()), - }, - Err(err) => Err(err), - }); - - result.unwrap_or(Ok(Codec::Null)) -} + Err(err) => Err(err), + }); -/// Main interface for reading Avro formatted values. -/// -/// To be used as an iterator: -/// -/// ```no_run -/// # use apache_avro::Reader; -/// # use std::io::Cursor; -/// # let input = Cursor::new(Vec::::new()); -/// for value in Reader::new(input).unwrap() { -/// match value { -/// Ok(v) => println!("{:?}", v), -/// Err(e) => println!("Error: {}", e), -/// }; -/// } -/// ``` -pub struct Reader<'a, R> { - block: Block<'a, R>, - reader_schema: Option<&'a Schema>, - errored: bool, - should_resolve_schema: bool, -} - -impl<'a, R: Read> Reader<'a, R> { - /// Creates a `Reader` given something implementing the `io::Read` trait to read from. - /// No reader `Schema` will be set. - /// - /// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`. - pub fn new(reader: R) -> AvroResult> { - let block = Block::new(reader, vec![])?; - let reader = Reader { - block, - reader_schema: None, - errored: false, - should_resolve_schema: false, - }; - Ok(reader) + result.unwrap_or(Ok(Codec::Null)) } - /// Creates a `Reader` given a reader `Schema` and something implementing the `io::Read` trait - /// to read from. + /// Main interface for reading Avro formatted values. /// - /// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`. - pub fn with_schema(schema: &'a Schema, reader: R) -> AvroResult> { - let block = Block::new(reader, vec![schema])?; - let mut reader = Reader { - block, - reader_schema: Some(schema), - errored: false, - should_resolve_schema: false, - }; - // Check if the reader and writer schemas disagree. - reader.should_resolve_schema = reader.writer_schema() != schema; - Ok(reader) - } - - /// Creates a `Reader` given a reader `Schema` and something implementing the `io::Read` trait - /// to read from. + /// To be used as an iterator: /// - /// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`. - pub fn with_schemata( - schema: &'a Schema, - schemata: Vec<&'a Schema>, - reader: R, - ) -> AvroResult> { - let block = Block::new(reader, schemata)?; - let mut reader = Reader { - block, - reader_schema: Some(schema), - errored: false, - should_resolve_schema: false, - }; - // Check if the reader and writer schemas disagree. - reader.should_resolve_schema = reader.writer_schema() != schema; - Ok(reader) + /// ```no_run + /// use apache_avro::Reader; + /// use std::io::Cursor; + /// async { + /// let input = Cursor::new(Vec::::new()); + /// for value in Reader::new(input).await.unwrap() { + /// match value { + /// Ok(v) => println!("{:?}", v), + /// Err(e) => println!("Error: {}", e), + /// }; + /// } + /// } + /// ``` + pub struct Reader<'a, R> { + block: Block<'a, R>, + reader_schema: Option<&'a Schema>, + errored: bool, + should_resolve_schema: bool, } - /// Get a reference to the writer `Schema`. - #[inline] - pub fn writer_schema(&self) -> &Schema { - &self.block.writer_schema - } + impl<'a, R: tokio::io::AsyncRead + Unpin> Reader<'a, R> { + /// Creates a `Reader` given something implementing the `io::Read` trait to read from. + /// No reader `Schema` will be set. + /// + /// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`. + pub async fn new(reader: R) -> AvroResult> { + let block = Block::new(reader, vec![]).await?; + let reader = Reader { + block, + reader_schema: None, + errored: false, + should_resolve_schema: false, + }; + Ok(reader) + } - /// Get a reference to the optional reader `Schema`. - #[inline] - pub fn reader_schema(&self) -> Option<&Schema> { - self.reader_schema - } + /// Creates a `Reader` given a reader `Schema` and something implementing the `io::Read` trait + /// to read from. + /// + /// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`. + pub async fn with_schema(schema: &'a Schema, reader: R) -> AvroResult> { + let block = Block::new(reader, vec![schema]).await?; + let mut reader = Reader { + block, + reader_schema: Some(schema), + errored: false, + should_resolve_schema: false, + }; + // Check if the reader and writer schemas disagree. + reader.should_resolve_schema = reader.writer_schema() != schema; + Ok(reader) + } - /// Get a reference to the user metadata - #[inline] - pub fn user_metadata(&self) -> &HashMap> { - &self.block.user_metadata - } + /// Creates a `Reader` given a reader `Schema` and something implementing the `io::Read` trait + /// to read from. + /// + /// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`. + pub async fn with_schemata( + schema: &'a Schema, + schemata: Vec<&'a Schema>, + reader: R, + ) -> AvroResult> { + let block = Block::new(reader, schemata).await?; + let mut reader = Reader { + block, + reader_schema: Some(schema), + errored: false, + should_resolve_schema: false, + }; + // Check if the reader and writer schemas disagree. + reader.should_resolve_schema = reader.writer_schema() != schema; + Ok(reader) + } + + /// Get a reference to the writer `Schema`. + #[inline] + pub fn writer_schema(&self) -> &Schema { + &self.block.writer_schema + } - #[inline] - fn read_next(&mut self) -> AvroResult> { - let read_schema = if self.should_resolve_schema { + /// Get a reference to the optional reader `Schema`. + #[inline] + pub fn reader_schema(&self) -> Option<&Schema> { self.reader_schema - } else { - None - }; + } + + /// Get a reference to the user metadata + #[inline] + pub fn user_metadata(&self) -> &HashMap> { + &self.block.user_metadata + } - self.block.read_next(read_schema) + #[inline] + async fn read_next(&mut self) -> AvroResult> { + let read_schema = if self.should_resolve_schema { + self.reader_schema + } else { + None + }; + + self.block.read_next(read_schema).await + } } -} -impl Iterator for Reader<'_, R> { - type Item = AvroResult; - - fn next(&mut self) -> Option { - // to prevent keep on reading after the first error occurs - if self.errored { - return None; - }; - match self.read_next() { - Ok(opt) => opt.map(Ok), - Err(e) => { - self.errored = true; - Some(Err(e)) + #[cfg(feature = "tokio")] + impl Stream for Reader<'_, R> { + type Item = AvroResult; + async fn poll_next( + mut self: Pin<&mut Self>, + _cx: &mut futures::task::Context<'_>, + ) -> Poll>> { + // to prevent keep on reading after the first error occurs + if self.errored { + return Poll::Ready(None); + }; + match self.read_next().await { + Ok(opt) => Poll::Ready(opt.map(Ok)), + Err(e) => { + self.errored = true; + Poll::Ready(Some(Err(e))) + } } } } -} -/// Decode a `Value` encoded in Avro format given its `Schema` and anything implementing `io::Read` -/// to read from. -/// -/// In case a reader `Schema` is provided, schema resolution will also be performed. -/// -/// **NOTE** This function has a quite small niche of usage and does NOT take care of reading the -/// header and consecutive data blocks; use [`Reader`](struct.Reader.html) if you don't know what -/// you are doing, instead. -pub fn from_avro_datum( - writer_schema: &Schema, - reader: &mut R, - reader_schema: Option<&Schema>, -) -> AvroResult { - let value = decode(writer_schema, reader)?; - match reader_schema { - Some(schema) => value.resolve(schema), - None => Ok(value), + #[cfg(feature = "sync")] + impl Iterator for Reader<'_, R> { + type Item = AvroResult; + + fn next(&mut self) -> Option { + // to prevent keep on reading after the first error occurs + if self.errored { + return None; + }; + match self.read_next() { + Ok(opt) => opt.map(Ok), + Err(e) => { + self.errored = true; + Some(Err(e)) + } + } + } } -} - -/// Decode a `Value` encoded in Avro format given the provided `Schema` and anything implementing `io::Read` -/// to read from. -/// If the writer schema is incomplete, i.e. contains `Schema::Ref`s then it will use the provided -/// schemata to resolve any dependencies. -/// -/// In case a reader `Schema` is provided, schema resolution will also be performed. -pub fn from_avro_datum_schemata( - writer_schema: &Schema, - writer_schemata: Vec<&Schema>, - reader: &mut R, - reader_schema: Option<&Schema>, -) -> AvroResult { - from_avro_datum_reader_schemata( - writer_schema, - writer_schemata, - reader, - reader_schema, - Vec::with_capacity(0), - ) -} -/// Decode a `Value` encoded in Avro format given the provided `Schema` and anything implementing `io::Read` -/// to read from. -/// If the writer schema is incomplete, i.e. contains `Schema::Ref`s then it will use the provided -/// schemata to resolve any dependencies. -/// -/// In case a reader `Schema` is provided, schema resolution will also be performed. -pub fn from_avro_datum_reader_schemata( - writer_schema: &Schema, - writer_schemata: Vec<&Schema>, - reader: &mut R, - reader_schema: Option<&Schema>, - reader_schemata: Vec<&Schema>, -) -> AvroResult { - let rs = ResolvedSchema::try_from(writer_schemata)?; - let value = decode_internal(writer_schema, rs.get_names(), &None, reader)?; - match reader_schema { - Some(schema) => { - if reader_schemata.is_empty() { - value.resolve(schema) - } else { - value.resolve_schemata(schema, reader_schemata) - } + /// Decode a `Value` encoded in Avro format given its `Schema` and anything implementing `io::Read` + /// to read from. + /// + /// In case a reader `Schema` is provided, schema resolution will also be performed. + /// + /// **NOTE** This function has a quite small niche of usage and does NOT take care of reading the + /// header and consecutive data blocks; use [`Reader`](struct.Reader.html) if you don't know what + /// you are doing, instead. + pub async fn from_avro_datum( + writer_schema: &Schema, + reader: &mut R, + reader_schema: Option<&Schema>, + ) -> AvroResult { + let value = decode(writer_schema, reader).await?; + match reader_schema { + Some(schema) => value.resolve(schema).await, + None => Ok(value), } - None => Ok(value), } -} -pub struct GenericSingleObjectReader { - write_schema: ResolvedOwnedSchema, - expected_header: Vec, -} + /// Decode a `Value` encoded in Avro format given the provided `Schema` and anything implementing `io::Read` + /// to read from. + /// If the writer schema is incomplete, i.e. contains `Schema::Ref`s then it will use the provided + /// schemata to resolve any dependencies. + /// + /// In case a reader `Schema` is provided, schema resolution will also be performed. + pub async fn from_avro_datum_schemata( + writer_schema: &Schema, + writer_schemata: Vec<&Schema>, + reader: &mut R, + reader_schema: Option<&Schema>, + ) -> AvroResult { + from_avro_datum_reader_schemata( + writer_schema, + writer_schemata, + reader, + reader_schema, + Vec::with_capacity(0), + ).await + } -impl GenericSingleObjectReader { - pub fn new(schema: Schema) -> AvroResult { - let header_builder = RabinFingerprintHeader::from_schema(&schema); - Self::new_with_header_builder(schema, header_builder) + /// Decode a `Value` encoded in Avro format given the provided `Schema` and anything implementing `io::Read` + /// to read from. + /// If the writer schema is incomplete, i.e. contains `Schema::Ref`s then it will use the provided + /// schemata to resolve any dependencies. + /// + /// In case a reader `Schema` is provided, schema resolution will also be performed. + pub async fn from_avro_datum_reader_schemata( + writer_schema: &Schema, + writer_schemata: Vec<&Schema>, + reader: &mut R, + reader_schema: Option<&Schema>, + reader_schemata: Vec<&Schema>, + ) -> AvroResult { + let rs = ResolvedSchema::try_from(writer_schemata)?; + let value = decode_internal(writer_schema, rs.get_names(), &None, reader).await?; + match reader_schema { + Some(schema) => { + if reader_schemata.is_empty() { + value.resolve(schema).await + } else { + value.resolve_schemata(schema, reader_schemata).await + } + } + None => Ok(value), + } } - pub fn new_with_header_builder( - schema: Schema, - header_builder: HB, - ) -> AvroResult { - let expected_header = header_builder.build_header(); - Ok(GenericSingleObjectReader { - write_schema: ResolvedOwnedSchema::try_from(schema)?, - expected_header, - }) + pub struct GenericSingleObjectReader { + write_schema: ResolvedOwnedSchema, + expected_header: Vec, } - pub fn read_value(&self, reader: &mut R) -> AvroResult { - let mut header = vec![0; self.expected_header.len()]; - match reader.read_exact(&mut header) { - Ok(_) => { - if self.expected_header == header { - decode_internal( - self.write_schema.get_root_schema(), - self.write_schema.get_names(), - &None, - reader, - ) - } else { - Err( - Details::SingleObjectHeaderMismatch(self.expected_header.clone(), header) - .into(), - ) + impl GenericSingleObjectReader { + pub fn new(schema: Schema) -> AvroResult { + let header_builder = RabinFingerprintHeader::from_schema(&schema); + Self::new_with_header_builder(schema, header_builder) + } + + pub fn new_with_header_builder( + schema: Schema, + header_builder: HB, + ) -> AvroResult { + let expected_header = header_builder.build_header(); + Ok(GenericSingleObjectReader { + write_schema: ResolvedOwnedSchema::try_from(schema)?, + expected_header, + }) + } + + pub async fn read_value(&self, reader: &mut R) -> AvroResult { + let mut header = vec![0; self.expected_header.len()]; + match reader.read_exact(&mut header).await { + Ok(_) => { + if self.expected_header == header { + decode_internal( + self.write_schema.get_root_schema(), + self.write_schema.get_names(), + &None, + reader, + ).await + } else { + Err(Details::SingleObjectHeaderMismatch( + self.expected_header.clone(), + header, + ) + .into()) + } } + Err(io_error) => Err(Details::ReadHeader(io_error).into()), } - Err(io_error) => Err(Details::ReadHeader(io_error).into()), } } -} -pub struct SpecificSingleObjectReader -where - T: AvroSchema, -{ - inner: GenericSingleObjectReader, - _model: PhantomData, -} + pub struct SpecificSingleObjectReader + where + T: AvroSchema, + { + inner: GenericSingleObjectReader, + _model: PhantomData, + } -impl SpecificSingleObjectReader -where - T: AvroSchema, -{ - pub fn new() -> AvroResult> { - Ok(SpecificSingleObjectReader { - inner: GenericSingleObjectReader::new(T::get_schema())?, - _model: PhantomData, - }) + impl SpecificSingleObjectReader + where + T: AvroSchema, + { + pub fn new() -> AvroResult> { + Ok(SpecificSingleObjectReader { + inner: GenericSingleObjectReader::new(T::get_schema())?, + _model: PhantomData, + }) + } } -} -impl SpecificSingleObjectReader -where - T: AvroSchema + From, -{ - pub fn read_from_value(&self, reader: &mut R) -> AvroResult { - self.inner.read_value(reader).map(|v| v.into()) + impl SpecificSingleObjectReader + where + T: AvroSchema + From, + { + pub async fn read_from_value(&self, reader: &mut R) -> AvroResult { + self.inner.read_value(reader).await.map(|v| v.into()) + } } -} -impl SpecificSingleObjectReader -where - T: AvroSchema + DeserializeOwned, -{ - pub fn read(&self, reader: &mut R) -> AvroResult { - from_value::(&self.inner.read_value(reader)?) + impl SpecificSingleObjectReader + where + T: AvroSchema + DeserializeOwned, + { + pub async fn read(&self, reader: &mut R) -> AvroResult { + from_value::(&self.inner.read_value(reader).await?) + } } -} -/// Reads the marker bytes from Avro bytes generated earlier by a `Writer` -pub fn read_marker(bytes: &[u8]) -> [u8; 16] { - assert!( - bytes.len() > 16, - "The bytes are too short to read a marker from them" - ); - let mut marker = [0_u8; 16]; - marker.clone_from_slice(&bytes[(bytes.len() - 16)..]); - marker -} + /// Reads the marker bytes from Avro bytes generated earlier by a `Writer` + pub fn read_marker(bytes: &[u8]) -> [u8; 16] { + assert!( + bytes.len() > 16, + "The bytes are too short to read a marker from them" + ); + let mut marker = [0_u8; 16]; + marker.clone_from_slice(&bytes[(bytes.len() - 16)..]); + marker + } -#[cfg(test)] -mod tests { - use super::*; - use crate::{encode::encode, headers::GlueSchemaUuidHeader, rabin::Rabin, types::Record}; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; - use serde::Deserialize; - use std::io::Cursor; - use uuid::Uuid; - - const SCHEMA: &str = r#" + #[cfg(test)] + mod tests { + use super::*; + use crate::{encode::encode, headers::GlueSchemaUuidHeader, rabin::Rabin, types::Record}; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; + use serde::Deserialize; + use std::io::Cursor; + use uuid::Uuid; + + const SCHEMA: &str = r#" { "type": "record", "name": "test", @@ -624,42 +678,43 @@ mod tests { ] } "#; - const UNION_SCHEMA: &str = r#"["null", "long"]"#; - const ENCODED: &[u8] = &[ - 79u8, 98u8, 106u8, 1u8, 4u8, 22u8, 97u8, 118u8, 114u8, 111u8, 46u8, 115u8, 99u8, 104u8, - 101u8, 109u8, 97u8, 222u8, 1u8, 123u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, - 114u8, 101u8, 99u8, 111u8, 114u8, 100u8, 34u8, 44u8, 34u8, 110u8, 97u8, 109u8, 101u8, 34u8, - 58u8, 34u8, 116u8, 101u8, 115u8, 116u8, 34u8, 44u8, 34u8, 102u8, 105u8, 101u8, 108u8, - 100u8, 115u8, 34u8, 58u8, 91u8, 123u8, 34u8, 110u8, 97u8, 109u8, 101u8, 34u8, 58u8, 34u8, - 97u8, 34u8, 44u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, 108u8, 111u8, 110u8, - 103u8, 34u8, 44u8, 34u8, 100u8, 101u8, 102u8, 97u8, 117u8, 108u8, 116u8, 34u8, 58u8, 52u8, - 50u8, 125u8, 44u8, 123u8, 34u8, 110u8, 97u8, 109u8, 101u8, 34u8, 58u8, 34u8, 98u8, 34u8, - 44u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, 115u8, 116u8, 114u8, 105u8, - 110u8, 103u8, 34u8, 125u8, 93u8, 125u8, 20u8, 97u8, 118u8, 114u8, 111u8, 46u8, 99u8, 111u8, - 100u8, 101u8, 99u8, 8u8, 110u8, 117u8, 108u8, 108u8, 0u8, 94u8, 61u8, 54u8, 221u8, 190u8, - 207u8, 108u8, 180u8, 158u8, 57u8, 114u8, 40u8, 173u8, 199u8, 228u8, 239u8, 4u8, 20u8, 54u8, - 6u8, 102u8, 111u8, 111u8, 84u8, 6u8, 98u8, 97u8, 114u8, 94u8, 61u8, 54u8, 221u8, 190u8, - 207u8, 108u8, 180u8, 158u8, 57u8, 114u8, 40u8, 173u8, 199u8, 228u8, 239u8, - ]; - - #[test] - fn test_from_avro_datum() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; - - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - let expected = record.into(); - - assert_eq!(from_avro_datum(&schema, &mut encoded, None)?, expected); - - Ok(()) - } + const UNION_SCHEMA: &str = r#"["null", "long"]"#; + const ENCODED: &[u8] = &[ + 79u8, 98u8, 106u8, 1u8, 4u8, 22u8, 97u8, 118u8, 114u8, 111u8, 46u8, 115u8, 99u8, 104u8, + 101u8, 109u8, 97u8, 222u8, 1u8, 123u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, + 34u8, 114u8, 101u8, 99u8, 111u8, 114u8, 100u8, 34u8, 44u8, 34u8, 110u8, 97u8, 109u8, + 101u8, 34u8, 58u8, 34u8, 116u8, 101u8, 115u8, 116u8, 34u8, 44u8, 34u8, 102u8, 105u8, + 101u8, 108u8, 100u8, 115u8, 34u8, 58u8, 91u8, 123u8, 34u8, 110u8, 97u8, 109u8, 101u8, + 34u8, 58u8, 34u8, 97u8, 34u8, 44u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, + 108u8, 111u8, 110u8, 103u8, 34u8, 44u8, 34u8, 100u8, 101u8, 102u8, 97u8, 117u8, 108u8, + 116u8, 34u8, 58u8, 52u8, 50u8, 125u8, 44u8, 123u8, 34u8, 110u8, 97u8, 109u8, 101u8, + 34u8, 58u8, 34u8, 98u8, 34u8, 44u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, + 115u8, 116u8, 114u8, 105u8, 110u8, 103u8, 34u8, 125u8, 93u8, 125u8, 20u8, 97u8, 118u8, + 114u8, 111u8, 46u8, 99u8, 111u8, 100u8, 101u8, 99u8, 8u8, 110u8, 117u8, 108u8, 108u8, + 0u8, 94u8, 61u8, 54u8, 221u8, 190u8, 207u8, 108u8, 180u8, 158u8, 57u8, 114u8, 40u8, + 173u8, 199u8, 228u8, 239u8, 4u8, 20u8, 54u8, 6u8, 102u8, 111u8, 111u8, 84u8, 6u8, 98u8, + 97u8, 114u8, 94u8, 61u8, 54u8, 221u8, 190u8, 207u8, 108u8, 180u8, 158u8, 57u8, 114u8, + 40u8, 173u8, 199u8, 228u8, 239u8, + ]; + + #[tokio::test] + async fn test_from_avro_datum() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; - #[test] - fn test_from_avro_datum_with_union_to_struct() -> TestResult { - const TEST_RECORD_SCHEMA_3240: &str = r#" + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + let expected = record.into(); + + assert_eq!(from_avro_datum(&schema, &mut encoded, None)?, expected); + + Ok(()) + } + + #[tokio::test] + async fn test_from_avro_datum_with_union_to_struct() -> TestResult { + const TEST_RECORD_SCHEMA_3240: &str = r#" { "type": "record", "name": "test", @@ -691,166 +746,166 @@ mod tests { ] } "#; - #[derive(Default, Debug, Deserialize, PartialEq, Eq)] - struct TestRecord3240 { - a: i64, - b: String, - a_nullable_array: Option>, - // we are missing the 'a_nullable_boolean' field to simulate missing keys - // a_nullable_boolean: Option, - a_nullable_string: Option, - } - - let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240)?; - let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; - - let expected_record: TestRecord3240 = TestRecord3240 { - a: 27i64, - b: String::from("foo"), - a_nullable_array: None, - a_nullable_string: None, - }; - - let avro_datum = from_avro_datum(&schema, &mut encoded, None)?; - let parsed_record: TestRecord3240 = match &avro_datum { - Value::Record(_) => from_value::(&avro_datum)?, - unexpected => { - panic!("could not map avro data to struct, found unexpected: {unexpected:?}") + #[derive(Default, Debug, Deserialize, PartialEq, Eq)] + struct TestRecord3240 { + a: i64, + b: String, + a_nullable_array: Option>, + // we are missing the 'a_nullable_boolean' field to simulate missing keys + // a_nullable_boolean: Option, + a_nullable_string: Option, } - }; - assert_eq!(parsed_record, expected_record); + let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240)?; + let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; + + let expected_record: TestRecord3240 = TestRecord3240 { + a: 27i64, + b: String::from("foo"), + a_nullable_array: None, + a_nullable_string: None, + }; + + let avro_datum = from_avro_datum(&schema, &mut encoded, None)?; + let parsed_record: TestRecord3240 = match &avro_datum { + Value::Record(_) => from_value::(&avro_datum)?, + unexpected => { + panic!("could not map avro data to struct, found unexpected: {unexpected:?}") + } + }; - Ok(()) - } + assert_eq!(parsed_record, expected_record); - #[test] - fn test_null_union() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA)?; - let mut encoded: &'static [u8] = &[2, 0]; + Ok(()) + } - assert_eq!( - from_avro_datum(&schema, &mut encoded, None)?, - Value::Union(1, Box::new(Value::Long(0))) - ); + #[tokio::test] + async fn test_null_union() -> TestResult { + let schema = Schema::parse_str(UNION_SCHEMA)?; + let mut encoded: &'static [u8] = &[2, 0]; - Ok(()) - } + assert_eq!( + from_avro_datum(&schema, &mut encoded, None)?, + Value::Union(1, Box::new(Value::Long(0))) + ); - #[test] - fn test_reader_iterator() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let reader = Reader::with_schema(&schema, ENCODED)?; + Ok(()) + } - let mut record1 = Record::new(&schema).unwrap(); - record1.put("a", 27i64); - record1.put("b", "foo"); + #[tokio::test] + async fn test_reader_iterator() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let reader = Reader::with_schema(&schema, ENCODED)?; - let mut record2 = Record::new(&schema).unwrap(); - record2.put("a", 42i64); - record2.put("b", "bar"); + let mut record1 = Record::new(&schema).unwrap(); + record1.put("a", 27i64); + record1.put("b", "foo"); - let expected = [record1.into(), record2.into()]; + let mut record2 = Record::new(&schema).unwrap(); + record2.put("a", 42i64); + record2.put("b", "bar"); - for (i, value) in reader.enumerate() { - assert_eq!(value?, expected[i]); - } + let expected = [record1.into(), record2.into()]; - Ok(()) - } + for (i, value) in reader.enumerate() { + assert_eq!(value?, expected[i]); + } - #[test] - fn test_reader_invalid_header() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let invalid = ENCODED.iter().copied().skip(1).collect::>(); - assert!(Reader::with_schema(&schema, &invalid[..]).is_err()); + Ok(()) + } - Ok(()) - } + #[tokio::test] + async fn test_reader_invalid_header() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let invalid = ENCODED.iter().copied().skip(1).collect::>(); + assert!(Reader::with_schema(&schema, &invalid[..]).is_err()); - #[test] - fn test_reader_invalid_block() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let invalid = ENCODED - .iter() - .copied() - .rev() - .skip(19) - .collect::>() - .into_iter() - .rev() - .collect::>(); - let reader = Reader::with_schema(&schema, &invalid[..])?; - for value in reader { - assert!(value.is_err()); + Ok(()) } - Ok(()) - } + #[tokio::test] + async fn test_reader_invalid_block() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let invalid = ENCODED + .iter() + .copied() + .rev() + .skip(19) + .collect::>() + .into_iter() + .rev() + .collect::>(); + let reader = Reader::with_schema(&schema, &invalid[..])?; + for value in reader { + assert!(value.is_err()); + } - #[test] - fn test_reader_empty_buffer() -> TestResult { - let empty = Cursor::new(Vec::new()); - assert!(Reader::new(empty).is_err()); + Ok(()) + } - Ok(()) - } + #[tokio::test] + async fn test_reader_empty_buffer() -> TestResult { + let empty = Cursor::new(Vec::new()); + assert!(Reader::new(empty).is_err()); - #[test] - fn test_reader_only_header() -> TestResult { - let invalid = ENCODED.iter().copied().take(165).collect::>(); - let reader = Reader::new(&invalid[..])?; - for value in reader { - assert!(value.is_err()); + Ok(()) } - Ok(()) - } + #[tokio::test] + async fn test_reader_only_header() -> TestResult { + let invalid = ENCODED.iter().copied().take(165).collect::>(); + let reader = Reader::new(&invalid[..])?; + for value in reader { + assert!(value.is_err()); + } - #[test] - fn test_avro_3405_read_user_metadata_success() -> TestResult { - use crate::writer::Writer; + Ok(()) + } - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + #[tokio::test] + async fn test_avro_3405_read_user_metadata_success() -> TestResult { + use crate::writer::Writer; - let mut user_meta_data: HashMap> = HashMap::new(); - user_meta_data.insert( - "stringKey".to_string(), - "stringValue".to_string().into_bytes(), - ); - user_meta_data.insert("bytesKey".to_string(), b"bytesValue".to_vec()); - user_meta_data.insert("vecKey".to_string(), vec![1, 2, 3]); + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); - for (k, v) in user_meta_data.iter() { - writer.add_user_metadata(k.to_string(), v)?; - } + let mut user_meta_data: HashMap> = HashMap::new(); + user_meta_data.insert( + "stringKey".to_string(), + "stringValue".to_string().into_bytes(), + ); + user_meta_data.insert("bytesKey".to_string(), b"bytesValue".to_vec()); + user_meta_data.insert("vecKey".to_string(), vec![1, 2, 3]); - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); + for (k, v) in user_meta_data.iter() { + writer.add_user_metadata(k.to_string(), v)?; + } - writer.append(record.clone())?; - writer.append(record.clone())?; - writer.flush()?; - let result = writer.into_inner()?; + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); - let reader = Reader::new(&result[..])?; - assert_eq!(reader.user_metadata(), &user_meta_data); + writer.append(record.clone())?; + writer.append(record.clone())?; + writer.flush()?; + let result = writer.into_inner()?; - Ok(()) - } + let reader = Reader::new(&result[..])?; + assert_eq!(reader.user_metadata(), &user_meta_data); - #[derive(Deserialize, Clone, PartialEq, Debug)] - struct TestSingleObjectReader { - a: i64, - b: f64, - c: Vec, - } + Ok(()) + } + + #[derive(Deserialize, Clone, PartialEq, Debug)] + struct TestSingleObjectReader { + a: i64, + b: f64, + c: Vec, + } - impl AvroSchema for TestSingleObjectReader { - fn get_schema() -> Schema { - let schema = r#" + impl AvroSchema for TestSingleObjectReader { + fn get_schema() -> Schema { + let schema = r#" { "type":"record", "name":"TestSingleObjectWrtierSerialize", @@ -873,206 +928,211 @@ mod tests { ] } "#; - Schema::parse_str(schema).unwrap() + Schema::parse_str(schema).unwrap() + } } - } - impl From for TestSingleObjectReader { - fn from(obj: Value) -> TestSingleObjectReader { - if let Value::Record(fields) = obj { - let mut a = None; - let mut b = None; - let mut c = vec![]; - for (field_name, v) in fields { - match (field_name.as_str(), v) { - ("a", Value::Long(i)) => a = Some(i), - ("b", Value::Double(d)) => b = Some(d), - ("c", Value::Array(v)) => { - for inner_val in v { - if let Value::String(s) = inner_val { - c.push(s); + impl From for TestSingleObjectReader { + fn from(obj: Value) -> TestSingleObjectReader { + if let Value::Record(fields) = obj { + let mut a = None; + let mut b = None; + let mut c = vec![]; + for (field_name, v) in fields { + match (field_name.as_str(), v) { + ("a", Value::Long(i)) => a = Some(i), + ("b", Value::Double(d)) => b = Some(d), + ("c", Value::Array(v)) => { + for inner_val in v { + if let Value::String(s) = inner_val { + c.push(s); + } } } + (key, value) => panic!("Unexpected pair: {key:?} -> {value:?}"), } - (key, value) => panic!("Unexpected pair: {key:?} -> {value:?}"), } + TestSingleObjectReader { + a: a.unwrap(), + b: b.unwrap(), + c, + } + } else { + panic!("Expected a Value::Record but was {obj:?}") } - TestSingleObjectReader { - a: a.unwrap(), - b: b.unwrap(), - c, - } - } else { - panic!("Expected a Value::Record but was {obj:?}") } } - } - impl From for Value { - fn from(obj: TestSingleObjectReader) -> Value { - Value::Record(vec![ - ("a".into(), obj.a.into()), - ("b".into(), obj.b.into()), - ( - "c".into(), - Value::Array(obj.c.into_iter().map(|s| s.into()).collect()), - ), - ]) + impl From for Value { + fn from(obj: TestSingleObjectReader) -> Value { + Value::Record(vec![ + ("a".into(), obj.a.into()), + ("b".into(), obj.b.into()), + ( + "c".into(), + Value::Array(obj.c.into_iter().map(|s| s.into()).collect()), + ), + ]) + } } - } - - #[test] - fn test_avro_3507_single_object_reader() -> TestResult { - let obj = TestSingleObjectReader { - a: 42, - b: 3.33, - c: vec!["cat".into(), "dog".into()], - }; - let mut to_read = Vec::::new(); - to_read.extend_from_slice(&[0xC3, 0x01]); - to_read.extend_from_slice( - &TestSingleObjectReader::get_schema() - .fingerprint::() - .bytes[..], - ); - encode( - &obj.clone().into(), - &TestSingleObjectReader::get_schema(), - &mut to_read, - ) - .expect("Encode should succeed"); - let mut to_read = &to_read[..]; - let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) - .expect("Schema should resolve"); - let val = generic_reader - .read_value(&mut to_read) - .expect("Should read"); - let expected_value: Value = obj.into(); - assert_eq!(expected_value, val); - - Ok(()) - } - #[test] - fn avro_3642_test_single_object_reader_incomplete_reads() -> TestResult { - let obj = TestSingleObjectReader { - a: 42, - b: 3.33, - c: vec!["cat".into(), "dog".into()], - }; - // The two-byte marker, to show that the message uses this single-record format - let to_read_1 = [0xC3, 0x01]; - let mut to_read_2 = Vec::::new(); - to_read_2.extend_from_slice( - &TestSingleObjectReader::get_schema() - .fingerprint::() - .bytes[..], - ); - let mut to_read_3 = Vec::::new(); - encode( - &obj.clone().into(), - &TestSingleObjectReader::get_schema(), - &mut to_read_3, - ) - .expect("Encode should succeed"); - let mut to_read = (&to_read_1[..]).chain(&to_read_2[..]).chain(&to_read_3[..]); - let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) - .expect("Schema should resolve"); - let val = generic_reader - .read_value(&mut to_read) - .expect("Should read"); - let expected_value: Value = obj.into(); - assert_eq!(expected_value, val); - - Ok(()) - } + #[tokio::test] + async fn test_avro_3507_single_object_reader() -> TestResult { + let obj = TestSingleObjectReader { + a: 42, + b: 3.33, + c: vec!["cat".into(), "dog".into()], + }; + let mut to_read = Vec::::new(); + to_read.extend_from_slice(&[0xC3, 0x01]); + to_read.extend_from_slice( + &TestSingleObjectReader::get_schema() + .fingerprint::() + .bytes[..], + ); + encode( + &obj.clone().into(), + &TestSingleObjectReader::get_schema(), + &mut to_read, + ) + .expect("Encode should succeed"); + let mut to_read = &to_read[..]; + let generic_reader = + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) + .expect("Schema should resolve"); + let val = generic_reader + .read_value(&mut to_read) + .expect("Should read"); + let expected_value: Value = obj.into(); + assert_eq!(expected_value, val); + + Ok(()) + } - #[test] - fn test_avro_3507_reader_parity() -> TestResult { - let obj = TestSingleObjectReader { - a: 42, - b: 3.33, - c: vec!["cat".into(), "dog".into()], - }; - - let mut to_read = Vec::::new(); - to_read.extend_from_slice(&[0xC3, 0x01]); - to_read.extend_from_slice( - &TestSingleObjectReader::get_schema() - .fingerprint::() - .bytes[..], - ); - encode( - &obj.clone().into(), - &TestSingleObjectReader::get_schema(), - &mut to_read, - ) - .expect("Encode should succeed"); - let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) - .expect("Schema should resolve"); - let specific_reader = SpecificSingleObjectReader::::new() - .expect("schema should resolve"); - let mut to_read1 = &to_read[..]; - let mut to_read2 = &to_read[..]; - let mut to_read3 = &to_read[..]; - - let val = generic_reader - .read_value(&mut to_read1) - .expect("Should read"); - let read_obj1 = specific_reader - .read_from_value(&mut to_read2) - .expect("Should read from value"); - let read_obj2 = specific_reader - .read(&mut to_read3) - .expect("Should read from deserilize"); - let expected_value: Value = obj.clone().into(); - assert_eq!(obj, read_obj1); - assert_eq!(obj, read_obj2); - assert_eq!(val, expected_value); - - Ok(()) - } + #[tokio::test] + async fn avro_3642_test_single_object_reader_incomplete_reads() -> TestResult { + let obj = TestSingleObjectReader { + a: 42, + b: 3.33, + c: vec!["cat".into(), "dog".into()], + }; + // The two-byte marker, to show that the message uses this single-record format + let to_read_1 = [0xC3, 0x01]; + let mut to_read_2 = Vec::::new(); + to_read_2.extend_from_slice( + &TestSingleObjectReader::get_schema() + .fingerprint::() + .bytes[..], + ); + let mut to_read_3 = Vec::::new(); + encode( + &obj.clone().into(), + &TestSingleObjectReader::get_schema(), + &mut to_read_3, + ) + .expect("Encode should succeed"); + let mut to_read = (&to_read_1[..]).chain(&to_read_2[..]).chain(&to_read_3[..]); + let generic_reader = + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) + .expect("Schema should resolve"); + let val = generic_reader + .read_value(&mut to_read) + .expect("Should read"); + let expected_value: Value = obj.into(); + assert_eq!(expected_value, val); + + Ok(()) + } - #[test] - fn avro_rs_164_generic_reader_alternate_header() -> TestResult { - let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; - let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); - let generic_reader = GenericSingleObjectReader::new_with_header_builder( - TestSingleObjectReader::get_schema(), - header_builder, - ) - .expect("failed to build reader"); - let data_to_read: Vec = vec![ - 3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, - ]; - let mut to_read = &data_to_read[..]; - let read_result = generic_reader - .read_value(&mut to_read) - .map_err(Error::into_details); - matches!(read_result, Err(Details::ReadBytes(_))); - Ok(()) - } + #[tokio::test] + async fn test_avro_3507_reader_parity() -> TestResult { + let obj = TestSingleObjectReader { + a: 42, + b: 3.33, + c: vec!["cat".into(), "dog".into()], + }; + + let mut to_read = Vec::::new(); + to_read.extend_from_slice(&[0xC3, 0x01]); + to_read.extend_from_slice( + &TestSingleObjectReader::get_schema() + .fingerprint::() + .bytes[..], + ); + encode( + &obj.clone().into(), + &TestSingleObjectReader::get_schema(), + &mut to_read, + ) + .expect("Encode should succeed"); + let generic_reader = + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) + .expect("Schema should resolve"); + let specific_reader = SpecificSingleObjectReader::::new() + .expect("schema should resolve"); + let mut to_read1 = &to_read[..]; + let mut to_read2 = &to_read[..]; + let mut to_read3 = &to_read[..]; + + let val = generic_reader + .read_value(&mut to_read1) + .expect("Should read"); + let read_obj1 = specific_reader + .read_from_value(&mut to_read2) + .expect("Should read from value"); + let read_obj2 = specific_reader + .read(&mut to_read3) + .expect("Should read from deserilize"); + let expected_value: Value = obj.clone().into(); + assert_eq!(obj, read_obj1); + assert_eq!(obj, read_obj2); + assert_eq!(val, expected_value); + + Ok(()) + } - #[cfg(not(feature = "snappy"))] - #[test] - fn test_avro_3549_read_not_enabled_codec() { - let snappy_compressed_avro = vec![ - 79, 98, 106, 1, 4, 22, 97, 118, 114, 111, 46, 115, 99, 104, 101, 109, 97, 210, 1, 123, - 34, 102, 105, 101, 108, 100, 115, 34, 58, 91, 123, 34, 110, 97, 109, 101, 34, 58, 34, - 110, 117, 109, 34, 44, 34, 116, 121, 112, 101, 34, 58, 34, 115, 116, 114, 105, 110, - 103, 34, 125, 93, 44, 34, 110, 97, 109, 101, 34, 58, 34, 101, 118, 101, 110, 116, 34, - 44, 34, 110, 97, 109, 101, 115, 112, 97, 99, 101, 34, 58, 34, 101, 120, 97, 109, 112, - 108, 101, 110, 97, 109, 101, 115, 112, 97, 99, 101, 34, 44, 34, 116, 121, 112, 101, 34, - 58, 34, 114, 101, 99, 111, 114, 100, 34, 125, 20, 97, 118, 114, 111, 46, 99, 111, 100, - 101, 99, 12, 115, 110, 97, 112, 112, 121, 0, 213, 209, 241, 208, 200, 110, 164, 47, - 203, 25, 90, 235, 161, 167, 195, 177, 2, 20, 4, 12, 6, 49, 50, 51, 115, 38, 58, 0, 213, - 209, 241, 208, 200, 110, 164, 47, 203, 25, 90, 235, 161, 167, 195, 177, - ]; + #[tokio::test] + async fn avro_rs_164_generic_reader_alternate_header() -> TestResult { + let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; + let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); + let generic_reader = GenericSingleObjectReader::new_with_header_builder( + TestSingleObjectReader::get_schema(), + header_builder, + ) + .expect("failed to build reader"); + let data_to_read: Vec = vec![ + 3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, + ]; + let mut to_read = &data_to_read[..]; + let read_result = generic_reader + .read_value(&mut to_read) + .map_err(Error::into_details); + matches!(read_result, Err(Details::ReadBytes(_))); + Ok(()) + } - if let Err(err) = Reader::new(snappy_compressed_avro.as_slice()) { - assert_eq!("Codec 'snappy' is not supported/enabled", err.to_string()); - } else { - panic!("Expected an error in the reading of the codec!"); + #[cfg(not(feature = "snappy"))] + #[tokio::test] + async fn test_avro_3549_read_not_enabled_codec() { + let snappy_compressed_avro = vec![ + 79, 98, 106, 1, 4, 22, 97, 118, 114, 111, 46, 115, 99, 104, 101, 109, 97, 210, 1, + 123, 34, 102, 105, 101, 108, 100, 115, 34, 58, 91, 123, 34, 110, 97, 109, 101, 34, + 58, 34, 110, 117, 109, 34, 44, 34, 116, 121, 112, 101, 34, 58, 34, 115, 116, 114, + 105, 110, 103, 34, 125, 93, 44, 34, 110, 97, 109, 101, 34, 58, 34, 101, 118, 101, + 110, 116, 34, 44, 34, 110, 97, 109, 101, 115, 112, 97, 99, 101, 34, 58, 34, 101, + 120, 97, 109, 112, 108, 101, 110, 97, 109, 101, 115, 112, 97, 99, 101, 34, 44, 34, + 116, 121, 112, 101, 34, 58, 34, 114, 101, 99, 111, 114, 100, 34, 125, 20, 97, 118, + 114, 111, 46, 99, 111, 100, 101, 99, 12, 115, 110, 97, 112, 112, 121, 0, 213, 209, + 241, 208, 200, 110, 164, 47, 203, 25, 90, 235, 161, 167, 195, 177, 2, 20, 4, 12, 6, + 49, 50, 51, 115, 38, 58, 0, 213, 209, 241, 208, 200, 110, 164, 47, 203, 25, 90, + 235, 161, 167, 195, 177, + ]; + + if let Err(err) = Reader::new(snappy_compressed_avro.as_slice()) { + assert_eq!("Codec 'snappy' is not supported/enabled", err.to_string()); + } else { + panic!("Expected an error in the reading of the codec!"); + } } } } diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 833ca30f..33894372 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -20,7 +20,6 @@ use crate::{ AvroResult, error::{Details, Error}, schema_equality, types, - util::MapHelper, validator::{ validate_enum_symbol_name, validate_namespace, validate_record_field_name, validate_schema_name, @@ -43,6 +42,10 @@ use std::{ str::FromStr, }; use strum_macros::{Display, EnumDiscriminants, EnumString}; +#[cfg(feature = "tokio")] +use crate::util::tokio::MapHelper; +#[cfg(feature = "sync")] +use crate::util::sync::MapHelper; /// Represents an Avro schema fingerprint /// More information about Avro schema fingerprints can be found in the @@ -662,7 +665,7 @@ pub enum RecordFieldOrder { impl RecordField { /// Parse a `serde_json::Value` into a `RecordField`. - fn parse( + async fn parse( field: &Map, position: usize, parser: &mut Parser, @@ -686,7 +689,7 @@ impl RecordField { &enclosing_record.fullname(None), &parser.parsed_schemas, &default, - )?; + ).await?; let aliases = field.get("aliases").and_then(|aliases| { aliases.as_array().map(|aliases| { @@ -716,7 +719,7 @@ impl RecordField { }) } - fn resolve_default_value( + async fn resolve_default_value( field_schema: &Schema, field_name: &str, record_name: &str, @@ -728,10 +731,10 @@ impl RecordField { match field_schema { Schema::Union(union_schema) => { let schemas = &union_schema.schemas; - let resolved = schemas.iter().any(|schema| { + let resolved = schemas.iter().any(async |schema| { avro_value .to_owned() - .resolve_internal(schema, names, &schema.namespace(), &None) + .resolve_internal(schema, names, &schema.namespace(), &None).await .is_ok() }); @@ -749,7 +752,7 @@ impl RecordField { } _ => { let resolved = avro_value - .resolve_internal(field_schema, names, &field_schema.namespace(), &None) + .resolve_internal(field_schema, names, &field_schema.namespace(), &None).await .is_ok(); if !resolved { @@ -964,7 +967,7 @@ impl UnionSchema { }) .unwrap_or_default(); - self.schemas.iter().enumerate().find(|(_, schema)| { + self.schemas.iter().enumerate().find(async |(_, schema)| { let resolved_schema = ResolvedSchema::new_with_known_schemata( vec![*schema], enclosing_namespace, @@ -979,7 +982,7 @@ impl UnionSchema { value .clone() - .resolve_internal(schema, &collected_names, namespace, &None) + .resolve_internal(schema, &collected_names, namespace, &None).await .is_ok() }) } diff --git a/avro/src/types.rs b/avro/src/types.rs index 4448eef2..cddacaae 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -615,8 +615,8 @@ impl Value { /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) /// in the Avro specification for the full set of rules of schema /// resolution. - pub fn resolve(self, schema: &Schema) -> AvroResult { - self.resolve_schemata(schema, Vec::with_capacity(0)) + pub async fn resolve(self, schema: &Schema) -> AvroResult { + self.resolve_schemata(schema, Vec::with_capacity(0)).await } /// Attempt to perform schema resolution on the value, with the given @@ -625,17 +625,17 @@ impl Value { /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) /// in the Avro specification for the full set of rules of schema /// resolution. - pub fn resolve_schemata(self, schema: &Schema, schemata: Vec<&Schema>) -> AvroResult { + pub async fn resolve_schemata(self, schema: &Schema, schemata: Vec<&Schema>) -> AvroResult { let enclosing_namespace = schema.namespace(); let rs = if schemata.is_empty() { ResolvedSchema::try_from(schema)? } else { ResolvedSchema::try_from(schemata)? }; - self.resolve_internal(schema, rs.get_names(), &enclosing_namespace, &None) + self.resolve_internal(schema, rs.get_names(), &enclosing_namespace, &None).await } - pub(crate) fn resolve_internal + Debug>( + pub(crate) async fn resolve_internal + Debug>( mut self, schema: &Schema, names: &HashMap, @@ -659,7 +659,7 @@ impl Value { if let Some(resolved) = names.get(&name) { debug!("Resolved {name:?}"); - self.resolve_internal(resolved.borrow(), names, &name.namespace, field_default) + Box::pin(self.resolve_internal(resolved.borrow(), names, &name.namespace, field_default)).await } else { error!("Failed to resolve schema {name:?}"); Err(Details::SchemaResolutionError(name.clone()).into()) @@ -675,7 +675,7 @@ impl Value { Schema::String => self.resolve_string(), Schema::Fixed(FixedSchema { size, .. }) => self.resolve_fixed(size), Schema::Union(ref inner) => { - self.resolve_union(inner, names, enclosing_namespace, field_default) + Box::pin(self.resolve_union(inner, names, enclosing_namespace, field_default)).await } Schema::Enum(EnumSchema { ref symbols, @@ -683,18 +683,18 @@ impl Value { .. }) => self.resolve_enum(symbols, default, field_default), Schema::Array(ref inner) => { - self.resolve_array(&inner.items, names, enclosing_namespace) + self.resolve_array(&inner.items, names, enclosing_namespace).await } Schema::Map(ref inner) => self.resolve_map(&inner.types, names, enclosing_namespace), Schema::Record(RecordSchema { ref fields, .. }) => { - self.resolve_record(fields, names, enclosing_namespace) + self.resolve_record(fields, names, enclosing_namespace).await } Schema::Decimal(DecimalSchema { scale, precision, ref inner, }) => self.resolve_decimal(precision, scale, inner), - Schema::BigDecimal => self.resolve_bigdecimal(), + Schema::BigDecimal => self.resolve_bigdecimal().await, Schema::Date => self.resolve_date(), Schema::TimeMillis => self.resolve_time_millis(), Schema::TimeMicros => self.resolve_time_micros(), @@ -719,10 +719,10 @@ impl Value { }) } - fn resolve_bigdecimal(self) -> Result { + async fn resolve_bigdecimal(self) -> Result { Ok(match self { bg @ Value::BigDecimal(_) => bg, - Value::Bytes(b) => Value::BigDecimal(deserialize_big_decimal(&b).unwrap()), + Value::Bytes(b) => Value::BigDecimal(deserialize_big_decimal(&b).await.unwrap()), other => return Err(Details::GetBigDecimal(other).into()), }) } @@ -1017,7 +1017,7 @@ impl Value { } } - fn resolve_union + Debug>( + async fn resolve_union + Debug>( self, schema: &UnionSchema, names: &HashMap, @@ -1039,11 +1039,11 @@ impl Value { Ok(Value::Union( i as u32, - Box::new(v.resolve_internal(inner, names, enclosing_namespace, field_default)?), + Box::new(v.resolve_internal(inner, names, enclosing_namespace, field_default).await?), )) } - fn resolve_array + Debug>( + async fn resolve_array + Debug>( self, schema: &Schema, names: &HashMap, @@ -1053,8 +1053,8 @@ impl Value { Value::Array(items) => Ok(Value::Array( items .into_iter() - .map(|item| item.resolve_internal(schema, names, enclosing_namespace, &None)) - .collect::>()?, + .map(async |item| item.resolve_internal(schema, names, enclosing_namespace, &None).await) + .collect::>()? )), other => Err(Details::GetArray { expected: schema.into(), @@ -1074,9 +1074,9 @@ impl Value { Value::Map(items) => Ok(Value::Map( items .into_iter() - .map(|(key, value)| { + .map(async |(key, value)| { value - .resolve_internal(schema, names, enclosing_namespace, &None) + .resolve_internal(schema, names, enclosing_namespace, &None).await .map(|value| (key, value)) }) .collect::>()?, @@ -1089,7 +1089,7 @@ impl Value { } } - fn resolve_record + Debug>( + async fn resolve_record + Debug>( self, fields: &[RecordField], names: &HashMap, @@ -1109,7 +1109,7 @@ impl Value { let new_fields = fields .iter() - .map(|field| { + .map(async |field| { let value = match items.remove(&field.name) { Some(value) => value, None => match field.default { @@ -1136,7 +1136,7 @@ impl Value { names, enclosing_namespace, &field.default, - )?), + ).await?), ), } } @@ -1148,7 +1148,7 @@ impl Value { }, }; value - .resolve_internal(&field.schema, names, enclosing_namespace, &field.default) + .resolve_internal(&field.schema, names, enclosing_namespace, &field.default).await .map(|value| (field.name.clone(), value)) }) .collect::, _>>()?; @@ -1156,8 +1156,8 @@ impl Value { Ok(Value::Record(new_fields)) } - fn try_u8(self) -> AvroResult { - let int = self.resolve(&Schema::Int)?; + async fn try_u8(self) -> AvroResult { + let int = self.resolve(&Schema::Int).await?; if let Value::Int(n) = int { if n >= 0 && n <= i32::from(u8::MAX) { return Ok(n as u8); @@ -1184,8 +1184,8 @@ mod tests { use pretty_assertions::assert_eq; use serde_json::json; - #[test] - fn avro_3809_validate_nested_records_with_implicit_namespace() -> TestResult { + #[tokio::test] + async fn avro_3809_validate_nested_records_with_implicit_namespace() -> TestResult { let schema = Schema::parse_str( r#"{ "name": "record_name", @@ -1234,8 +1234,8 @@ mod tests { Ok(()) } - #[test] - fn validate() -> TestResult { + #[tokio::test] + async fn validate() -> TestResult { let value_schema_valid = vec![ (Value::Int(42), Schema::Int, true, ""), (Value::Int(43), Schema::Long, true, ""), @@ -1387,8 +1387,8 @@ mod tests { Ok(()) } - #[test] - fn validate_fixed() -> TestResult { + #[tokio::test] + async fn validate_fixed() -> TestResult { let schema = Schema::Fixed(FixedSchema { size: 4, name: Name::new("some_fixed").unwrap(), @@ -1423,8 +1423,8 @@ mod tests { Ok(()) } - #[test] - fn validate_enum() -> TestResult { + #[tokio::test] + async fn validate_enum() -> TestResult { let schema = Schema::Enum(EnumSchema { name: Name::new("some_enum").unwrap(), aliases: None, @@ -1499,8 +1499,8 @@ mod tests { Ok(()) } - #[test] - fn validate_record() -> TestResult { + #[tokio::test] + async fn validate_record() -> TestResult { // { // "type": "record", // "fields": [ @@ -1671,60 +1671,60 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn resolve_bytes_ok() -> TestResult { + #[tokio::test] + async fn resolve_bytes_ok() -> TestResult { let value = Value::Array(vec![Value::Int(0), Value::Int(42)]); assert_eq!( - value.resolve(&Schema::Bytes)?, + value.resolve(&Schema::Bytes).await?, Value::Bytes(vec![0u8, 42u8]) ); Ok(()) } - #[test] - fn resolve_string_from_bytes() -> TestResult { + #[tokio::test] + async fn resolve_string_from_bytes() -> TestResult { let value = Value::Bytes(vec![97, 98, 99]); assert_eq!( - value.resolve(&Schema::String)?, + value.resolve(&Schema::String).await?, Value::String("abc".to_string()) ); Ok(()) } - #[test] - fn resolve_string_from_fixed() -> TestResult { + #[tokio::test] + async fn resolve_string_from_fixed() -> TestResult { let value = Value::Fixed(3, vec![97, 98, 99]); assert_eq!( - value.resolve(&Schema::String)?, + value.resolve(&Schema::String).await?, Value::String("abc".to_string()) ); Ok(()) } - #[test] - fn resolve_bytes_failure() { + #[tokio::test] + async fn resolve_bytes_failure() { let value = Value::Array(vec![Value::Int(2000), Value::Int(-42)]); - assert!(value.resolve(&Schema::Bytes).is_err()); + assert!(value.resolve(&Schema::Bytes).await.is_err()); } - #[test] - fn resolve_decimal_bytes() -> TestResult { + #[tokio::test] + async fn resolve_decimal_bytes() -> TestResult { let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); value.clone().resolve(&Schema::Decimal(DecimalSchema { precision: 10, scale: 4, inner: Box::new(Schema::Bytes), - }))?; - assert!(value.resolve(&Schema::String).is_err()); + })).await?; + assert!(value.resolve(&Schema::String).await.is_err()); Ok(()) } - #[test] - fn resolve_decimal_invalid_scale() { + #[tokio::test] + async fn resolve_decimal_invalid_scale() { let value = Value::Decimal(Decimal::from(vec![1, 2])); assert!( value @@ -1732,13 +1732,13 @@ Field with name '"b"' is not a member of the map items"#, precision: 2, scale: 3, inner: Box::new(Schema::Bytes), - })) + })).await .is_err() ); } - #[test] - fn resolve_decimal_invalid_precision_for_length() { + #[tokio::test] + async fn resolve_decimal_invalid_precision_for_length() { let value = Value::Decimal(Decimal::from((1u8..=8u8).rev().collect::>())); assert!( value @@ -1746,13 +1746,13 @@ Field with name '"b"' is not a member of the map items"#, precision: 1, scale: 0, inner: Box::new(Schema::Bytes), - })) + })).await .is_ok() ); } - #[test] - fn resolve_decimal_fixed() { + #[tokio::test] + async fn resolve_decimal_fixed() { let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); assert!( value @@ -1768,122 +1768,122 @@ Field with name '"b"' is not a member of the map items"#, default: None, attributes: Default::default(), })) - })) + })).await .is_ok() ); - assert!(value.resolve(&Schema::String).is_err()); + assert!(value.resolve(&Schema::String).await.is_err()); } - #[test] - fn resolve_date() { + #[tokio::test] + async fn resolve_date() { let value = Value::Date(2345); - assert!(value.clone().resolve(&Schema::Date).is_ok()); - assert!(value.resolve(&Schema::String).is_err()); + assert!(value.clone().resolve(&Schema::Date).await.is_ok()); + assert!(value.resolve(&Schema::String).await.is_err()); } - #[test] - fn resolve_time_millis() { + #[tokio::test] + async fn resolve_time_millis() { let value = Value::TimeMillis(10); - assert!(value.clone().resolve(&Schema::TimeMillis).is_ok()); - assert!(value.resolve(&Schema::TimeMicros).is_err()); + assert!(value.clone().resolve(&Schema::TimeMillis).await.is_ok()); + assert!(value.resolve(&Schema::TimeMicros).await.is_err()); } - #[test] - fn resolve_time_micros() { + #[tokio::test] + async fn resolve_time_micros() { let value = Value::TimeMicros(10); - assert!(value.clone().resolve(&Schema::TimeMicros).is_ok()); - assert!(value.resolve(&Schema::TimeMillis).is_err()); + assert!(value.clone().resolve(&Schema::TimeMicros).await.is_ok()); + assert!(value.resolve(&Schema::TimeMillis).await.is_err()); } - #[test] - fn resolve_timestamp_millis() { + #[tokio::test] + async fn resolve_timestamp_millis() { let value = Value::TimestampMillis(10); - assert!(value.clone().resolve(&Schema::TimestampMillis).is_ok()); - assert!(value.resolve(&Schema::Float).is_err()); + assert!(value.clone().resolve(&Schema::TimestampMillis).await.is_ok()); + assert!(value.resolve(&Schema::Float).await.is_err()); let value = Value::Float(10.0f32); - assert!(value.resolve(&Schema::TimestampMillis).is_err()); + assert!(value.resolve(&Schema::TimestampMillis).await.is_err()); } - #[test] - fn resolve_timestamp_micros() { + #[tokio::test] + async fn resolve_timestamp_micros() { let value = Value::TimestampMicros(10); - assert!(value.clone().resolve(&Schema::TimestampMicros).is_ok()); - assert!(value.resolve(&Schema::Int).is_err()); + assert!(value.clone().resolve(&Schema::TimestampMicros).await.is_ok()); + assert!(value.resolve(&Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::TimestampMicros).is_err()); + assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); } - #[test] - fn test_avro_3914_resolve_timestamp_nanos() { + #[tokio::test] + async fn test_avro_3914_resolve_timestamp_nanos() { let value = Value::TimestampNanos(10); - assert!(value.clone().resolve(&Schema::TimestampNanos).is_ok()); - assert!(value.resolve(&Schema::Int).is_err()); + assert!(value.clone().resolve(&Schema::TimestampNanos).await.is_ok()); + assert!(value.resolve(&Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::TimestampNanos).is_err()); + assert!(value.resolve(&Schema::TimestampNanos).await.is_err()); } - #[test] - fn test_avro_3853_resolve_timestamp_millis() { + #[tokio::test] + async fn test_avro_3853_resolve_timestamp_millis() { let value = Value::LocalTimestampMillis(10); - assert!(value.clone().resolve(&Schema::LocalTimestampMillis).is_ok()); - assert!(value.resolve(&Schema::Float).is_err()); + assert!(value.clone().resolve(&Schema::LocalTimestampMillis).await.is_ok()); + assert!(value.resolve(&Schema::Float).await.is_err()); let value = Value::Float(10.0f32); - assert!(value.resolve(&Schema::LocalTimestampMillis).is_err()); + assert!(value.resolve(&Schema::LocalTimestampMillis).await.is_err()); } - #[test] - fn test_avro_3853_resolve_timestamp_micros() { + #[tokio::test] + async fn test_avro_3853_resolve_timestamp_micros() { let value = Value::LocalTimestampMicros(10); - assert!(value.clone().resolve(&Schema::LocalTimestampMicros).is_ok()); - assert!(value.resolve(&Schema::Int).is_err()); + assert!(value.clone().resolve(&Schema::LocalTimestampMicros).await.is_ok()); + assert!(value.resolve(&Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::LocalTimestampMicros).is_err()); + assert!(value.resolve(&Schema::LocalTimestampMicros).await.is_err()); } - #[test] - fn test_avro_3916_resolve_timestamp_nanos() { + #[tokio::test] + async fn test_avro_3916_resolve_timestamp_nanos() { let value = Value::LocalTimestampNanos(10); - assert!(value.clone().resolve(&Schema::LocalTimestampNanos).is_ok()); - assert!(value.resolve(&Schema::Int).is_err()); + assert!(value.clone().resolve(&Schema::LocalTimestampNanos).await.is_ok()); + assert!(value.resolve(&Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::LocalTimestampNanos).is_err()); + assert!(value.resolve(&Schema::LocalTimestampNanos).await.is_err()); } - #[test] - fn resolve_duration() { + #[tokio::test] + async fn resolve_duration() { let value = Value::Duration(Duration::new( Months::new(10), Days::new(5), Millis::new(3000), )); - assert!(value.clone().resolve(&Schema::Duration).is_ok()); - assert!(value.resolve(&Schema::TimestampMicros).is_err()); - assert!(Value::Long(1i64).resolve(&Schema::Duration).is_err()); + assert!(value.clone().resolve(&Schema::Duration).await.is_ok()); + assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); + assert!(Value::Long(1i64).resolve(&Schema::Duration).await.is_err()); } - #[test] - fn resolve_uuid() -> TestResult { + #[tokio::test] + async fn resolve_uuid() -> TestResult { let value = Value::Uuid(Uuid::parse_str("1481531d-ccc9-46d9-a56f-5b67459c0537")?); - assert!(value.clone().resolve(&Schema::Uuid).is_ok()); - assert!(value.resolve(&Schema::TimestampMicros).is_err()); + assert!(value.clone().resolve(&Schema::Uuid).await.is_ok()); + assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); Ok(()) } - #[test] - fn avro_3678_resolve_float_to_double() { + #[tokio::test] + async fn avro_3678_resolve_float_to_double() { let value = Value::Float(2345.1); - assert!(value.resolve(&Schema::Double).is_ok()); + assert!(value.resolve(&Schema::Double).await.is_ok()); } - #[test] - fn test_avro_3621_resolve_to_nullable_union() -> TestResult { + #[tokio::test] + async fn test_avro_3621_resolve_to_nullable_union() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "record", @@ -1922,19 +1922,19 @@ Field with name '"b"' is not a member of the map items"#, "event".to_string(), Value::Record(vec![("amount".to_string(), Value::Int(200))]), )]); - assert!(value.resolve(&schema).is_ok()); + assert!(value.resolve(&schema).await.is_ok()); let value = Value::Record(vec![( "event".to_string(), Value::Record(vec![("size".to_string(), Value::Int(1))]), )]); - assert!(value.resolve(&schema).is_err()); + assert!(value.resolve(&schema).await.is_err()); Ok(()) } - #[test] - fn json_from_avro() -> TestResult { + #[tokio::test] + async fn json_from_avro() -> TestResult { assert_eq!(JsonValue::try_from(Value::Null)?, JsonValue::Null); assert_eq!( JsonValue::try_from(Value::Boolean(true))?, @@ -2108,8 +2108,8 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn test_avro_3433_recursive_resolves_record() -> TestResult { + #[tokio::test] + async fn test_avro_3433_recursive_resolves_record() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2139,14 +2139,14 @@ Field with name '"b"' is not a member of the map items"#, let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); let outer = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); outer - .resolve(&schema) + .resolve(&schema).await .expect("Record definition defined in one field must be available in other field"); Ok(()) } - #[test] - fn test_avro_3433_recursive_resolves_array() -> TestResult { + #[tokio::test] + async fn test_avro_3433_recursive_resolves_array() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2188,14 +2188,14 @@ Field with name '"b"' is not a member of the map items"#, ), ]); outer_value - .resolve(&schema) + .resolve(&schema).await .expect("Record defined in array definition must be resolvable from map"); Ok(()) } - #[test] - fn test_avro_3433_recursive_resolves_map() -> TestResult { + #[tokio::test] + async fn test_avro_3433_recursive_resolves_map() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2234,14 +2234,14 @@ Field with name '"b"' is not a member of the map items"#, ), ]); outer_value - .resolve(&schema) + .resolve(&schema).await .expect("Record defined in record field must be resolvable from map field"); Ok(()) } - #[test] - fn test_avro_3433_recursive_resolves_record_wrapper() -> TestResult { + #[tokio::test] + async fn test_avro_3433_recursive_resolves_record_wrapper() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2281,13 +2281,13 @@ Field with name '"b"' is not a member of the map items"#, )]); let outer_value = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - outer_value.resolve(&schema).expect("Record schema defined in field must be resolvable in Record schema defined in other field"); + outer_value.resolve(&schema).await.expect("Record schema defined in field must be resolvable in Record schema defined in other field"); Ok(()) } - #[test] - fn test_avro_3433_recursive_resolves_map_and_array() -> TestResult { + #[tokio::test] + async fn test_avro_3433_recursive_resolves_map_and_array() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2329,14 +2329,14 @@ Field with name '"b"' is not a member of the map items"#, ("b".into(), Value::Array(vec![inner_value1])), ]); outer_value - .resolve(&schema) + .resolve(&schema).await .expect("Record defined in map definition must be resolvable from array"); Ok(()) } - #[test] - fn test_avro_3433_recursive_resolves_union() -> TestResult { + #[tokio::test] + async fn test_avro_3433_recursive_resolves_union() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2369,18 +2369,18 @@ Field with name '"b"' is not a member of the map items"#, ("b".into(), inner_value2.clone()), ]); outer1 - .resolve(&schema) + .resolve(&schema).await .expect("Record definition defined in union must be resolved in other field"); let outer2 = Value::Record(vec![("a".into(), Value::Null), ("b".into(), inner_value2)]); outer2 - .resolve(&schema) + .resolve(&schema).await .expect("Record definition defined in union must be resolved in other field"); Ok(()) } - #[test] - fn test_avro_3461_test_multi_level_resolve_outer_namespace() -> TestResult { + #[tokio::test] + async fn test_avro_3461_test_multi_level_resolve_outer_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -2455,20 +2455,20 @@ Field with name '"b"' is not a member of the map items"#, ]); outer_record_variation_1 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); outer_record_variation_2 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); outer_record_variation_3 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); Ok(()) } - #[test] - fn test_avro_3461_test_multi_level_resolve_middle_namespace() -> TestResult { + #[tokio::test] + async fn test_avro_3461_test_multi_level_resolve_middle_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -2544,20 +2544,20 @@ Field with name '"b"' is not a member of the map items"#, ]); outer_record_variation_1 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); outer_record_variation_2 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); outer_record_variation_3 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); Ok(()) } - #[test] - fn test_avro_3461_test_multi_level_resolve_inner_namespace() -> TestResult { + #[tokio::test] + async fn test_avro_3461_test_multi_level_resolve_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -2635,20 +2635,20 @@ Field with name '"b"' is not a member of the map items"#, ]); outer_record_variation_1 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); outer_record_variation_2 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); outer_record_variation_3 - .resolve(&schema) + .resolve(&schema).await .expect("Should be able to resolve value to the schema that is it's definition"); Ok(()) } - #[test] - fn test_avro_3460_validation_with_refs() -> TestResult { + #[tokio::test] + async fn test_avro_3460_validation_with_refs() -> TestResult { let schema = Schema::parse_str( r#" { @@ -2699,8 +2699,8 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn test_avro_3460_validation_with_refs_real_struct() -> TestResult { + #[tokio::test] + async fn test_avro_3460_validation_with_refs_real_struct() -> TestResult { use crate::ser::Serializer; use serde::Serialize; @@ -2789,7 +2789,7 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - fn avro_3674_with_or_without_namespace(with_namespace: bool) -> TestResult { + async fn avro_3674_with_or_without_namespace(with_namespace: bool) -> TestResult { use crate::ser::Serializer; use serde::Serialize; @@ -2865,24 +2865,24 @@ Field with name '"b"' is not a member of the map items"#, let test_value: Value = msg.serialize(&mut ser)?; assert!(test_value.validate(&schema), "test_value should validate"); assert!( - test_value.resolve(&schema).is_ok(), + test_value.resolve(&schema).await.is_ok(), "test_value should resolve" ); Ok(()) } - #[test] - fn test_avro_3674_validate_no_namespace_resolution() -> TestResult { - avro_3674_with_or_without_namespace(false) + #[tokio::test] + async fn test_avro_3674_validate_no_namespace_resolution() -> TestResult { + avro_3674_with_or_without_namespace(false).await } - #[test] - fn test_avro_3674_validate_with_namespace_resolution() -> TestResult { - avro_3674_with_or_without_namespace(true) + #[tokio::test] + async fn test_avro_3674_validate_with_namespace_resolution() -> TestResult { + avro_3674_with_or_without_namespace(true).await } - fn avro_3688_schema_resolution_panic(set_field_b: bool) -> TestResult { + async fn avro_3688_schema_resolution_panic(set_field_b: bool) -> TestResult { use crate::ser::Serializer; use serde::{Deserialize, Serialize}; @@ -2948,25 +2948,25 @@ Field with name '"b"' is not a member of the map items"#, let test_value: Value = msg.serialize(&mut ser)?; assert!(test_value.validate(&schema), "test_value should validate"); assert!( - test_value.resolve(&schema).is_ok(), + test_value.resolve(&schema).await.is_ok(), "test_value should resolve" ); Ok(()) } - #[test] - fn test_avro_3688_field_b_not_set() -> TestResult { - avro_3688_schema_resolution_panic(false) + #[tokio::test] + async fn test_avro_3688_field_b_not_set() -> TestResult { + avro_3688_schema_resolution_panic(false).await } - #[test] - fn test_avro_3688_field_b_set() -> TestResult { - avro_3688_schema_resolution_panic(true) + #[tokio::test] + async fn test_avro_3688_field_b_set() -> TestResult { + avro_3688_schema_resolution_panic(true).await } - #[test] - fn test_avro_3764_use_resolve_schemata() -> TestResult { + #[tokio::test] + async fn test_avro_3764_use_resolve_schemata() -> TestResult { let referenced_schema = r#"{"name": "enumForReference", "type": "enum", "symbols": ["A", "B"]}"#; let main_schema = r#"{"name": "recordWithReference", "type": "record", "fields": [{"name": "reference", "type": "enumForReference"}]}"#; @@ -2986,14 +2986,14 @@ Field with name '"b"' is not a member of the map items"#, let main_schema = schemas.first().unwrap(); let schemata: Vec<_> = schemas.iter().skip(1).collect(); - let resolve_result = avro_value.clone().resolve_schemata(main_schema, schemata); + let resolve_result = avro_value.clone().resolve_schemata(main_schema, schemata).await; assert!( resolve_result.is_ok(), "result of resolving with schemata should be ok, got: {resolve_result:?}" ); - let resolve_result = avro_value.resolve(main_schema); + let resolve_result = avro_value.resolve(main_schema).await; assert!( resolve_result.is_err(), "result of resolving without schemata should be err, got: {resolve_result:?}" @@ -3002,8 +3002,8 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn test_avro_3767_union_resolve_complex_refs() -> TestResult { + #[tokio::test] + async fn test_avro_3767_union_resolve_complex_refs() -> TestResult { let referenced_enum = r#"{"name": "enumForReference", "type": "enum", "symbols": ["A", "B"]}"#; let referenced_record = r#"{"name": "recordForReference", "type": "record", "fields": [{"name": "refInRecord", "type": "enumForReference"}]}"#; @@ -3026,7 +3026,7 @@ Field with name '"b"' is not a member of the map items"#, let main_schema = schemata.last().unwrap(); let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); - let resolve_result = avro_value.resolve_schemata(main_schema, other_schemata); + let resolve_result = avro_value.resolve_schemata(main_schema, other_schemata).await; assert!( resolve_result.is_ok(), @@ -3041,15 +3041,15 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn test_avro_3782_incorrect_decimal_resolving() -> TestResult { + #[tokio::test] + async fn test_avro_3782_incorrect_decimal_resolving() -> TestResult { let schema = r#"{"name": "decimalSchema", "logicalType": "decimal", "type": "fixed", "precision": 8, "scale": 0, "size": 8}"#; let avro_value = Value::Decimal(Decimal::from( BigInt::from(12345678u32).to_signed_bytes_be(), )); let schema = Schema::parse_str(schema)?; - let resolve_result = avro_value.resolve(&schema); + let resolve_result = avro_value.resolve(&schema).await; assert!( resolve_result.is_ok(), "resolve result must be ok, got: {resolve_result:?}" @@ -3058,14 +3058,14 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn test_avro_3779_bigdecimal_resolving() -> TestResult { + #[tokio::test] + async fn test_avro_3779_bigdecimal_resolving() -> TestResult { let schema = r#"{"name": "bigDecimalSchema", "logicalType": "big-decimal", "type": "bytes" }"#; let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); let schema = Schema::parse_str(schema)?; - let resolve_result: AvroResult = avro_value.resolve(&schema); + let resolve_result: AvroResult = avro_value.resolve(&schema).await; assert!( resolve_result.is_ok(), "resolve result must be ok, got: {resolve_result:?}" @@ -3074,8 +3074,8 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn test_avro_3892_resolve_fixed_from_bytes() -> TestResult { + #[tokio::test] + async fn test_avro_3892_resolve_fixed_from_bytes() -> TestResult { let value = Value::Bytes(vec![97, 98, 99]); assert_eq!( value.resolve(&Schema::Fixed(FixedSchema { @@ -3085,7 +3085,7 @@ Field with name '"b"' is not a member of the map items"#, size: 3, default: None, attributes: Default::default() - }))?, + })).await?, Value::Fixed(3, vec![97, 98, 99]) ); @@ -3099,7 +3099,7 @@ Field with name '"b"' is not a member of the map items"#, size: 3, default: None, attributes: Default::default() - })) + })).await .is_err(), ); @@ -3113,15 +3113,15 @@ Field with name '"b"' is not a member of the map items"#, size: 3, default: None, attributes: Default::default() - })) + })).await .is_err(), ); Ok(()) } - #[test] - fn avro_3928_from_serde_value_to_types_value() { + #[tokio::test] + async fn avro_3928_from_serde_value_to_types_value() { assert_eq!(Value::from(serde_json::Value::Null), Value::Null); assert_eq!(Value::from(json!(true)), Value::Boolean(true)); assert_eq!(Value::from(json!(false)), Value::Boolean(false)); @@ -3165,11 +3165,11 @@ Field with name '"b"' is not a member of the map items"#, ); } - #[test] - fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { + #[tokio::test] + async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { let schema = Schema::parse_str(r#"{"type": "double"}"#)?; let value = Value::String("unknown".to_owned()); - match value.resolve(&schema).map_err(Error::into_details) { + match value.resolve(&schema).await.map_err(Error::into_details) { Err(err @ Details::GetDouble(_)) => { assert_eq!( format!("{err:?}"), @@ -3183,11 +3183,11 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { + #[tokio::test] + async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { let schema = Schema::parse_str(r#"{"type": "float"}"#)?; let value = Value::String("unknown".to_owned()); - match value.resolve(&schema).map_err(Error::into_details) { + match value.resolve(&schema).await.map_err(Error::into_details) { Err(err @ Details::GetFloat(_)) => { assert_eq!( format!("{err:?}"), @@ -3201,8 +3201,8 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn avro_4029_resolve_from_unsupported_err() -> TestResult { + #[tokio::test] + async fn avro_4029_resolve_from_unsupported_err() -> TestResult { let data: Vec<(&str, Value, &str)> = vec![ ( r#"{ "name": "NAME", "type": "int" }"#, @@ -3338,7 +3338,7 @@ Field with name '"b"' is not a member of the map items"#, for (schema_str, value, expected_error) in data { let schema = Schema::parse_str(schema_str)?; - match value.resolve(&schema) { + match value.resolve(&schema).await { Err(error) => { assert_eq!(format!("{error}"), expected_error); } @@ -3350,8 +3350,8 @@ Field with name '"b"' is not a member of the map items"#, Ok(()) } - #[test] - fn avro_rs_130_get_from_record() -> TestResult { + #[tokio::test] + async fn avro_rs_130_get_from_record() -> TestResult { let schema = r#" { "type": "record", diff --git a/avro/src/util.rs b/avro/src/util.rs index a751fcd5..020adf54 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -15,15 +15,15 @@ // specific language governing permissions and limitations // under the License. -use crate::{AvroResult, error::Details, schema::Documentation}; -use serde_json::{Map, Value}; use std::{ - io::{Read, Write}, sync::{ Once, - atomic::{AtomicBool, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize}, }, }; +use std::sync::atomic::Ordering; +use crate::AvroResult; +use crate::error::Details; /// Maximum number of bytes that can be allocated when decoding /// Avro-encoded values. This is a protection against ill-formed @@ -40,108 +40,8 @@ static MAX_ALLOCATION_BYTES_ONCE: Once = Once::new(); pub(crate) static SERDE_HUMAN_READABLE: AtomicBool = AtomicBool::new(true); static SERDE_HUMAN_READABLE_ONCE: Once = Once::new(); -pub trait MapHelper { - fn string(&self, key: &str) -> Option; - - fn name(&self) -> Option { - self.string("name") - } - - fn doc(&self) -> Documentation { - self.string("doc") - } - - fn aliases(&self) -> Option>; -} - -impl MapHelper for Map { - fn string(&self, key: &str) -> Option { - self.get(key) - .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - } - - fn aliases(&self) -> Option> { - // FIXME no warning when aliases aren't a json array of json strings - self.get("aliases") - .and_then(|aliases| aliases.as_array()) - .and_then(|aliases| { - aliases - .iter() - .map(|alias| alias.as_str()) - .map(|alias| alias.map(|a| a.to_string())) - .collect::>() - }) - } -} - -pub fn read_long(reader: &mut R) -> AvroResult { - zag_i64(reader) -} - -pub fn zig_i32(n: i32, buffer: W) -> AvroResult { - zig_i64(n as i64, buffer) -} - -pub fn zig_i64(n: i64, writer: W) -> AvroResult { - encode_variable(((n << 1) ^ (n >> 63)) as u64, writer) -} - -pub fn zag_i32(reader: &mut R) -> AvroResult { - let i = zag_i64(reader)?; - i32::try_from(i).map_err(|e| Details::ZagI32(e, i).into()) -} - -pub fn zag_i64(reader: &mut R) -> AvroResult { - let z = decode_variable(reader)?; - Ok(if z & 0x1 == 0 { - (z >> 1) as i64 - } else { - !(z >> 1) as i64 - }) -} - -fn encode_variable(mut z: u64, mut writer: W) -> AvroResult { - let mut buffer = [0u8; 10]; - let mut i: usize = 0; - loop { - if z <= 0x7F { - buffer[i] = (z & 0x7F) as u8; - i += 1; - break; - } else { - buffer[i] = (0x80 | (z & 0x7F)) as u8; - i += 1; - z >>= 7; - } - } - writer - .write(&buffer[..i]) - .map_err(|e| Details::WriteBytes(e).into()) -} - -fn decode_variable(reader: &mut R) -> AvroResult { - let mut i = 0u64; - let mut buf = [0u8; 1]; - - let mut j = 0; - loop { - if j > 9 { - // if j * 7 > 64 - return Err(Details::IntegerOverflow.into()); - } - reader - .read_exact(&mut buf[..]) - .map_err(Details::ReadVariableIntegerBytes)?; - i |= (u64::from(buf[0] & 0x7F)) << (j * 7); - if (buf[0] >> 7) == 0 { - break; - } else { - j += 1; - } - } - - Ok(i) +pub(crate) fn is_human_readable() -> bool { + SERDE_HUMAN_READABLE.load(Ordering::Acquire) } /// Set a new maximum number of bytes that can be allocated when decoding data. @@ -157,7 +57,6 @@ pub fn max_allocation_bytes(num_bytes: usize) -> usize { }); MAX_ALLOCATION_BYTES.load(Ordering::Acquire) } - pub fn safe_len(len: usize) -> AvroResult { let max_bytes = max_allocation_bytes(DEFAULT_MAX_ALLOCATION_BYTES); @@ -168,7 +67,7 @@ pub fn safe_len(len: usize) -> AvroResult { desired: len, maximum: max_bytes, } - .into()) + .into()) } } @@ -186,111 +85,238 @@ pub fn set_serde_human_readable(human_readable: bool) { }); } -pub(crate) fn is_human_readable() -> bool { - SERDE_HUMAN_READABLE.load(Ordering::Acquire) -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + #[tokio::test] => #[test] + ); + } +)] +mod util { + #[synca::cfg(tokio)] + use tokio::io::AsyncReadExt; + + use std::{ + io::{Write}, + }; + use crate::schema::Documentation; + use serde_json::{Map, Value}; + use crate::AvroResult; + use crate::error::Details; + + pub trait MapHelper { + fn string(&self, key: &str) -> Option; + + fn name(&self) -> Option { + self.string("name") + } + + fn doc(&self) -> Documentation { + self.string("doc") + } -#[cfg(test)] -mod tests { - use super::*; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; - - #[test] - fn test_zigzag() { - let mut a = Vec::new(); - let mut b = Vec::new(); - zig_i32(42i32, &mut a).unwrap(); - zig_i64(42i64, &mut b).unwrap(); - assert_eq!(a, b); + fn aliases(&self) -> Option>; } - #[test] - fn test_zig_i64() { - let mut s = Vec::new(); + impl MapHelper for Map { + fn string(&self, key: &str) -> Option { + self.get(key) + .and_then(|v| v.as_str()) + .map(|v| v.to_string()) + } - zig_i64(0, &mut s).unwrap(); - assert_eq!(s, [0]); + fn aliases(&self) -> Option> { + // FIXME no warning when aliases aren't a json array of json strings + self.get("aliases") + .and_then(|aliases| aliases.as_array()) + .and_then(|aliases| { + aliases + .iter() + .map(|alias| alias.as_str()) + .map(|alias| alias.map(|a| a.to_string())) + .collect::>() + }) + } + } - s.clear(); - zig_i64(-1, &mut s).unwrap(); - assert_eq!(s, [1]); + pub async fn read_long(reader: &mut R) -> AvroResult { + zag_i64(reader).await + } - s.clear(); - zig_i64(1, &mut s).unwrap(); - assert_eq!(s, [2]); + pub fn zig_i32(n: i32, buffer: W) -> AvroResult { + zig_i64(n as i64, buffer) + } - s.clear(); - zig_i64(-64, &mut s).unwrap(); - assert_eq!(s, [127]); + pub fn zig_i64(n: i64, writer: W) -> AvroResult { + encode_variable(((n << 1) ^ (n >> 63)) as u64, writer) + } - s.clear(); - zig_i64(64, &mut s).unwrap(); - assert_eq!(s, [128, 1]); + pub async fn zag_i32(reader: &mut R) -> AvroResult { + let i = zag_i64(reader).await?; + i32::try_from(i).map_err(|e| Details::ZagI32(e, i).into()) + } - s.clear(); - zig_i64(i32::MAX as i64, &mut s).unwrap(); - assert_eq!(s, [254, 255, 255, 255, 15]); + pub async fn zag_i64(reader: &mut R) -> AvroResult { + let z = decode_variable(reader).await?; + Ok(if z & 0x1 == 0 { + (z >> 1) as i64 + } else { + !(z >> 1) as i64 + }) + } - s.clear(); - zig_i64(i32::MAX as i64 + 1, &mut s).unwrap(); - assert_eq!(s, [128, 128, 128, 128, 16]); + fn encode_variable(mut z: u64, mut writer: W) -> AvroResult { + let mut buffer = [0u8; 10]; + let mut i: usize = 0; + loop { + if z <= 0x7F { + buffer[i] = (z & 0x7F) as u8; + i += 1; + break; + } else { + buffer[i] = (0x80 | (z & 0x7F)) as u8; + i += 1; + z >>= 7; + } + } + writer + .write(&buffer[..i]) + .map_err(|e| Details::WriteBytes(e).into()) + } - s.clear(); - zig_i64(i32::MIN as i64, &mut s).unwrap(); - assert_eq!(s, [255, 255, 255, 255, 15]); + async fn decode_variable(reader: &mut R) -> AvroResult { + let mut i = 0u64; + let mut buf = [0u8; 1]; + + let mut j = 0; + loop { + if j > 9 { + // if j * 7 > 64 + return Err(Details::IntegerOverflow.into()); + } + reader + .read_exact(&mut buf[..]).await + .map_err(Details::ReadVariableIntegerBytes)?; + i |= (u64::from(buf[0] & 0x7F)) << (j * 7); + if (buf[0] >> 7) == 0 { + break; + } else { + j += 1; + } + } - s.clear(); - zig_i64(i32::MIN as i64 - 1, &mut s).unwrap(); - assert_eq!(s, [129, 128, 128, 128, 16]); + Ok(i) + } - s.clear(); - zig_i64(i64::MAX, &mut s).unwrap(); - assert_eq!(s, [254, 255, 255, 255, 255, 255, 255, 255, 255, 1]); - s.clear(); - zig_i64(i64::MIN, &mut s).unwrap(); - assert_eq!(s, [255, 255, 255, 255, 255, 255, 255, 255, 255, 1]); - } + #[cfg(test)] + mod tests { + use super::*; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; + use crate::util::safe_len; + + #[test] + fn test_zigzag() { + let mut a = Vec::new(); + let mut b = Vec::new(); + zig_i32(42i32, &mut a).unwrap(); + zig_i64(42i64, &mut b).unwrap(); + assert_eq!(a, b); + } - #[test] - fn test_zig_i32() { - let mut s = Vec::new(); - zig_i32(i32::MAX / 2, &mut s).unwrap(); - assert_eq!(s, [254, 255, 255, 255, 7]); + #[test] + fn test_zig_i64() { + let mut s = Vec::new(); - s.clear(); - zig_i32(i32::MIN / 2, &mut s).unwrap(); - assert_eq!(s, [255, 255, 255, 255, 7]); + zig_i64(0, &mut s).unwrap(); + assert_eq!(s, [0]); - s.clear(); - zig_i32(-(i32::MIN / 2), &mut s).unwrap(); - assert_eq!(s, [128, 128, 128, 128, 8]); + s.clear(); + zig_i64(-1, &mut s).unwrap(); + assert_eq!(s, [1]); - s.clear(); - zig_i32(i32::MIN / 2 - 1, &mut s).unwrap(); - assert_eq!(s, [129, 128, 128, 128, 8]); + s.clear(); + zig_i64(1, &mut s).unwrap(); + assert_eq!(s, [2]); - s.clear(); - zig_i32(i32::MAX, &mut s).unwrap(); - assert_eq!(s, [254, 255, 255, 255, 15]); + s.clear(); + zig_i64(-64, &mut s).unwrap(); + assert_eq!(s, [127]); - s.clear(); - zig_i32(i32::MIN, &mut s).unwrap(); - assert_eq!(s, [255, 255, 255, 255, 15]); - } + s.clear(); + zig_i64(64, &mut s).unwrap(); + assert_eq!(s, [128, 1]); - #[test] - fn test_overflow() { - let causes_left_shift_overflow: &[u8] = &[0xe1, 0xe1, 0xe1, 0xe1, 0xe1]; - assert!(decode_variable(&mut &*causes_left_shift_overflow).is_err()); - } + s.clear(); + zig_i64(i32::MAX as i64, &mut s).unwrap(); + assert_eq!(s, [254, 255, 255, 255, 15]); + + s.clear(); + zig_i64(i32::MAX as i64 + 1, &mut s).unwrap(); + assert_eq!(s, [128, 128, 128, 128, 16]); + + s.clear(); + zig_i64(i32::MIN as i64, &mut s).unwrap(); + assert_eq!(s, [255, 255, 255, 255, 15]); + + s.clear(); + zig_i64(i32::MIN as i64 - 1, &mut s).unwrap(); + assert_eq!(s, [129, 128, 128, 128, 16]); + + s.clear(); + zig_i64(i64::MAX, &mut s).unwrap(); + assert_eq!(s, [254, 255, 255, 255, 255, 255, 255, 255, 255, 1]); + + s.clear(); + zig_i64(i64::MIN, &mut s).unwrap(); + assert_eq!(s, [255, 255, 255, 255, 255, 255, 255, 255, 255, 1]); + } + + #[test] + fn test_zig_i32() { + let mut s = Vec::new(); + zig_i32(i32::MAX / 2, &mut s).unwrap(); + assert_eq!(s, [254, 255, 255, 255, 7]); - #[test] - fn test_safe_len() -> TestResult { - assert_eq!(42usize, safe_len(42usize)?); - assert!(safe_len(1024 * 1024 * 1024).is_err()); + s.clear(); + zig_i32(i32::MIN / 2, &mut s).unwrap(); + assert_eq!(s, [255, 255, 255, 255, 7]); - Ok(()) + s.clear(); + zig_i32(-(i32::MIN / 2), &mut s).unwrap(); + assert_eq!(s, [128, 128, 128, 128, 8]); + + s.clear(); + zig_i32(i32::MIN / 2 - 1, &mut s).unwrap(); + assert_eq!(s, [129, 128, 128, 128, 8]); + + s.clear(); + zig_i32(i32::MAX, &mut s).unwrap(); + assert_eq!(s, [254, 255, 255, 255, 15]); + + s.clear(); + zig_i32(i32::MIN, &mut s).unwrap(); + assert_eq!(s, [255, 255, 255, 255, 15]); + } + + #[test] + fn test_overflow() { + let causes_left_shift_overflow: &[u8] = &[0xe1, 0xe1, 0xe1, 0xe1, 0xe1]; + assert!(decode_variable(&mut &*causes_left_shift_overflow).is_err()); + } + + #[test] + fn test_safe_len() -> TestResult { + assert_eq!(42usize, safe_len(42usize)?); + assert!(safe_len(1024 * 1024 * 1024).is_err()); + + Ok(()) + } } } From 5355929acef10a12cbc856ee3109d2d51fab07a5 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 29 Jul 2025 16:29:36 +0300 Subject: [PATCH 02/47] More WIP Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 323 +- avro/src/codec.rs | 543 +- avro/src/de.rs | 2795 ++++----- avro/src/decimal.rs | 303 +- avro/src/decode.rs | 121 +- avro/src/encode.rs | 1264 ++-- avro/src/error.rs | 953 +-- avro/src/headers.rs | 262 +- avro/src/lib.rs | 172 +- avro/src/reader.rs | 87 +- avro/src/schema.rs | 9633 +++++++++++++++--------------- avro/src/schema_compatibility.rs | 2478 ++++---- avro/src/schema_equality.rs | 991 +-- avro/src/ser.rs | 1791 +++--- avro/src/ser_schema.rs | 4562 +++++++------- avro/src/types.rs | 5274 ++++++++-------- avro/src/util.rs | 57 +- avro/src/validator.rs | 530 +- avro/src/writer.rs | 2844 ++++----- 19 files changed, 17810 insertions(+), 17173 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 563bbe82..08a96f6d 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -22,89 +22,113 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, #[tokio::test] => #[test] ); } )] -mod bigdecimal {} -use crate::{ - AvroResult, - decode::tokio::{decode_len, decode_long}, - encode::{encode_bytes, encode_long}, - error::Details, - types::Value, -}; -pub use bigdecimal::BigDecimal; -use num_bigint::BigInt; -use std::io::Read; - -pub(crate) fn big_decimal_as_bytes(decimal: &BigDecimal) -> AvroResult> { - let mut buffer: Vec = Vec::new(); - let (big_int, exponent): (BigInt, i64) = decimal.as_bigint_and_exponent(); - let big_endian_value: Vec = big_int.to_signed_bytes_be(); - encode_bytes(&big_endian_value, &mut buffer)?; - encode_long(exponent, &mut buffer)?; - - Ok(buffer) -} - -pub(crate) fn serialize_big_decimal(decimal: &BigDecimal) -> AvroResult> { - // encode big decimal, without global size - let buffer = big_decimal_as_bytes(decimal)?; +mod bigdecimal { + use crate::{ + AvroResult, + decode::tokio::{decode_len, decode_long}, + encode::tokio::{encode_bytes, encode_long}, + error::tokio::Details, + types::tokio::Value, + }; + pub use bigdecimal::BigDecimal; + use num_bigint::BigInt; + use std::io::Read; + + pub(crate) fn big_decimal_as_bytes(decimal: &BigDecimal) -> AvroResult> { + let mut buffer: Vec = Vec::new(); + let (big_int, exponent): (BigInt, i64) = decimal.as_bigint_and_exponent(); + let big_endian_value: Vec = big_int.to_signed_bytes_be(); + encode_bytes(&big_endian_value, &mut buffer)?; + encode_long(exponent, &mut buffer)?; + + Ok(buffer) + } - // encode global size and content - let mut final_buffer: Vec = Vec::new(); - encode_bytes(&buffer, &mut final_buffer)?; + pub(crate) fn serialize_big_decimal(decimal: &BigDecimal) -> AvroResult> { + // encode big decimal, without global size + let buffer = big_decimal_as_bytes(decimal)?; - Ok(final_buffer) -} + // encode global size and content + let mut final_buffer: Vec = Vec::new(); + encode_bytes(&buffer, &mut final_buffer)?; -pub(crate) async fn deserialize_big_decimal(bytes: &Vec) -> AvroResult { - let mut bytes: &[u8] = bytes.as_slice(); - let mut big_decimal_buffer = match decode_len(&mut bytes).await { - Ok(size) => vec![0u8; size], - Err(err) => return Err(Details::BigDecimalLen(Box::new(err)).into()), - }; - - bytes - .read_exact(&mut big_decimal_buffer[..]) - .map_err(Details::ReadDouble)?; + Ok(final_buffer) + } - match decode_long(&mut bytes).await { - Ok(Value::Long(scale_value)) => { - let big_int: BigInt = BigInt::from_signed_bytes_be(&big_decimal_buffer); - let decimal = BigDecimal::new(big_int, scale_value); - Ok(decimal) + pub(crate) async fn deserialize_big_decimal(bytes: &Vec) -> AvroResult { + let mut bytes: &[u8] = bytes.as_slice(); + let mut big_decimal_buffer = match decode_len(&mut bytes).await { + Ok(size) => vec![0u8; size], + Err(err) => return Err(Details::BigDecimalLen(Box::new(err)).into()), + }; + + bytes + .read_exact(&mut big_decimal_buffer[..]) + .map_err(Details::ReadDouble)?; + + match decode_long(&mut bytes).await { + Ok(Value::Long(scale_value)) => { + let big_int: BigInt = BigInt::from_signed_bytes_be(&big_decimal_buffer); + let decimal = BigDecimal::new(big_int, scale_value); + Ok(decimal) + } + _ => Err(Details::BigDecimalScale.into()), } - _ => Err(Details::BigDecimalScale.into()), } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::{Codec, Reader, Schema, Writer, error::Error, types::Record}; - use apache_avro_test_helper::TestResult; - use bigdecimal::{One, Zero}; - use pretty_assertions::assert_eq; - use std::{ - fs::File, - io::BufReader, - ops::{Div, Mul}, - str::FromStr, - }; - - #[tokio::test] - async fn test_avro_3779_bigdecimal_serial() -> TestResult { - let value: BigDecimal = - bigdecimal::BigDecimal::from(-1421).div(bigdecimal::BigDecimal::from(2)); - let mut current: BigDecimal = BigDecimal::one(); - - for iter in 1..180 { - let buffer: Vec = serialize_big_decimal(¤t)?; + #[cfg(test)] + mod tests { + use super::*; + use crate::{ + codec::tokio::Codec, error::tokio::Error, reader::tokio::Reader, schema::tokio::Schema, + types::tokio::Record, writer::tokio::Writer, + }; + use apache_avro_test_helper::TestResult; + use bigdecimal::{One, Zero}; + use pretty_assertions::assert_eq; + use std::{ + fs::File, + io::BufReader, + ops::{Div, Mul}, + str::FromStr, + }; + + #[tokio::test] + async fn test_avro_3779_bigdecimal_serial() -> TestResult { + let value: BigDecimal = + bigdecimal::BigDecimal::from(-1421).div(bigdecimal::BigDecimal::from(2)); + let mut current: BigDecimal = BigDecimal::one(); + + for iter in 1..180 { + let buffer: Vec = serialize_big_decimal(¤t)?; + + let mut as_slice = buffer.as_slice(); + decode_long(&mut as_slice).await?; + + let mut result: Vec = Vec::new(); + result.extend_from_slice(as_slice); + + let deserialize_big_decimal: Result = + deserialize_big_decimal(&result); + assert!( + deserialize_big_decimal.is_ok(), + "can't deserialize for iter {iter}" + ); + assert_eq!(current, deserialize_big_decimal?, "not equals for {iter}"); + current = current.mul(&value); + } + let buffer: Vec = serialize_big_decimal(&BigDecimal::zero())?; let mut as_slice = buffer.as_slice(); decode_long(&mut as_slice).await?; @@ -115,36 +139,20 @@ mod tests { deserialize_big_decimal(&result); assert!( deserialize_big_decimal.is_ok(), - "can't deserialize for iter {iter}" + "can't deserialize for zero" + ); + assert_eq!( + BigDecimal::zero(), + deserialize_big_decimal?, + "not equals for zero" ); - assert_eq!(current, deserialize_big_decimal?, "not equals for {iter}"); - current = current.mul(&value); - } - let buffer: Vec = serialize_big_decimal(&BigDecimal::zero())?; - let mut as_slice = buffer.as_slice(); - decode_long(&mut as_slice).await?; - - let mut result: Vec = Vec::new(); - result.extend_from_slice(as_slice); - - let deserialize_big_decimal: Result = deserialize_big_decimal(&result); - assert!( - deserialize_big_decimal.is_ok(), - "can't deserialize for zero" - ); - assert_eq!( - BigDecimal::zero(), - deserialize_big_decimal?, - "not equals for zero" - ); - - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3779_record_with_bg() -> TestResult { - let schema_str = r#" + #[tokio::test] + async fn test_avro_3779_record_with_bg() -> TestResult { + let schema_str = r#" { "type": "record", "name": "test", @@ -157,66 +165,67 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema_str)?; - - // build record with big decimal value - let mut record = Record::new(&schema).unwrap(); - let val = BigDecimal::new(BigInt::from(12), 2); - record.put("field_name", val.clone()); - - // write a record - let codec = Codec::Null; - let mut writer = Writer::builder() - .schema(&schema) - .codec(codec) - .writer(Vec::new()) - .build(); - - writer.append(record.clone())?; - writer.flush()?; - - // read record - let wrote_data = writer.into_inner()?; - let mut reader = Reader::new(&wrote_data[..])?; - - let value = reader.next().await.unwrap()?; - - // extract field value - let big_decimal_value: &Value = match value { - Value::Record(ref fields) => Ok(&fields[0].1), - other => Err(format!("Expected a Value::Record, got: {other:?}")), - }?; - - let x1res: &BigDecimal = match big_decimal_value { - Value::BigDecimal(s) => Ok(s), - other => Err(format!("Expected Value::BigDecimal, got: {other:?}")), - }?; - assert_eq!(&val, x1res); - - Ok(()) - } + let schema = Schema::parse_str(schema_str)?; + + // build record with big decimal value + let mut record = Record::new(&schema).unwrap(); + let val = BigDecimal::new(BigInt::from(12), 2); + record.put("field_name", val.clone()); + + // write a record + let codec = Codec::Null; + let mut writer = Writer::builder() + .schema(&schema) + .codec(codec) + .writer(Vec::new()) + .build(); + + writer.append(record.clone())?; + writer.flush()?; + + // read record + let wrote_data = writer.into_inner()?; + let mut reader = Reader::new(&wrote_data[..])?; + + let value = reader.next().await.unwrap()?; + + // extract field value + let big_decimal_value: &Value = match value { + Value::Record(ref fields) => Ok(&fields[0].1), + other => Err(format!("Expected a Value::Record, got: {other:?}")), + }?; + + let x1res: &BigDecimal = match big_decimal_value { + Value::BigDecimal(s) => Ok(s), + other => Err(format!("Expected Value::BigDecimal, got: {other:?}")), + }?; + assert_eq!(&val, x1res); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3779_from_java_file() -> TestResult { - // Open file generated with Java code to ensure compatibility - // with Java big decimal logical type. - let file: File = File::open("./tests/bigdec.avro")?; - let mut reader = Reader::new(BufReader::new(&file))?; - let next_element = reader.next().await; - assert!(next_element.is_some()); - let value = next_element.unwrap()?; - let bg = match value { - Value::Record(ref fields) => Ok(&fields[0].1), - other => Err(format!("Expected a Value::Record, got: {other:?}")), - }?; - let value_big_decimal = match bg { - Value::BigDecimal(val) => Ok(val), - other => Err(format!("Expected a Value::BigDecimal, got: {other:?}")), - }?; - - let ref_value = BigDecimal::from_str("2.24")?; - assert_eq!(&ref_value, value_big_decimal); - - Ok(()) + #[tokio::test] + async fn test_avro_3779_from_java_file() -> TestResult { + // Open file generated with Java code to ensure compatibility + // with Java big decimal logical type. + let file: File = File::open("./tests/bigdec.avro")?; + let mut reader = Reader::new(BufReader::new(&file))?; + let next_element = reader.next().await; + assert!(next_element.is_some()); + let value = next_element.unwrap()?; + let bg = match value { + Value::Record(ref fields) => Ok(&fields[0].1), + other => Err(format!("Expected a Value::Record, got: {other:?}")), + }?; + let value_big_decimal = match bg { + Value::BigDecimal(val) => Ok(val), + other => Err(format!("Expected a Value::BigDecimal, got: {other:?}")), + }?; + + let ref_value = BigDecimal::from_str("2.24")?; + assert_eq!(&ref_value, value_big_decimal); + + Ok(()) + } } } diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 48855358..93d661d4 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -16,129 +16,149 @@ // under the License. //! Logic for all supported compression codecs in Avro. -use crate::{AvroResult, Error, error::Details, types::Value}; -use strum_macros::{EnumIter, EnumString, IntoStaticStr}; -/// Settings for the `Deflate` codec. -#[derive(Clone, Copy, Eq, PartialEq, Debug)] -pub struct DeflateSettings { - compression_level: miniz_oxide::deflate::CompressionLevel, -} - -impl DeflateSettings { - pub fn new(compression_level: miniz_oxide::deflate::CompressionLevel) -> Self { - DeflateSettings { compression_level } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod codec { + use crate::{AvroResult, error::tokio::Details, error::tokio::Error, types::tokio::Value}; + use strum_macros::{EnumIter, EnumString, IntoStaticStr}; + + /// Settings for the `Deflate` codec. + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct DeflateSettings { + compression_level: miniz_oxide::deflate::CompressionLevel, } - fn compression_level(&self) -> u8 { - self.compression_level as u8 - } -} + impl DeflateSettings { + pub fn new(compression_level: miniz_oxide::deflate::CompressionLevel) -> Self { + DeflateSettings { compression_level } + } -impl Default for DeflateSettings { - /// Default compression level is `miniz_oxide::deflate::CompressionLevel::DefaultCompression`. - fn default() -> Self { - Self::new(miniz_oxide::deflate::CompressionLevel::DefaultCompression) + fn compression_level(&self) -> u8 { + self.compression_level as u8 + } } -} -/// The compression codec used to compress blocks. -#[derive(Clone, Copy, Debug, Eq, PartialEq, EnumIter, EnumString, IntoStaticStr)] -#[strum(serialize_all = "kebab_case")] -pub enum Codec { - /// The `Null` codec simply passes through data uncompressed. - Null, - /// The `Deflate` codec writes the data block using the deflate algorithm - /// as specified in RFC 1951, and typically implemented using the zlib library. - /// Note that this format (unlike the "zlib format" in RFC 1950) does not have a checksum. - Deflate(DeflateSettings), - #[cfg(feature = "snappy")] - /// The `Snappy` codec uses Google's [Snappy](http://google.github.io/snappy/) - /// compression library. Each compressed block is followed by the 4-byte, big-endian - /// CRC32 checksum of the uncompressed data in the block. - Snappy, - #[cfg(feature = "zstandard")] - /// The `Zstandard` codec uses Facebook's [Zstandard](https://facebook.github.io/zstd/) - Zstandard(zstandard::ZstandardSettings), - #[cfg(feature = "bzip")] - /// The `BZip2` codec uses [BZip2](https://sourceware.org/bzip2/) - /// compression library. - Bzip2(bzip::Bzip2Settings), - #[cfg(feature = "xz")] - /// The `Xz` codec uses [Xz utils](https://tukaani.org/xz/) - /// compression library. - Xz(xz::XzSettings), -} - -impl From for Value { - fn from(value: Codec) -> Self { - Self::Bytes(<&str>::from(value).as_bytes().to_vec()) + impl Default for DeflateSettings { + /// Default compression level is `miniz_oxide::deflate::CompressionLevel::DefaultCompression`. + fn default() -> Self { + Self::new(miniz_oxide::deflate::CompressionLevel::DefaultCompression) + } } -} -impl Codec { - /// Compress a stream of bytes in-place. - pub fn compress(self, stream: &mut Vec) -> AvroResult<()> { - match self { - Codec::Null => (), - Codec::Deflate(settings) => { - let compressed = - miniz_oxide::deflate::compress_to_vec(stream, settings.compression_level()); - *stream = compressed; - } - #[cfg(feature = "snappy")] - Codec::Snappy => { - let mut encoded: Vec = vec![0; snap::raw::max_compress_len(stream.len())]; - let compressed_size = snap::raw::Encoder::new() - .compress(&stream[..], &mut encoded[..]) - .map_err(Details::SnappyCompress)?; - - let mut hasher = crc32fast::Hasher::new(); - hasher.update(&stream[..]); - let checksum = hasher.finalize(); - let checksum_as_bytes = checksum.to_be_bytes(); - let checksum_len = checksum_as_bytes.len(); - encoded.truncate(compressed_size + checksum_len); - encoded[compressed_size..].copy_from_slice(&checksum_as_bytes); - - *stream = encoded; - } - #[cfg(feature = "zstandard")] - Codec::Zstandard(settings) => { - use std::io::Write; - let mut encoder = - zstd::Encoder::new(Vec::new(), settings.compression_level as i32).unwrap(); - encoder.write_all(stream).map_err(Details::ZstdCompress)?; - *stream = encoder.finish().unwrap(); - } - #[cfg(feature = "bzip")] - Codec::Bzip2(settings) => { - use bzip2::read::BzEncoder; - use std::io::Read; + /// The compression codec used to compress blocks. + #[derive(Clone, Copy, Debug, Eq, PartialEq, EnumIter, EnumString, IntoStaticStr)] + #[strum(serialize_all = "kebab_case")] + pub enum Codec { + /// The `Null` codec simply passes through data uncompressed. + Null, + /// The `Deflate` codec writes the data block using the deflate algorithm + /// as specified in RFC 1951, and typically implemented using the zlib library. + /// Note that this format (unlike the "zlib format" in RFC 1950) does not have a checksum. + Deflate(DeflateSettings), + #[cfg(feature = "snappy")] + /// The `Snappy` codec uses Google's [Snappy](http://google.github.io/snappy/) + /// compression library. Each compressed block is followed by the 4-byte, big-endian + /// CRC32 checksum of the uncompressed data in the block. + Snappy, + #[cfg(feature = "zstandard")] + /// The `Zstandard` codec uses Facebook's [Zstandard](https://facebook.github.io/zstd/) + Zstandard(zstandard::ZstandardSettings), + #[cfg(feature = "bzip")] + /// The `BZip2` codec uses [BZip2](https://sourceware.org/bzip2/) + /// compression library. + Bzip2(bzip::Bzip2Settings), + #[cfg(feature = "xz")] + /// The `Xz` codec uses [Xz utils](https://tukaani.org/xz/) + /// compression library. + Xz(xz::XzSettings), + } - let mut encoder = BzEncoder::new(&stream[..], settings.compression()); - let mut buffer = Vec::new(); - encoder.read_to_end(&mut buffer).unwrap(); - *stream = buffer; - } - #[cfg(feature = "xz")] - Codec::Xz(settings) => { - use std::io::Read; - use xz2::read::XzEncoder; + impl From for Value { + fn from(value: Codec) -> Self { + Self::Bytes(<&str>::from(value).as_bytes().to_vec()) + } + } - let mut encoder = XzEncoder::new(&stream[..], settings.compression_level as u32); - let mut buffer = Vec::new(); - encoder.read_to_end(&mut buffer).unwrap(); - *stream = buffer; - } - }; + impl Codec { + /// Compress a stream of bytes in-place. + pub fn compress(self, stream: &mut Vec) -> AvroResult<()> { + match self { + Codec::Null => (), + Codec::Deflate(settings) => { + let compressed = + miniz_oxide::deflate::compress_to_vec(stream, settings.compression_level()); + *stream = compressed; + } + #[cfg(feature = "snappy")] + Codec::Snappy => { + let mut encoded: Vec = vec![0; snap::raw::max_compress_len(stream.len())]; + let compressed_size = snap::raw::Encoder::new() + .compress(&stream[..], &mut encoded[..]) + .map_err(Details::SnappyCompress)?; + + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&stream[..]); + let checksum = hasher.finalize(); + let checksum_as_bytes = checksum.to_be_bytes(); + let checksum_len = checksum_as_bytes.len(); + encoded.truncate(compressed_size + checksum_len); + encoded[compressed_size..].copy_from_slice(&checksum_as_bytes); + + *stream = encoded; + } + #[cfg(feature = "zstandard")] + Codec::Zstandard(settings) => { + use std::io::Write; + let mut encoder = + zstd::Encoder::new(Vec::new(), settings.compression_level as i32).unwrap(); + encoder.write_all(stream).map_err(Details::ZstdCompress)?; + *stream = encoder.finish().unwrap(); + } + #[cfg(feature = "bzip")] + Codec::Bzip2(settings) => { + use bzip2::read::BzEncoder; + use std::io::Read; + + let mut encoder = BzEncoder::new(&stream[..], settings.compression()); + let mut buffer = Vec::new(); + encoder.read_to_end(&mut buffer).unwrap(); + *stream = buffer; + } + #[cfg(feature = "xz")] + Codec::Xz(settings) => { + use std::io::Read; + use xz2::read::XzEncoder; + + let mut encoder = + XzEncoder::new(&stream[..], settings.compression_level as u32); + let mut buffer = Vec::new(); + encoder.read_to_end(&mut buffer).unwrap(); + *stream = buffer; + } + }; - Ok(()) - } + Ok(()) + } - /// Decompress a stream of bytes in-place. - pub fn decompress(self, stream: &mut Vec) -> AvroResult<()> { - *stream = match self { + /// Decompress a stream of bytes in-place. + pub fn decompress(self, stream: &mut Vec) -> AvroResult<()> { + *stream = match self { Codec::Null => return Ok(()), Codec::Deflate(_settings) => miniz_oxide::inflate::decompress_to_vec(stream).map_err(|e| { let err = { @@ -211,195 +231,196 @@ impl Codec { decoded } }; - Ok(()) + Ok(()) + } } -} - -#[cfg(feature = "bzip")] -pub mod bzip { - use bzip2::Compression; - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct Bzip2Settings { - pub compression_level: u8, - } + #[cfg(feature = "bzip")] + pub mod bzip { + use bzip2::Compression; - impl Bzip2Settings { - pub fn new(compression_level: u8) -> Self { - Self { compression_level } + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct Bzip2Settings { + pub compression_level: u8, } - pub(crate) fn compression(&self) -> Compression { - Compression::new(self.compression_level as u32) + impl Bzip2Settings { + pub fn new(compression_level: u8) -> Self { + Self { compression_level } + } + + pub(crate) fn compression(&self) -> Compression { + Compression::new(self.compression_level as u32) + } } - } - impl Default for Bzip2Settings { - fn default() -> Self { - Bzip2Settings::new(Compression::best().level() as u8) + impl Default for Bzip2Settings { + fn default() -> Self { + Bzip2Settings::new(Compression::best().level() as u8) + } } } -} -#[cfg(feature = "zstandard")] -pub mod zstandard { - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct ZstandardSettings { - pub compression_level: u8, - } + #[cfg(feature = "zstandard")] + pub mod zstandard { + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct ZstandardSettings { + pub compression_level: u8, + } - impl ZstandardSettings { - pub fn new(compression_level: u8) -> Self { - Self { compression_level } + impl ZstandardSettings { + pub fn new(compression_level: u8) -> Self { + Self { compression_level } + } } - } - impl Default for ZstandardSettings { - fn default() -> Self { - Self::new(0) + impl Default for ZstandardSettings { + fn default() -> Self { + Self::new(0) + } } } -} -#[cfg(feature = "xz")] -pub mod xz { - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct XzSettings { - pub compression_level: u8, - } + #[cfg(feature = "xz")] + pub mod xz { + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct XzSettings { + pub compression_level: u8, + } - impl XzSettings { - pub fn new(compression_level: u8) -> Self { - Self { compression_level } + impl XzSettings { + pub fn new(compression_level: u8) -> Self { + Self { compression_level } + } } - } - impl Default for XzSettings { - fn default() -> Self { - XzSettings::new(9) + impl Default for XzSettings { + fn default() -> Self { + XzSettings::new(9) + } } } -} -#[cfg(test)] -mod tests { - use super::*; - use apache_avro_test_helper::TestResult; - use miniz_oxide::deflate::CompressionLevel; - use pretty_assertions::{assert_eq, assert_ne}; - - const INPUT: &[u8] = b"theanswertolifetheuniverseandeverythingis42theanswertolifetheuniverseandeverythingis4theanswertolifetheuniverseandeverythingis2"; - - #[test] - fn null_compress_and_decompress() -> TestResult { - let codec = Codec::Null; - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream)?; - assert_eq!(INPUT, stream.as_slice()); - codec.decompress(&mut stream)?; - assert_eq!(INPUT, stream.as_slice()); - Ok(()) - } + #[cfg(test)] + mod tests { + use super::*; + use apache_avro_test_helper::TestResult; + use miniz_oxide::deflate::CompressionLevel; + use pretty_assertions::{assert_eq, assert_ne}; + + const INPUT: &[u8] = b"theanswertolifetheuniverseandeverythingis42theanswertolifetheuniverseandeverythingis4theanswertolifetheuniverseandeverythingis2"; + + #[test] + fn null_compress_and_decompress() -> TestResult { + let codec = Codec::Null; + let mut stream = INPUT.to_vec(); + codec.compress(&mut stream)?; + assert_eq!(INPUT, stream.as_slice()); + codec.decompress(&mut stream)?; + assert_eq!(INPUT, stream.as_slice()); + Ok(()) + } - #[test] - fn deflate_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Deflate(DeflateSettings::new( - CompressionLevel::BestCompression, - ))) - } + #[test] + fn deflate_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Deflate(DeflateSettings::new( + CompressionLevel::BestCompression, + ))) + } - #[cfg(feature = "snappy")] - #[test] - fn snappy_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Snappy) - } + #[cfg(feature = "snappy")] + #[test] + fn snappy_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Snappy) + } - #[cfg(feature = "zstandard")] - #[test] - fn zstd_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Zstandard(zstandard::ZstandardSettings::default())) - } + #[cfg(feature = "zstandard")] + #[test] + fn zstd_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Zstandard(zstandard::ZstandardSettings::default())) + } - #[cfg(feature = "bzip")] - #[test] - fn bzip_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Bzip2(bzip::Bzip2Settings::default())) - } + #[cfg(feature = "bzip")] + #[test] + fn bzip_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Bzip2(bzip::Bzip2Settings::default())) + } - #[cfg(feature = "xz")] - #[test] - fn xz_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Xz(xz::XzSettings::default())) - } + #[cfg(feature = "xz")] + #[test] + fn xz_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Xz(xz::XzSettings::default())) + } - fn compress_and_decompress(codec: Codec) -> TestResult { - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream)?; - assert_ne!(INPUT, stream.as_slice()); - assert!(INPUT.len() > stream.len()); - codec.decompress(&mut stream)?; - assert_eq!(INPUT, stream.as_slice()); - Ok(()) - } + fn compress_and_decompress(codec: Codec) -> TestResult { + let mut stream = INPUT.to_vec(); + codec.compress(&mut stream)?; + assert_ne!(INPUT, stream.as_slice()); + assert!(INPUT.len() > stream.len()); + codec.decompress(&mut stream)?; + assert_eq!(INPUT, stream.as_slice()); + Ok(()) + } - #[test] - fn codec_to_str() { - assert_eq!(<&str>::from(Codec::Null), "null"); - assert_eq!( - <&str>::from(Codec::Deflate(DeflateSettings::default())), - "deflate" - ); + #[test] + fn codec_to_str() { + assert_eq!(<&str>::from(Codec::Null), "null"); + assert_eq!( + <&str>::from(Codec::Deflate(DeflateSettings::default())), + "deflate" + ); - #[cfg(feature = "snappy")] - assert_eq!(<&str>::from(Codec::Snappy), "snappy"); + #[cfg(feature = "snappy")] + assert_eq!(<&str>::from(Codec::Snappy), "snappy"); - #[cfg(feature = "zstandard")] - assert_eq!( - <&str>::from(Codec::Zstandard(zstandard::ZstandardSettings::default())), - "zstandard" - ); + #[cfg(feature = "zstandard")] + assert_eq!( + <&str>::from(Codec::Zstandard(zstandard::ZstandardSettings::default())), + "zstandard" + ); - #[cfg(feature = "bzip")] - assert_eq!( - <&str>::from(Codec::Bzip2(bzip::Bzip2Settings::default())), - "bzip2" - ); + #[cfg(feature = "bzip")] + assert_eq!( + <&str>::from(Codec::Bzip2(bzip::Bzip2Settings::default())), + "bzip2" + ); - #[cfg(feature = "xz")] - assert_eq!(<&str>::from(Codec::Xz(xz::XzSettings::default())), "xz"); - } + #[cfg(feature = "xz")] + assert_eq!(<&str>::from(Codec::Xz(xz::XzSettings::default())), "xz"); + } - #[test] - fn codec_from_str() { - use std::str::FromStr; + #[test] + fn codec_from_str() { + use std::str::FromStr; - assert_eq!(Codec::from_str("null").unwrap(), Codec::Null); - assert_eq!( - Codec::from_str("deflate").unwrap(), - Codec::Deflate(DeflateSettings::default()) - ); + assert_eq!(Codec::from_str("null").unwrap(), Codec::Null); + assert_eq!( + Codec::from_str("deflate").unwrap(), + Codec::Deflate(DeflateSettings::default()) + ); - #[cfg(feature = "snappy")] - assert_eq!(Codec::from_str("snappy").unwrap(), Codec::Snappy); + #[cfg(feature = "snappy")] + assert_eq!(Codec::from_str("snappy").unwrap(), Codec::Snappy); - #[cfg(feature = "zstandard")] - assert_eq!( - Codec::from_str("zstandard").unwrap(), - Codec::Zstandard(zstandard::ZstandardSettings::default()) - ); + #[cfg(feature = "zstandard")] + assert_eq!( + Codec::from_str("zstandard").unwrap(), + Codec::Zstandard(zstandard::ZstandardSettings::default()) + ); - #[cfg(feature = "bzip")] - assert_eq!( - Codec::from_str("bzip2").unwrap(), - Codec::Bzip2(bzip::Bzip2Settings::default()) - ); + #[cfg(feature = "bzip")] + assert_eq!( + Codec::from_str("bzip2").unwrap(), + Codec::Bzip2(bzip::Bzip2Settings::default()) + ); - #[cfg(feature = "xz")] - assert_eq!( - Codec::from_str("xz").unwrap(), - Codec::Xz(xz::XzSettings::default()) - ); + #[cfg(feature = "xz")] + assert_eq!( + Codec::from_str("xz").unwrap(), + Codec::Xz(xz::XzSettings::default()) + ); - assert!(Codec::from_str("not a codec").is_err()); + assert!(Codec::from_str("not a codec").is_err()); + } } } diff --git a/avro/src/de.rs b/avro/src/de.rs index 13021260..55601515 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -16,323 +16,335 @@ // under the License. //! Logic for serde-compatible deserialization. -use crate::{Error, bytes::DE_BYTES_BORROWED, error::Details, types::Value}; -use serde::{ - Deserialize, - de::{self, DeserializeSeed, Deserializer as _, Visitor}, - forward_to_deserialize_any, -}; -use std::{ - collections::{ - HashMap, - hash_map::{Keys, Values}, - }, - slice::Iter, -}; - -pub struct Deserializer<'de> { - input: &'de Value, -} - -struct SeqDeserializer<'de> { - input: Iter<'de, Value>, -} - -struct MapDeserializer<'de> { - input_keys: Keys<'de, String, Value>, - input_values: Values<'de, String, Value>, -} - -struct RecordDeserializer<'de> { - input: Iter<'de, (String, Value)>, - value: Option<&'de Value>, -} - -pub struct EnumUnitDeserializer<'a> { - input: &'a str, -} - -pub struct EnumDeserializer<'de> { - input: &'de [(String, Value)], -} - -/// A `serde::de::EnumAccess` and `serde::de::VariantAccess` implementation for deserializing -/// union types. A `UnionDeserializer` is returned when deserializing into an enum, and the value -/// being deserialized is an Avro union. The enum type being deserialized into should match the -/// structure of the union type of the value being deserialized, (i.e. matching number of variants -/// and schemas of variants.) -/// -/// The `input` field is the name of the variant. Since Avro union variants don't have names, this -/// should come from one of the variant names passed into the `serde::de::Deserializer::deserialize_enum` -/// function call that returned this `UnionDeserializer` -/// -/// `value` the value of the variant, deserialized into a [`crate::types::Value`]. -struct UnionDeserializer<'de> { - input: &'static str, - value: &'de Value, -} -impl<'de> Deserializer<'de> { - pub fn new(input: &'de Value) -> Self { - Deserializer { input } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + headers::tokio => headers::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod de { + + use crate::{ + Error, bytes::DE_BYTES_BORROWED, error::tokio::Details, schema::tokio::SchemaKind, + types::tokio::Value, + }; + #[cfg(feature = "tokio")] + use futures::FutureExt; + use serde::{ + Deserialize, + de::{self, DeserializeSeed, Deserializer as _, Visitor}, + forward_to_deserialize_any, + }; + use std::{ + collections::{ + HashMap, + hash_map::{Keys, Values}, + }, + slice::Iter, + }; + + pub struct Deserializer<'de> { + input: &'de Value, + } + + struct SeqDeserializer<'de> { + input: Iter<'de, Value>, + } + + struct MapDeserializer<'de> { + input_keys: Keys<'de, String, Value>, + input_values: Values<'de, String, Value>, + } + + struct RecordDeserializer<'de> { + input: Iter<'de, (String, Value)>, + value: Option<&'de Value>, + } + + pub struct EnumUnitDeserializer<'a> { + input: &'a str, + } + + pub struct EnumDeserializer<'de> { + input: &'de [(String, Value)], + } + + /// A `serde::de::EnumAccess` and `serde::de::VariantAccess` implementation for deserializing + /// union types. A `UnionDeserializer` is returned when deserializing into an enum, and the value + /// being deserialized is an Avro union. The enum type being deserialized into should match the + /// structure of the union type of the value being deserialized, (i.e. matching number of variants + /// and schemas of variants.) + /// + /// The `input` field is the name of the variant. Since Avro union variants don't have names, this + /// should come from one of the variant names passed into the `serde::de::Deserializer::deserialize_enum` + /// function call that returned this `UnionDeserializer` + /// + /// `value` the value of the variant, deserialized into a [`crate::types::Value`]. + struct UnionDeserializer<'de> { + input: &'static str, + value: &'de Value, + } + + impl<'de> Deserializer<'de> { + pub fn new(input: &'de Value) -> Self { + Deserializer { input } + } } -} -impl<'de> SeqDeserializer<'de> { - pub fn new(input: &'de [Value]) -> Self { - SeqDeserializer { - input: input.iter(), + impl<'de> SeqDeserializer<'de> { + pub fn new(input: &'de [Value]) -> Self { + SeqDeserializer { + input: input.iter(), + } } } -} -impl<'de> MapDeserializer<'de> { - pub fn new(input: &'de HashMap) -> Self { - MapDeserializer { - input_keys: input.keys(), - input_values: input.values(), + impl<'de> MapDeserializer<'de> { + pub fn new(input: &'de HashMap) -> Self { + MapDeserializer { + input_keys: input.keys(), + input_values: input.values(), + } } } -} -impl<'de> RecordDeserializer<'de> { - pub fn new(input: &'de [(String, Value)]) -> Self { - RecordDeserializer { - input: input.iter(), - value: None, + impl<'de> RecordDeserializer<'de> { + pub fn new(input: &'de [(String, Value)]) -> Self { + RecordDeserializer { + input: input.iter(), + value: None, + } } } -} -impl<'a> EnumUnitDeserializer<'a> { - pub fn new(input: &'a str) -> Self { - EnumUnitDeserializer { input } + impl<'a> EnumUnitDeserializer<'a> { + pub fn new(input: &'a str) -> Self { + EnumUnitDeserializer { input } + } } -} -impl<'de> EnumDeserializer<'de> { - pub fn new(input: &'de [(String, Value)]) -> Self { - EnumDeserializer { input } + impl<'de> EnumDeserializer<'de> { + pub fn new(input: &'de [(String, Value)]) -> Self { + EnumDeserializer { input } + } } -} -impl<'de> UnionDeserializer<'de> { - pub fn new(input: &'static str, value: &'de Value) -> Self { - UnionDeserializer { input, value } + impl<'de> UnionDeserializer<'de> { + pub fn new(input: &'static str, value: &'de Value) -> Self { + UnionDeserializer { input, value } + } } -} -impl<'de> de::EnumAccess<'de> for EnumUnitDeserializer<'de> { - type Error = Error; - type Variant = Self; + impl<'de> de::EnumAccess<'de> for EnumUnitDeserializer<'de> { + type Error = Error; + type Variant = Self; - fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> - where - V: DeserializeSeed<'de>, - { - Ok(( - seed.deserialize(StringDeserializer { - input: self.input.to_owned(), - })?, - self, - )) + fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> + where + V: DeserializeSeed<'de>, + { + Ok(( + seed.deserialize(StringDeserializer { + input: self.input.to_owned(), + })?, + self, + )) + } } -} -impl<'de> de::VariantAccess<'de> for EnumUnitDeserializer<'de> { - type Error = Error; + impl<'de> de::VariantAccess<'de> for EnumUnitDeserializer<'de> { + type Error = Error; - fn unit_variant(self) -> Result<(), Error> { - Ok(()) - } + fn unit_variant(self) -> Result<(), Error> { + Ok(()) + } - fn newtype_variant_seed(self, _seed: T) -> Result - where - T: DeserializeSeed<'de>, - { - Err(de::Error::custom("Unexpected Newtype variant")) - } + fn newtype_variant_seed(self, _seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + Err(de::Error::custom("Unexpected Newtype variant")) + } - fn tuple_variant(self, _len: usize, _visitor: V) -> Result - where - V: Visitor<'de>, - { - Err(de::Error::custom("Unexpected tuple variant")) - } + fn tuple_variant(self, _len: usize, _visitor: V) -> Result + where + V: Visitor<'de>, + { + Err(de::Error::custom("Unexpected tuple variant")) + } - fn struct_variant( - self, - _fields: &'static [&'static str], - _visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - Err(de::Error::custom("Unexpected struct variant")) + fn struct_variant( + self, + _fields: &'static [&'static str], + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + Err(de::Error::custom("Unexpected struct variant")) + } } -} -impl<'de> de::EnumAccess<'de> for EnumDeserializer<'de> { - type Error = Error; - type Variant = Self; + impl<'de> de::EnumAccess<'de> for EnumDeserializer<'de> { + type Error = Error; + type Variant = Self; - fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> - where - V: DeserializeSeed<'de>, - { - self.input.first().map_or( - Err(de::Error::custom("A record must have a least one field")), - |item| match (item.0.as_ref(), &item.1) { - ("type", Value::String(x)) | ("type", Value::Enum(_, x)) => Ok(( - seed.deserialize(StringDeserializer { - input: x.to_owned(), - })?, - self, - )), - (field, Value::String(_)) => Err(de::Error::custom(format!( - "Expected first field named 'type': got '{field}' instead" - ))), - (_, _) => Err(de::Error::custom( - "Expected first field of type String or Enum for the type name".to_string(), - )), - }, - ) + fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> + where + V: DeserializeSeed<'de>, + { + self.input.first().map_or( + Err(de::Error::custom("A record must have a least one field")), + |item| match (item.0.as_ref(), &item.1) { + ("type", Value::String(x)) | ("type", Value::Enum(_, x)) => Ok(( + seed.deserialize(StringDeserializer { + input: x.to_owned(), + })?, + self, + )), + (field, Value::String(_)) => Err(de::Error::custom(format!( + "Expected first field named 'type': got '{field}' instead" + ))), + (_, _) => Err(de::Error::custom( + "Expected first field of type String or Enum for the type name".to_string(), + )), + }, + ) + } } -} -impl<'de> de::VariantAccess<'de> for EnumDeserializer<'de> { - type Error = Error; + impl<'de> de::VariantAccess<'de> for EnumDeserializer<'de> { + type Error = Error; - fn unit_variant(self) -> Result<(), Error> { - Ok(()) - } + fn unit_variant(self) -> Result<(), Error> { + Ok(()) + } - fn newtype_variant_seed(self, seed: T) -> Result - where - T: DeserializeSeed<'de>, - { - self.input.get(1).map_or( - Err(de::Error::custom( - "Expected a newtype variant, got nothing instead.", - )), - |item| seed.deserialize(&Deserializer::new(&item.1)), - ) - } + fn newtype_variant_seed(self, seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + self.input.get(1).map_or( + Err(de::Error::custom( + "Expected a newtype variant, got nothing instead.", + )), + |item| seed.deserialize(&Deserializer::new(&item.1)), + ) + } - fn tuple_variant(self, _len: usize, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.input.get(1).map_or( - Err(de::Error::custom( - "Expected a tuple variant, got nothing instead.", - )), - |item| de::Deserializer::deserialize_seq(&Deserializer::new(&item.1), visitor), - ) - } + fn tuple_variant(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.input.get(1).map_or( + Err(de::Error::custom( + "Expected a tuple variant, got nothing instead.", + )), + |item| de::Deserializer::deserialize_seq(&Deserializer::new(&item.1), visitor), + ) + } - fn struct_variant( - self, - fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.input.get(1).map_or( - Err(de::Error::custom("Expected a struct variant, got nothing")), - |item| { - de::Deserializer::deserialize_struct( - &Deserializer::new(&item.1), - "", - fields, - visitor, - ) - }, - ) + fn struct_variant( + self, + fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.input.get(1).map_or( + Err(de::Error::custom("Expected a struct variant, got nothing")), + |item| { + de::Deserializer::deserialize_struct( + &Deserializer::new(&item.1), + "", + fields, + visitor, + ) + }, + ) + } } -} -impl<'de> de::EnumAccess<'de> for UnionDeserializer<'de> { - type Error = Error; - type Variant = Self; + impl<'de> de::EnumAccess<'de> for UnionDeserializer<'de> { + type Error = Error; + type Variant = Self; - /// Deserialize the variant name - fn variant_seed(self, seed: V) -> Result<(V::Value, Self), Self::Error> - where - V: DeserializeSeed<'de>, - { - Ok(( - seed.deserialize(StringDeserializer { - input: String::from(self.input), - })?, - self, - )) + /// Deserialize the variant name + fn variant_seed(self, seed: V) -> Result<(V::Value, Self), Self::Error> + where + V: DeserializeSeed<'de>, + { + Ok(( + seed.deserialize(StringDeserializer { + input: String::from(self.input), + })?, + self, + )) + } } -} -impl<'de> de::VariantAccess<'de> for UnionDeserializer<'de> { - type Error = Error; + impl<'de> de::VariantAccess<'de> for UnionDeserializer<'de> { + type Error = Error; - fn unit_variant(self) -> Result<(), Self::Error> { - match self.value { - Value::Null => Ok(()), - _ => Err(Details::GetNull(self.value.clone()).into()), + fn unit_variant(self) -> Result<(), Self::Error> { + match self.value { + Value::Null => Ok(()), + _ => Err(Details::GetNull(self.value.clone()).into()), + } } - } - fn newtype_variant_seed(self, seed: T) -> Result - where - T: DeserializeSeed<'de>, - { - seed.deserialize(&Deserializer::new(self.value)) - } + fn newtype_variant_seed(self, seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + seed.deserialize(&Deserializer::new(self.value)) + } - fn tuple_variant(self, len: usize, visitor: V) -> Result - where - V: Visitor<'de>, - { - Deserializer::new(self.value).deserialize_tuple(len, visitor) - } + fn tuple_variant(self, len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + Deserializer::new(self.value).deserialize_tuple(len, visitor) + } - fn struct_variant( - self, - fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - let des = Deserializer::new(self.value); - des.deserialize_struct(self.input, fields, visitor) + fn struct_variant( + self, + fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + let des = Deserializer::new(self.value); + des.deserialize_struct(self.input, fields, visitor) + } } -} -impl<'de> de::Deserializer<'de> for &Deserializer<'de> { - type Error = Error; + impl<'de> de::Deserializer<'de> for &Deserializer<'de> { + type Error = Error; - fn deserialize_any(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match self.input { - Value::Null => visitor.visit_unit(), - &Value::Boolean(b) => visitor.visit_bool(b), - Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => visitor.visit_i32(*i), - Value::Long(i) - | Value::TimeMicros(i) - | Value::TimestampMillis(i) - | Value::TimestampMicros(i) - | Value::TimestampNanos(i) - | Value::LocalTimestampMillis(i) - | Value::LocalTimestampMicros(i) - | Value::LocalTimestampNanos(i) => visitor.visit_i64(*i), - &Value::Float(f) => visitor.visit_f32(f), - &Value::Double(d) => visitor.visit_f64(d), - Value::Union(_i, u) => match **u { + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match self.input { Value::Null => visitor.visit_unit(), - Value::Boolean(b) => visitor.visit_bool(b), - Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => visitor.visit_i32(i), + &Value::Boolean(b) => visitor.visit_bool(b), + Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => visitor.visit_i32(*i), Value::Long(i) | Value::TimeMicros(i) | Value::TimestampMillis(i) @@ -340,450 +352,472 @@ impl<'de> de::Deserializer<'de> for &Deserializer<'de> { | Value::TimestampNanos(i) | Value::LocalTimestampMillis(i) | Value::LocalTimestampMicros(i) - | Value::LocalTimestampNanos(i) => visitor.visit_i64(i), - Value::Float(f) => visitor.visit_f32(f), - Value::Double(d) => visitor.visit_f64(d), - Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), - Value::Array(ref fields) => visitor.visit_seq(SeqDeserializer::new(fields)), - Value::String(ref s) => visitor.visit_borrowed_str(s), + | Value::LocalTimestampNanos(i) => visitor.visit_i64(*i), + &Value::Float(f) => visitor.visit_f32(f), + &Value::Double(d) => visitor.visit_f64(d), + Value::Union(_i, u) => match **u { + Value::Null => visitor.visit_unit(), + Value::Boolean(b) => visitor.visit_bool(b), + Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => visitor.visit_i32(i), + Value::Long(i) + | Value::TimeMicros(i) + | Value::TimestampMillis(i) + | Value::TimestampMicros(i) + | Value::TimestampNanos(i) + | Value::LocalTimestampMillis(i) + | Value::LocalTimestampMicros(i) + | Value::LocalTimestampNanos(i) => visitor.visit_i64(i), + Value::Float(f) => visitor.visit_f32(f), + Value::Double(d) => visitor.visit_f64(d), + Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), + Value::Array(ref fields) => visitor.visit_seq(SeqDeserializer::new(fields)), + Value::String(ref s) => visitor.visit_borrowed_str(s), + Value::Uuid(uuid) => visitor.visit_str(&uuid.to_string()), + Value::Map(ref items) => visitor.visit_map(MapDeserializer::new(items)), + Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { + visitor.visit_bytes(bytes) + } + Value::Decimal(ref d) => visitor.visit_bytes(&d.to_vec()?), + Value::Enum(_, ref s) => visitor.visit_borrowed_str(s), + _ => Err(de::Error::custom(format!( + "unsupported union: {:?}", + self.input + ))), + }, + Value::Record(fields) => visitor.visit_map(RecordDeserializer::new(fields)), + Value::Array(fields) => visitor.visit_seq(SeqDeserializer::new(fields)), + Value::String(s) => visitor.visit_borrowed_str(s), Value::Uuid(uuid) => visitor.visit_str(&uuid.to_string()), - Value::Map(ref items) => visitor.visit_map(MapDeserializer::new(items)), - Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => visitor.visit_bytes(bytes), - Value::Decimal(ref d) => visitor.visit_bytes(&d.to_vec()?), - Value::Enum(_, ref s) => visitor.visit_borrowed_str(s), - _ => Err(de::Error::custom(format!( - "unsupported union: {:?}", - self.input + Value::Map(items) => visitor.visit_map(MapDeserializer::new(items)), + Value::Bytes(bytes) | Value::Fixed(_, bytes) => visitor.visit_bytes(bytes), + Value::Decimal(d) => visitor.visit_bytes(&d.to_vec()?), + Value::Enum(_, s) => visitor.visit_borrowed_str(s), + value => Err(de::Error::custom(format!( + "incorrect value of type: {:?}", + SchemaKind::from(value) ))), - }, - Value::Record(fields) => visitor.visit_map(RecordDeserializer::new(fields)), - Value::Array(fields) => visitor.visit_seq(SeqDeserializer::new(fields)), - Value::String(s) => visitor.visit_borrowed_str(s), - Value::Uuid(uuid) => visitor.visit_str(&uuid.to_string()), - Value::Map(items) => visitor.visit_map(MapDeserializer::new(items)), - Value::Bytes(bytes) | Value::Fixed(_, bytes) => visitor.visit_bytes(bytes), - Value::Decimal(d) => visitor.visit_bytes(&d.to_vec()?), - Value::Enum(_, s) => visitor.visit_borrowed_str(s), - value => Err(de::Error::custom(format!( - "incorrect value of type: {:?}", - crate::schema::SchemaKind::from(value) - ))), + } } - } - - forward_to_deserialize_any! { - bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 - } - fn deserialize_char(self, _: V) -> Result - where - V: Visitor<'de>, - { - Err(de::Error::custom("avro does not support char")) - } + forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 + } - fn deserialize_str(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::String(ref s) => visitor.visit_borrowed_str(s), - Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => ::std::str::from_utf8(bytes) - .map_err(|e| de::Error::custom(e.to_string())) - .and_then(|s| visitor.visit_borrowed_str(s)), - Value::Uuid(ref u) => visitor.visit_str(&u.to_string()), - _ => Err(de::Error::custom(format!( - "Expected a String|Bytes|Fixed|Uuid, but got {:?}", - self.input - ))), + fn deserialize_char(self, _: V) -> Result + where + V: Visitor<'de>, + { + Err(de::Error::custom("avro does not support char")) } - } - fn deserialize_string(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::Enum(_, ref s) | Value::String(ref s) => visitor.visit_borrowed_str(s), - Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { - String::from_utf8(bytes.to_owned()) - .map_err(|e| de::Error::custom(e.to_string())) - .and_then(|s| visitor.visit_string(s)) - } - Value::Uuid(ref u) => visitor.visit_str(&u.to_string()), - Value::Union(_i, ref x) => match **x { + fn deserialize_str(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { Value::String(ref s) => visitor.visit_borrowed_str(s), + Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { + ::std::str::from_utf8(bytes) + .map_err(|e| de::Error::custom(e.to_string())) + .and_then(|s| visitor.visit_borrowed_str(s)) + } + Value::Uuid(ref u) => visitor.visit_str(&u.to_string()), + _ => Err(de::Error::custom(format!( + "Expected a String|Bytes|Fixed|Uuid, but got {:?}", + self.input + ))), + } + } + + fn deserialize_string(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { + Value::Enum(_, ref s) | Value::String(ref s) => visitor.visit_borrowed_str(s), Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { String::from_utf8(bytes.to_owned()) .map_err(|e| de::Error::custom(e.to_string())) .and_then(|s| visitor.visit_string(s)) } Value::Uuid(ref u) => visitor.visit_str(&u.to_string()), + Value::Union(_i, ref x) => match **x { + Value::String(ref s) => visitor.visit_borrowed_str(s), + Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { + String::from_utf8(bytes.to_owned()) + .map_err(|e| de::Error::custom(e.to_string())) + .and_then(|s| visitor.visit_string(s)) + } + Value::Uuid(ref u) => visitor.visit_str(&u.to_string()), + _ => Err(de::Error::custom(format!( + "Expected a String|Bytes|Fixed|Uuid, but got {x:?}" + ))), + }, _ => Err(de::Error::custom(format!( - "Expected a String|Bytes|Fixed|Uuid, but got {x:?}" + "Expected a String|Bytes|Fixed|Uuid|Union|Enum, but got {:?}", + self.input ))), - }, - _ => Err(de::Error::custom(format!( - "Expected a String|Bytes|Fixed|Uuid|Union|Enum, but got {:?}", - self.input - ))), + } } - } - fn deserialize_bytes(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::String(ref s) => visitor.visit_bytes(s.as_bytes()), - Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { - if DE_BYTES_BORROWED.get() { - visitor.visit_borrowed_bytes(bytes) - } else { - visitor.visit_bytes(bytes) + fn deserialize_bytes(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { + Value::String(ref s) => visitor.visit_bytes(s.as_bytes()), + Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { + if DE_BYTES_BORROWED.get() { + visitor.visit_borrowed_bytes(bytes) + } else { + visitor.visit_bytes(bytes) + } } + Value::Uuid(ref u) => visitor.visit_bytes(u.as_bytes()), + Value::Decimal(ref d) => visitor.visit_bytes(&d.to_vec()?), + _ => Err(de::Error::custom(format!( + "Expected a String|Bytes|Fixed|Uuid|Decimal, but got {:?}", + self.input + ))), } - Value::Uuid(ref u) => visitor.visit_bytes(u.as_bytes()), - Value::Decimal(ref d) => visitor.visit_bytes(&d.to_vec()?), - _ => Err(de::Error::custom(format!( - "Expected a String|Bytes|Fixed|Uuid|Decimal, but got {:?}", - self.input - ))), } - } - fn deserialize_byte_buf(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::String(ref s) => visitor.visit_byte_buf(s.clone().into_bytes()), - Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { - visitor.visit_byte_buf(bytes.to_owned()) + fn deserialize_byte_buf(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { + Value::String(ref s) => visitor.visit_byte_buf(s.clone().into_bytes()), + Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { + visitor.visit_byte_buf(bytes.to_owned()) + } + _ => Err(de::Error::custom(format!( + "Expected a String|Bytes|Fixed, but got {:?}", + self.input + ))), } - _ => Err(de::Error::custom(format!( - "Expected a String|Bytes|Fixed, but got {:?}", - self.input - ))), } - } - fn deserialize_option(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::Union(_i, ref inner) if inner.as_ref() == &Value::Null => visitor.visit_none(), - Value::Union(_i, ref inner) => visitor.visit_some(&Deserializer::new(inner)), - _ => Err(de::Error::custom(format!( - "Expected a Union, but got {:?}", - self.input - ))), + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { + Value::Union(_i, ref inner) if inner.as_ref() == &Value::Null => { + visitor.visit_none() + } + Value::Union(_i, ref inner) => visitor.visit_some(&Deserializer::new(inner)), + _ => Err(de::Error::custom(format!( + "Expected a Union, but got {:?}", + self.input + ))), + } } - } - fn deserialize_unit(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::Null => visitor.visit_unit(), - Value::Union(_i, ref x) => match **x { + fn deserialize_unit(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { Value::Null => visitor.visit_unit(), + Value::Union(_i, ref x) => match **x { + Value::Null => visitor.visit_unit(), + _ => Err(de::Error::custom(format!( + "Expected a Null, but got {:?}", + self.input + ))), + }, _ => Err(de::Error::custom(format!( - "Expected a Null, but got {:?}", + "Expected a Null|Union, but got {:?}", self.input ))), - }, - _ => Err(de::Error::custom(format!( - "Expected a Null|Union, but got {:?}", - self.input - ))), + } } - } - fn deserialize_unit_struct( - self, - _struct_name: &'static str, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.deserialize_unit(visitor) - } + fn deserialize_unit_struct( + self, + _struct_name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_unit(visitor) + } - fn deserialize_newtype_struct( - self, - _struct_name: &'static str, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - visitor.visit_newtype_struct(self) - } + fn deserialize_newtype_struct( + self, + _struct_name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } - fn deserialize_seq(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::Array(ref items) => visitor.visit_seq(SeqDeserializer::new(items)), - Value::Union(_i, ref inner) => match **inner { + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { Value::Array(ref items) => visitor.visit_seq(SeqDeserializer::new(items)), - Value::Null => visitor.visit_seq(SeqDeserializer::new(&[])), + Value::Union(_i, ref inner) => match **inner { + Value::Array(ref items) => visitor.visit_seq(SeqDeserializer::new(items)), + Value::Null => visitor.visit_seq(SeqDeserializer::new(&[])), + _ => Err(de::Error::custom(format!( + "Expected an Array or Null, but got: {inner:?}" + ))), + }, _ => Err(de::Error::custom(format!( - "Expected an Array or Null, but got: {inner:?}" + "Expected an Array or Union, but got: {:?}", + self.input ))), - }, - _ => Err(de::Error::custom(format!( - "Expected an Array or Union, but got: {:?}", - self.input - ))), + } } - } - fn deserialize_tuple(self, _: usize, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.deserialize_seq(visitor) - } + fn deserialize_tuple(self, _: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } - fn deserialize_tuple_struct( - self, - _struct_name: &'static str, - _len: usize, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.deserialize_seq(visitor) - } + fn deserialize_tuple_struct( + self, + _struct_name: &'static str, + _len: usize, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } - fn deserialize_map(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::Map(ref items) => visitor.visit_map(MapDeserializer::new(items)), - Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), - _ => Err(de::Error::custom(format_args!( - "Expected a record or a map. Got: {:?}", - &self.input - ))), + fn deserialize_map(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match *self.input { + Value::Map(ref items) => visitor.visit_map(MapDeserializer::new(items)), + Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), + _ => Err(de::Error::custom(format_args!( + "Expected a record or a map. Got: {:?}", + &self.input + ))), + } } - } - fn deserialize_struct( - self, - _struct_name: &'static str, - _fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - match *self.input { - Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), - Value::Union(_i, ref inner) => match **inner { + fn deserialize_struct( + self, + _struct_name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + match *self.input { Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), - Value::Null => visitor.visit_map(RecordDeserializer::new(&[])), + Value::Union(_i, ref inner) => match **inner { + Value::Record(ref fields) => visitor.visit_map(RecordDeserializer::new(fields)), + Value::Null => visitor.visit_map(RecordDeserializer::new(&[])), + _ => Err(de::Error::custom(format!( + "Expected a Record or Null, got: {inner:?}" + ))), + }, _ => Err(de::Error::custom(format!( - "Expected a Record or Null, got: {inner:?}" + "Expected a Record or Union, got: {:?}", + self.input ))), - }, - _ => Err(de::Error::custom(format!( - "Expected a Record or Union, got: {:?}", - self.input - ))), + } } - } - fn deserialize_enum( - self, - _enum_name: &'static str, - variants: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - match *self.input { - // This branch can be anything... - Value::Record(ref fields) => visitor.visit_enum(EnumDeserializer::new(fields)), - Value::String(ref field) => visitor.visit_enum(EnumUnitDeserializer::new(field)), - Value::Union(idx, ref inner) => { - if (idx as usize) < variants.len() { - visitor.visit_enum(UnionDeserializer::new( - variants[idx as usize], - inner.as_ref(), - )) - } else { - Err(Details::GetUnionVariant { - index: idx as i64, - num_variants: variants.len(), + fn deserialize_enum( + self, + _enum_name: &'static str, + variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + match *self.input { + // This branch can be anything... + Value::Record(ref fields) => visitor.visit_enum(EnumDeserializer::new(fields)), + Value::String(ref field) => visitor.visit_enum(EnumUnitDeserializer::new(field)), + Value::Union(idx, ref inner) => { + if (idx as usize) < variants.len() { + visitor.visit_enum(UnionDeserializer::new( + variants[idx as usize], + inner.as_ref(), + )) + } else { + Err(Details::GetUnionVariant { + index: idx as i64, + num_variants: variants.len(), + } + .into()) } - .into()) } + // This has to be a unit Enum + Value::Enum(_index, ref field) => { + visitor.visit_enum(EnumUnitDeserializer::new(field)) + } + _ => Err(de::Error::custom(format!( + "Expected a Record|Enum, but got {:?}", + self.input + ))), } - // This has to be a unit Enum - Value::Enum(_index, ref field) => visitor.visit_enum(EnumUnitDeserializer::new(field)), - _ => Err(de::Error::custom(format!( - "Expected a Record|Enum, but got {:?}", - self.input - ))), } - } - fn deserialize_identifier(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.deserialize_str(visitor) - } + fn deserialize_identifier(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_str(visitor) + } - fn deserialize_ignored_any(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.deserialize_any(visitor) - } + fn deserialize_ignored_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } - fn is_human_readable(&self) -> bool { - crate::util::is_human_readable() + fn is_human_readable(&self) -> bool { + crate::util::is_human_readable() + } } -} -impl<'de> de::SeqAccess<'de> for SeqDeserializer<'de> { - type Error = Error; + impl<'de> de::SeqAccess<'de> for SeqDeserializer<'de> { + type Error = Error; - fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> - where - T: DeserializeSeed<'de>, - { - match self.input.next() { - Some(item) => seed.deserialize(&Deserializer::new(item)).map(Some), - None => Ok(None), + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + match self.input.next() { + Some(item) => seed.deserialize(&Deserializer::new(item)).map(Some), + None => Ok(None), + } } } -} -impl<'de> de::MapAccess<'de> for MapDeserializer<'de> { - type Error = Error; + impl<'de> de::MapAccess<'de> for MapDeserializer<'de> { + type Error = Error; - fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> - where - K: DeserializeSeed<'de>, - { - match self.input_keys.next() { - Some(key) => seed - .deserialize(StringDeserializer { - input: (*key).clone(), - }) - .map(Some), - None => Ok(None), + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: DeserializeSeed<'de>, + { + match self.input_keys.next() { + Some(key) => seed + .deserialize(StringDeserializer { + input: (*key).clone(), + }) + .map(Some), + None => Ok(None), + } } - } - fn next_value_seed(&mut self, seed: V) -> Result - where - V: DeserializeSeed<'de>, - { - match self.input_values.next() { - Some(value) => seed.deserialize(&Deserializer::new(value)), - None => Err(de::Error::custom("should not happen - too many values")), + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + match self.input_values.next() { + Some(value) => seed.deserialize(&Deserializer::new(value)), + None => Err(de::Error::custom("should not happen - too many values")), + } } } -} -impl<'de> de::MapAccess<'de> for RecordDeserializer<'de> { - type Error = Error; + impl<'de> de::MapAccess<'de> for RecordDeserializer<'de> { + type Error = Error; - fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> - where - K: DeserializeSeed<'de>, - { - match self.input.next() { - Some(item) => { - let (ref field, ref value) = *item; - self.value = Some(value); - seed.deserialize(StringDeserializer { - input: field.clone(), - }) - .map(Some) + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: DeserializeSeed<'de>, + { + match self.input.next() { + Some(item) => { + let (ref field, ref value) = *item; + self.value = Some(value); + seed.deserialize(StringDeserializer { + input: field.clone(), + }) + .map(Some) + } + None => Ok(None), } - None => Ok(None), } - } - fn next_value_seed(&mut self, seed: V) -> Result - where - V: DeserializeSeed<'de>, - { - match self.value.take() { - Some(value) => seed.deserialize(&Deserializer::new(value)), - None => Err(de::Error::custom("should not happen - too many values")), + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + match self.value.take() { + Some(value) => seed.deserialize(&Deserializer::new(value)), + None => Err(de::Error::custom("should not happen - too many values")), + } } } -} -#[derive(Clone)] -struct StringDeserializer { - input: String, -} + #[derive(Clone)] + struct StringDeserializer { + input: String, + } -impl<'de> de::Deserializer<'de> for StringDeserializer { - type Error = Error; + impl<'de> de::Deserializer<'de> for StringDeserializer { + type Error = Error; - fn deserialize_any(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_string(self.input) - } + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_string(self.input) + } - forward_to_deserialize_any! { - bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string unit option - seq bytes byte_buf map unit_struct newtype_struct - tuple_struct struct tuple enum identifier ignored_any + forward_to_deserialize_any! { + bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string unit option + seq bytes byte_buf map unit_struct newtype_struct + tuple_struct struct tuple enum identifier ignored_any + } } -} -/// Interpret a `Value` as an instance of type `D`. -/// -/// This conversion can fail if the structure of the `Value` does not match the -/// structure expected by `D`. -pub fn from_value<'de, D: Deserialize<'de>>(value: &'de Value) -> Result { - let de = Deserializer::new(value); - D::deserialize(&de) -} + /// Interpret a `Value` as an instance of type `D`. + /// + /// This conversion can fail if the structure of the `Value` does not match the + /// structure expected by `D`. + pub fn from_value<'de, D: Deserialize<'de>>(value: &'de Value) -> Result { + let de = Deserializer::new(value); + D::deserialize(&de) + } -#[cfg(test)] -mod tests { - use num_bigint::BigInt; - use pretty_assertions::assert_eq; - use serde::{Deserialize, Serialize}; - use serial_test::serial; - use std::sync::atomic::Ordering; - use uuid::Uuid; + #[cfg(test)] + mod tests { + use num_bigint::BigInt; + use pretty_assertions::assert_eq; + use serde::{Deserialize, Serialize}; + use serial_test::serial; + use std::sync::atomic::Ordering; + use uuid::Uuid; - use apache_avro_test_helper::TestResult; + use apache_avro_test_helper::TestResult; - use crate::Decimal; + use crate::Decimal; - use super::*; + use super::*; - #[derive(PartialEq, Eq, Serialize, Deserialize, Debug)] - pub struct StringEnum { - pub source: String, - } + #[derive(PartialEq, Eq, Serialize, Deserialize, Debug)] + pub struct StringEnum { + pub source: String, + } - #[test] - fn avro_3955_decode_enum() -> TestResult { - let schema_content = r#" + #[test] + fn avro_3955_decode_enum() -> TestResult { + let schema_content = r#" { "name": "AccessLog", "namespace": "com.clevercloud.accesslogs.common.avro", @@ -802,29 +836,29 @@ mod tests { } "#; - let schema = crate::Schema::parse_str(schema_content)?; - let data = StringEnum { - source: "SOZU".to_string(), - }; + let schema = crate::Schema::parse_str(schema_content)?; + let data = StringEnum { + source: "SOZU".to_string(), + }; - // encode into avro - let value = crate::to_value(&data)?; + // encode into avro + let value = crate::to_value(&data)?; - let mut buf = std::io::Cursor::new(crate::to_avro_datum(&schema, value)?); + let mut buf = std::io::Cursor::new(crate::to_avro_datum(&schema, value)?); - // decode from avro - let value = crate::from_avro_datum(&schema, &mut buf, None)?; + // decode from avro + let value = crate::from_avro_datum(&schema, &mut buf, None)?; - let decoded_data: StringEnum = crate::from_value(&value)?; + let decoded_data: StringEnum = crate::from_value(&value)?; - assert_eq!(decoded_data, data); + assert_eq!(decoded_data, data); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3955_encode_enum_data_with_wrong_content() -> TestResult { - let schema_content = r#" + #[test] + fn avro_3955_encode_enum_data_with_wrong_content() -> TestResult { + let schema_content = r#" { "name": "AccessLog", "namespace": "com.clevercloud.accesslogs.common.avro", @@ -843,827 +877,828 @@ mod tests { } "#; - let schema = crate::Schema::parse_str(schema_content)?; - let data = StringEnum { - source: "WRONG_ITEM".to_string(), - }; - - // encode into avro - let value = crate::to_value(data)?; - - // The following sentence have to fail has the data is wrong. - let encoded_data = crate::to_avro_datum(&schema, value); - - assert!(encoded_data.is_err()); + let schema = crate::Schema::parse_str(schema_content)?; + let data = StringEnum { + source: "WRONG_ITEM".to_string(), + }; - Ok(()) - } + // encode into avro + let value = crate::to_value(data)?; - #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] - struct Test { - a: i64, - b: String, - c: Decimal, - } + // The following sentence have to fail has the data is wrong. + let encoded_data = crate::to_avro_datum(&schema, value); - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - struct TestInner { - a: Test, - b: i32, - } - - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - struct TestUnitExternalEnum { - a: UnitExternalEnum, - } + assert!(encoded_data.is_err()); - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - enum UnitExternalEnum { - Val1, - Val2, - } - - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - struct TestUnitInternalEnum { - a: UnitInternalEnum, - } - - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - #[serde(tag = "t")] - enum UnitInternalEnum { - Val1, - Val2, - } + Ok(()) + } - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - struct TestUnitAdjacentEnum { - a: UnitAdjacentEnum, - } + #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] + struct Test { + a: i64, + b: String, + c: Decimal, + } - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - #[serde(tag = "t", content = "v")] - enum UnitAdjacentEnum { - Val1, - Val2, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + struct TestInner { + a: Test, + b: i32, + } - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - struct TestUnitUntaggedEnum { - a: UnitUntaggedEnum, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + struct TestUnitExternalEnum { + a: UnitExternalEnum, + } - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - #[serde(untagged)] - enum UnitUntaggedEnum { - Val1, - Val2, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + enum UnitExternalEnum { + Val1, + Val2, + } - #[derive(Debug, Serialize, Deserialize, PartialEq)] - struct TestSingleValueExternalEnum { - a: SingleValueExternalEnum, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + struct TestUnitInternalEnum { + a: UnitInternalEnum, + } - #[derive(Debug, Serialize, Deserialize, PartialEq)] - enum SingleValueExternalEnum { - Double(f64), - String(String), - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + #[serde(tag = "t")] + enum UnitInternalEnum { + Val1, + Val2, + } - #[derive(Debug, Serialize, Deserialize, PartialEq)] - struct TestStructExternalEnum { - a: StructExternalEnum, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + struct TestUnitAdjacentEnum { + a: UnitAdjacentEnum, + } - #[derive(Debug, Serialize, Deserialize, PartialEq)] - enum StructExternalEnum { - Val1 { x: f32, y: f32 }, - Val2 { x: f32, y: f32 }, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + #[serde(tag = "t", content = "v")] + enum UnitAdjacentEnum { + Val1, + Val2, + } - #[derive(Debug, Serialize, Deserialize, PartialEq)] - struct TestTupleExternalEnum { - a: TupleExternalEnum, - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + struct TestUnitUntaggedEnum { + a: UnitUntaggedEnum, + } - #[derive(Debug, Serialize, Deserialize, PartialEq)] - enum TupleExternalEnum { - Val1(f32, f32), - Val2(f32, f32, f32), - } + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + #[serde(untagged)] + enum UnitUntaggedEnum { + Val1, + Val2, + } - #[test] - fn test_from_value() -> TestResult { - let test = Value::Record(vec![ - ("a".to_owned(), Value::Long(27)), - ("b".to_owned(), Value::String("foo".to_owned())), - ("c".to_owned(), Value::Decimal(Decimal::from(vec![1, 24]))), - ]); - let expected = Test { - a: 27, - b: "foo".to_owned(), - c: Decimal::from(vec![1, 24]), - }; - let final_value: Test = from_value(&test)?; - assert_eq!(final_value, expected); - - let test_inner = Value::Record(vec![ - ( - "a".to_owned(), - Value::Record(vec![ - ("a".to_owned(), Value::Long(27)), - ("b".to_owned(), Value::String("foo".to_owned())), - ("c".to_owned(), Value::Decimal(Decimal::from(vec![1, 24]))), - ]), - ), - ("b".to_owned(), Value::Int(35)), - ]); + #[derive(Debug, Serialize, Deserialize, PartialEq)] + struct TestSingleValueExternalEnum { + a: SingleValueExternalEnum, + } - let expected_inner = TestInner { a: expected, b: 35 }; - let final_value: TestInner = from_value(&test_inner)?; - assert_eq!(final_value, expected_inner); + #[derive(Debug, Serialize, Deserialize, PartialEq)] + enum SingleValueExternalEnum { + Double(f64), + String(String), + } - Ok(()) - } + #[derive(Debug, Serialize, Deserialize, PartialEq)] + struct TestStructExternalEnum { + a: StructExternalEnum, + } - #[test] - fn test_from_value_unit_enum() -> TestResult { - let expected = TestUnitExternalEnum { - a: UnitExternalEnum::Val1, - }; - - let test = Value::Record(vec![("a".to_owned(), Value::Enum(0, "Val1".to_owned()))]); - let final_value: TestUnitExternalEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "Error deserializing unit external enum" - ); - - let expected = TestUnitInternalEnum { - a: UnitInternalEnum::Val1, - }; - - let test = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), - )]); - let final_value: TestUnitInternalEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "Error deserializing unit internal enum" - ); - let expected = TestUnitAdjacentEnum { - a: UnitAdjacentEnum::Val1, - }; - - let test = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), - )]); - let final_value: TestUnitAdjacentEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "Error deserializing unit adjacent enum" - ); - let expected = TestUnitUntaggedEnum { - a: UnitUntaggedEnum::Val1, - }; - - let test = Value::Record(vec![("a".to_owned(), Value::Null)]); - let final_value: TestUnitUntaggedEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "Error deserializing unit untagged enum" - ); - Ok(()) - } + #[derive(Debug, Serialize, Deserialize, PartialEq)] + enum StructExternalEnum { + Val1 { x: f32, y: f32 }, + Val2 { x: f32, y: f32 }, + } - #[test] - fn avro_3645_3646_test_from_value_enum() -> TestResult { - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - struct TestNullExternalEnum { - a: NullExternalEnum, + #[derive(Debug, Serialize, Deserialize, PartialEq)] + struct TestTupleExternalEnum { + a: TupleExternalEnum, } - #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] - enum NullExternalEnum { - Val1, - Val2(), - Val3(()), - Val4(u64), + #[derive(Debug, Serialize, Deserialize, PartialEq)] + enum TupleExternalEnum { + Val1(f32, f32), + Val2(f32, f32, f32), } - let data = vec![ - ( - TestNullExternalEnum { - a: NullExternalEnum::Val1, - }, - Value::Record(vec![("a".to_owned(), Value::Enum(0, "Val1".to_owned()))]), - ), - ( - TestNullExternalEnum { - a: NullExternalEnum::Val2(), - }, - Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::Enum(1, "Val2".to_owned())), - ("value".to_owned(), Value::Union(1, Box::new(Value::Null))), - ]), - )]), - ), - ( - TestNullExternalEnum { - a: NullExternalEnum::Val2(), - }, - Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::Enum(1, "Val2".to_owned())), - ("value".to_owned(), Value::Array(vec![])), - ]), - )]), - ), - ( - TestNullExternalEnum { - a: NullExternalEnum::Val3(()), - }, - Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::Enum(2, "Val3".to_owned())), - ("value".to_owned(), Value::Union(2, Box::new(Value::Null))), - ]), - )]), - ), - ( - TestNullExternalEnum { - a: NullExternalEnum::Val4(123), - }, - Value::Record(vec![( + #[test] + fn test_from_value() -> TestResult { + let test = Value::Record(vec![ + ("a".to_owned(), Value::Long(27)), + ("b".to_owned(), Value::String("foo".to_owned())), + ("c".to_owned(), Value::Decimal(Decimal::from(vec![1, 24]))), + ]); + let expected = Test { + a: 27, + b: "foo".to_owned(), + c: Decimal::from(vec![1, 24]), + }; + let final_value: Test = from_value(&test)?; + assert_eq!(final_value, expected); + + let test_inner = Value::Record(vec![ + ( "a".to_owned(), Value::Record(vec![ - ("type".to_owned(), Value::Enum(3, "Val4".to_owned())), - ("value".to_owned(), Value::Union(3, Value::Long(123).into())), + ("a".to_owned(), Value::Long(27)), + ("b".to_owned(), Value::String("foo".to_owned())), + ("c".to_owned(), Value::Decimal(Decimal::from(vec![1, 24]))), ]), - )]), - ), - ]; + ), + ("b".to_owned(), Value::Int(35)), + ]); - for (expected, test) in data.iter() { - let actual: TestNullExternalEnum = from_value(test)?; - assert_eq!(actual, *expected); + let expected_inner = TestInner { a: expected, b: 35 }; + let final_value: TestInner = from_value(&test_inner)?; + assert_eq!(final_value, expected_inner); + + Ok(()) } - Ok(()) - } + #[test] + fn test_from_value_unit_enum() -> TestResult { + let expected = TestUnitExternalEnum { + a: UnitExternalEnum::Val1, + }; - #[test] - fn test_from_value_single_value_enum() -> TestResult { - let expected = TestSingleValueExternalEnum { - a: SingleValueExternalEnum::Double(64.0), - }; + let test = Value::Record(vec![("a".to_owned(), Value::Enum(0, "Val1".to_owned()))]); + let final_value: TestUnitExternalEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "Error deserializing unit external enum" + ); - let test = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::String("Double".to_owned())), - ( - "value".to_owned(), - Value::Union(1, Box::new(Value::Double(64.0))), - ), - ]), - )]); - let final_value: TestSingleValueExternalEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "Error deserializing single value external enum(union)" - ); - - Ok(()) - } + let expected = TestUnitInternalEnum { + a: UnitInternalEnum::Val1, + }; - #[test] - fn test_from_value_struct_enum() -> TestResult { - let expected = TestStructExternalEnum { - a: StructExternalEnum::Val1 { x: 1.0, y: 2.0 }, - }; + let test = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), + )]); + let final_value: TestUnitInternalEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "Error deserializing unit internal enum" + ); + let expected = TestUnitAdjacentEnum { + a: UnitAdjacentEnum::Val1, + }; + + let test = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), + )]); + let final_value: TestUnitAdjacentEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "Error deserializing unit adjacent enum" + ); + let expected = TestUnitUntaggedEnum { + a: UnitUntaggedEnum::Val1, + }; + + let test = Value::Record(vec![("a".to_owned(), Value::Null)]); + let final_value: TestUnitUntaggedEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "Error deserializing unit untagged enum" + ); + Ok(()) + } + + #[test] + fn avro_3645_3646_test_from_value_enum() -> TestResult { + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + struct TestNullExternalEnum { + a: NullExternalEnum, + } + + #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] + enum NullExternalEnum { + Val1, + Val2(), + Val3(()), + Val4(u64), + } - let test = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::String("Val1".to_owned())), + let data = vec![ ( - "value".to_owned(), - Value::Union( - 0, - Box::new(Value::Record(vec![ - ("x".to_owned(), Value::Float(1.0)), - ("y".to_owned(), Value::Float(2.0)), - ])), - ), + TestNullExternalEnum { + a: NullExternalEnum::Val1, + }, + Value::Record(vec![("a".to_owned(), Value::Enum(0, "Val1".to_owned()))]), ), - ]), - )]); - let final_value: TestStructExternalEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "error deserializing struct external enum(union)" - ); - - Ok(()) - } + ( + TestNullExternalEnum { + a: NullExternalEnum::Val2(), + }, + Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::Enum(1, "Val2".to_owned())), + ("value".to_owned(), Value::Union(1, Box::new(Value::Null))), + ]), + )]), + ), + ( + TestNullExternalEnum { + a: NullExternalEnum::Val2(), + }, + Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::Enum(1, "Val2".to_owned())), + ("value".to_owned(), Value::Array(vec![])), + ]), + )]), + ), + ( + TestNullExternalEnum { + a: NullExternalEnum::Val3(()), + }, + Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::Enum(2, "Val3".to_owned())), + ("value".to_owned(), Value::Union(2, Box::new(Value::Null))), + ]), + )]), + ), + ( + TestNullExternalEnum { + a: NullExternalEnum::Val4(123), + }, + Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::Enum(3, "Val4".to_owned())), + ("value".to_owned(), Value::Union(3, Value::Long(123).into())), + ]), + )]), + ), + ]; - #[test] - fn test_avro_3692_from_value_struct_flatten() -> TestResult { - #[derive(Deserialize, PartialEq, Debug)] - struct S1 { - f1: String, - #[serde(flatten)] - inner: S2, - } - #[derive(Deserialize, PartialEq, Debug)] - struct S2 { - f2: String, - } - let expected = S1 { - f1: "Hello".to_owned(), - inner: S2 { - f2: "World".to_owned(), - }, - }; - - let test = Value::Record(vec![ - ("f1".to_owned(), "Hello".into()), - ("f2".to_owned(), "World".into()), - ]); - let final_value: S1 = from_value(&test)?; - assert_eq!(final_value, expected); - - Ok(()) - } + for (expected, test) in data.iter() { + let actual: TestNullExternalEnum = from_value(test)?; + assert_eq!(actual, *expected); + } - #[test] - fn test_from_value_tuple_enum() -> TestResult { - let expected = TestTupleExternalEnum { - a: TupleExternalEnum::Val1(1.0, 2.0), - }; + Ok(()) + } - let test = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::String("Val1".to_owned())), - ( - "value".to_owned(), - Value::Union( - 0, - Box::new(Value::Array(vec![Value::Float(1.0), Value::Float(2.0)])), + #[test] + fn test_from_value_single_value_enum() -> TestResult { + let expected = TestSingleValueExternalEnum { + a: SingleValueExternalEnum::Double(64.0), + }; + + let test = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::String("Double".to_owned())), + ( + "value".to_owned(), + Value::Union(1, Box::new(Value::Double(64.0))), ), - ), - ]), - )]); - let final_value: TestTupleExternalEnum = from_value(&test)?; - assert_eq!( - final_value, expected, - "error serializing tuple external enum(union)" - ); - - Ok(()) - } + ]), + )]); + let final_value: TestSingleValueExternalEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "Error deserializing single value external enum(union)" + ); + + Ok(()) + } - #[test] - fn test_date() -> TestResult { - let raw_value = 1; - let value = Value::Date(raw_value); - let result = crate::from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + #[test] + fn test_from_value_struct_enum() -> TestResult { + let expected = TestStructExternalEnum { + a: StructExternalEnum::Val1 { x: 1.0, y: 2.0 }, + }; - #[test] - fn test_time_millis() -> TestResult { - let raw_value = 1; - let value = Value::TimeMillis(raw_value); - let result = crate::from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + let test = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::String("Val1".to_owned())), + ( + "value".to_owned(), + Value::Union( + 0, + Box::new(Value::Record(vec![ + ("x".to_owned(), Value::Float(1.0)), + ("y".to_owned(), Value::Float(2.0)), + ])), + ), + ), + ]), + )]); + let final_value: TestStructExternalEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "error deserializing struct external enum(union)" + ); + + Ok(()) + } - #[test] - fn test_time_micros() -> TestResult { - let raw_value = 1; - let value = Value::TimeMicros(raw_value); - let result = crate::from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + #[test] + fn test_avro_3692_from_value_struct_flatten() -> TestResult { + #[derive(Deserialize, PartialEq, Debug)] + struct S1 { + f1: String, + #[serde(flatten)] + inner: S2, + } + #[derive(Deserialize, PartialEq, Debug)] + struct S2 { + f2: String, + } + let expected = S1 { + f1: "Hello".to_owned(), + inner: S2 { + f2: "World".to_owned(), + }, + }; - #[test] - fn test_timestamp_millis() -> TestResult { - let raw_value = 1; - let value = Value::TimestampMillis(raw_value); - let result = crate::from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + let test = Value::Record(vec![ + ("f1".to_owned(), "Hello".into()), + ("f2".to_owned(), "World".into()), + ]); + let final_value: S1 = from_value(&test)?; + assert_eq!(final_value, expected); - #[test] - fn test_timestamp_micros() -> TestResult { - let raw_value = 1; - let value = Value::TimestampMicros(raw_value); - let result = from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3916_timestamp_nanos() -> TestResult { - let raw_value = 1; - let value = Value::TimestampNanos(raw_value); - let result = from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + #[test] + fn test_from_value_tuple_enum() -> TestResult { + let expected = TestTupleExternalEnum { + a: TupleExternalEnum::Val1(1.0, 2.0), + }; - #[test] - fn test_avro_3853_local_timestamp_millis() -> TestResult { - let raw_value = 1; - let value = Value::LocalTimestampMillis(raw_value); - let result = from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + let test = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::String("Val1".to_owned())), + ( + "value".to_owned(), + Value::Union( + 0, + Box::new(Value::Array(vec![Value::Float(1.0), Value::Float(2.0)])), + ), + ), + ]), + )]); + let final_value: TestTupleExternalEnum = from_value(&test)?; + assert_eq!( + final_value, expected, + "error serializing tuple external enum(union)" + ); + + Ok(()) + } - #[test] - fn test_avro_3853_local_timestamp_micros() -> TestResult { - let raw_value = 1; - let value = Value::LocalTimestampMicros(raw_value); - let result = crate::from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + #[test] + fn test_date() -> TestResult { + let raw_value = 1; + let value = Value::Date(raw_value); + let result = crate::from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } - #[test] - fn test_avro_3916_local_timestamp_nanos() -> TestResult { - let raw_value = 1; - let value = Value::LocalTimestampNanos(raw_value); - let result = crate::from_value::(&value)?; - assert_eq!(result, raw_value); - Ok(()) - } + #[test] + fn test_time_millis() -> TestResult { + let raw_value = 1; + let value = Value::TimeMillis(raw_value); + let result = crate::from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } - #[test] - fn test_from_value_uuid_str() -> TestResult { - let raw_value = "9ec535ff-3e2a-45bd-91d3-0a01321b5a49"; - let value = Value::Uuid(Uuid::parse_str(raw_value)?); - let result = from_value::(&value)?; - assert_eq!(result.to_string(), raw_value); - Ok(()) - } + #[test] + fn test_time_micros() -> TestResult { + let raw_value = 1; + let value = Value::TimeMicros(raw_value); + let result = crate::from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } - #[test] - fn test_from_value_uuid_slice() -> TestResult { - let raw_value = &[4, 54, 67, 12, 43, 2, 2, 76, 32, 50, 87, 5, 1, 33, 43, 87]; - let value = Value::Uuid(Uuid::from_slice(raw_value)?); - let result = crate::from_value::(&value)?; - assert_eq!(result.as_bytes(), raw_value); - Ok(()) - } + #[test] + fn test_timestamp_millis() -> TestResult { + let raw_value = 1; + let value = Value::TimestampMillis(raw_value); + let result = crate::from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } - #[test] - fn test_from_value_with_union() -> TestResult { - // AVRO-3232 test for deserialize_any on missing fields on the destination struct: - // Error: DeserializeValue("Unsupported union") - // Error: DeserializeValue("incorrect value of type: String") - #[derive(Debug, Deserialize, PartialEq, Eq)] - struct RecordInUnion { - record_in_union: i32, + #[test] + fn test_timestamp_micros() -> TestResult { + let raw_value = 1; + let value = Value::TimestampMicros(raw_value); + let result = from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) } - #[derive(Debug, Deserialize, PartialEq, Eq)] - enum EnumInStruct { - Val1, + #[test] + fn test_avro_3916_timestamp_nanos() -> TestResult { + let raw_value = 1; + let value = Value::TimestampNanos(raw_value); + let result = from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } + + #[test] + fn test_avro_3853_local_timestamp_millis() -> TestResult { + let raw_value = 1; + let value = Value::LocalTimestampMillis(raw_value); + let result = from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } + + #[test] + fn test_avro_3853_local_timestamp_micros() -> TestResult { + let raw_value = 1; + let value = Value::LocalTimestampMicros(raw_value); + let result = crate::from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) } - #[derive(Debug, Deserialize, PartialEq, Eq)] - struct StructWithMissingFields { - a_string: String, - a_record: Option, - an_array: Option<[bool; 2]>, - a_union_map: Option>, - an_enum: EnumInStruct, - } - - let raw_map: HashMap = [ - ("long_one".to_string(), 1), - ("long_two".to_string(), 2), - ("long_three".to_string(), 3), - ("time_micros_a".to_string(), 123), - ("timestamp_millis_b".to_string(), 234), - ("timestamp_micros_c".to_string(), 345), - ("timestamp_nanos_d".to_string(), 345_001), - ("local_timestamp_millis_d".to_string(), 678), - ("local_timestamp_micros_e".to_string(), 789), - ("local_timestamp_nanos_f".to_string(), 345_002), - ] - .iter() - .cloned() - .collect(); - - let value_map = raw_map + #[test] + fn test_avro_3916_local_timestamp_nanos() -> TestResult { + let raw_value = 1; + let value = Value::LocalTimestampNanos(raw_value); + let result = crate::from_value::(&value)?; + assert_eq!(result, raw_value); + Ok(()) + } + + #[test] + fn test_from_value_uuid_str() -> TestResult { + let raw_value = "9ec535ff-3e2a-45bd-91d3-0a01321b5a49"; + let value = Value::Uuid(Uuid::parse_str(raw_value)?); + let result = from_value::(&value)?; + assert_eq!(result.to_string(), raw_value); + Ok(()) + } + + #[test] + fn test_from_value_uuid_slice() -> TestResult { + let raw_value = &[4, 54, 67, 12, 43, 2, 2, 76, 32, 50, 87, 5, 1, 33, 43, 87]; + let value = Value::Uuid(Uuid::from_slice(raw_value)?); + let result = crate::from_value::(&value)?; + assert_eq!(result.as_bytes(), raw_value); + Ok(()) + } + + #[test] + fn test_from_value_with_union() -> TestResult { + // AVRO-3232 test for deserialize_any on missing fields on the destination struct: + // Error: DeserializeValue("Unsupported union") + // Error: DeserializeValue("incorrect value of type: String") + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct RecordInUnion { + record_in_union: i32, + } + + #[derive(Debug, Deserialize, PartialEq, Eq)] + enum EnumInStruct { + Val1, + } + + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct StructWithMissingFields { + a_string: String, + a_record: Option, + an_array: Option<[bool; 2]>, + a_union_map: Option>, + an_enum: EnumInStruct, + } + + let raw_map: HashMap = [ + ("long_one".to_string(), 1), + ("long_two".to_string(), 2), + ("long_three".to_string(), 3), + ("time_micros_a".to_string(), 123), + ("timestamp_millis_b".to_string(), 234), + ("timestamp_micros_c".to_string(), 345), + ("timestamp_nanos_d".to_string(), 345_001), + ("local_timestamp_millis_d".to_string(), 678), + ("local_timestamp_micros_e".to_string(), 789), + ("local_timestamp_nanos_f".to_string(), 345_002), + ] .iter() - .map(|(k, v)| match k { - key if key.starts_with("long_") => (k.clone(), Value::Long(*v)), - key if key.starts_with("time_micros_") => (k.clone(), Value::TimeMicros(*v)), - key if key.starts_with("timestamp_millis_") => { - (k.clone(), Value::TimestampMillis(*v)) - } - key if key.starts_with("timestamp_micros_") => { - (k.clone(), Value::TimestampMicros(*v)) - } - key if key.starts_with("timestamp_nanos_") => { - (k.clone(), Value::TimestampNanos(*v)) - } - key if key.starts_with("local_timestamp_millis_") => { - (k.clone(), Value::LocalTimestampMillis(*v)) - } - key if key.starts_with("local_timestamp_micros_") => { - (k.clone(), Value::LocalTimestampMicros(*v)) - } - key if key.starts_with("local_timestamp_nanos_") => { - (k.clone(), Value::LocalTimestampNanos(*v)) - } - _ => unreachable!("unexpected key: {:?}", k), - }) + .cloned() .collect(); - let record = Value::Record(vec![ - ( - "a_string".to_string(), - Value::String("a valid message field".to_string()), - ), - ( - "a_non_existing_string".to_string(), - Value::String("a string".to_string()), - ), - ( - "a_union_string".to_string(), - Value::Union(0, Box::new(Value::String("a union string".to_string()))), - ), - ( - "a_union_long".to_string(), - Value::Union(0, Box::new(Value::Long(412))), - ), - ( - "a_union_long".to_string(), - Value::Union(0, Box::new(Value::Long(412))), - ), - ( - "a_time_micros".to_string(), - Value::Union(0, Box::new(Value::TimeMicros(123))), - ), - ( - "a_non_existing_time_micros".to_string(), - Value::Union(0, Box::new(Value::TimeMicros(-123))), - ), - ( - "a_timestamp_millis".to_string(), - Value::Union(0, Box::new(Value::TimestampMillis(234))), - ), - ( - "a_non_existing_timestamp_millis".to_string(), - Value::Union(0, Box::new(Value::TimestampMillis(-234))), - ), - ( - "a_timestamp_micros".to_string(), - Value::Union(0, Box::new(Value::TimestampMicros(345))), - ), - ( - "a_non_existing_timestamp_micros".to_string(), - Value::Union(0, Box::new(Value::TimestampMicros(-345))), - ), - ( - "a_timestamp_nanos".to_string(), - Value::Union(0, Box::new(Value::TimestampNanos(345))), - ), - ( - "a_non_existing_timestamp_nanos".to_string(), - Value::Union(0, Box::new(Value::TimestampNanos(-345))), - ), - ( - "a_local_timestamp_millis".to_string(), - Value::Union(0, Box::new(Value::LocalTimestampMillis(678))), - ), - ( - "a_non_existing_local_timestamp_millis".to_string(), - Value::Union(0, Box::new(Value::LocalTimestampMillis(-678))), - ), - ( - "a_local_timestamp_micros".to_string(), - Value::Union(0, Box::new(Value::LocalTimestampMicros(789))), - ), - ( - "a_non_existing_local_timestamp_micros".to_string(), - Value::Union(0, Box::new(Value::LocalTimestampMicros(-789))), - ), - ( - "a_local_timestamp_nanos".to_string(), - Value::Union(0, Box::new(Value::LocalTimestampNanos(789))), - ), - ( - "a_non_existing_local_timestamp_nanos".to_string(), - Value::Union(0, Box::new(Value::LocalTimestampNanos(-789))), - ), - ( - "a_record".to_string(), - Value::Union( - 0, - Box::new(Value::Record(vec![( - "record_in_union".to_string(), - Value::Int(-2), - )])), + let value_map = raw_map + .iter() + .map(|(k, v)| match k { + key if key.starts_with("long_") => (k.clone(), Value::Long(*v)), + key if key.starts_with("time_micros_") => (k.clone(), Value::TimeMicros(*v)), + key if key.starts_with("timestamp_millis_") => { + (k.clone(), Value::TimestampMillis(*v)) + } + key if key.starts_with("timestamp_micros_") => { + (k.clone(), Value::TimestampMicros(*v)) + } + key if key.starts_with("timestamp_nanos_") => { + (k.clone(), Value::TimestampNanos(*v)) + } + key if key.starts_with("local_timestamp_millis_") => { + (k.clone(), Value::LocalTimestampMillis(*v)) + } + key if key.starts_with("local_timestamp_micros_") => { + (k.clone(), Value::LocalTimestampMicros(*v)) + } + key if key.starts_with("local_timestamp_nanos_") => { + (k.clone(), Value::LocalTimestampNanos(*v)) + } + _ => unreachable!("unexpected key: {:?}", k), + }) + .collect(); + + let record = Value::Record(vec![ + ( + "a_string".to_string(), + Value::String("a valid message field".to_string()), + ), + ( + "a_non_existing_string".to_string(), + Value::String("a string".to_string()), ), - ), - ( - "a_non_existing_record".to_string(), - Value::Union( - 0, - Box::new(Value::Record(vec![("blah".to_string(), Value::Int(-22))])), + ( + "a_union_string".to_string(), + Value::Union(0, Box::new(Value::String("a union string".to_string()))), ), - ), - ( - "an_array".to_string(), - Value::Union( - 0, - Box::new(Value::Array(vec![ - Value::Boolean(true), - Value::Boolean(false), - ])), + ( + "a_union_long".to_string(), + Value::Union(0, Box::new(Value::Long(412))), ), - ), - ( - "a_non_existing_array".to_string(), - Value::Union( - 0, - Box::new(Value::Array(vec![ - Value::Boolean(false), - Value::Boolean(true), - ])), + ( + "a_union_long".to_string(), + Value::Union(0, Box::new(Value::Long(412))), ), - ), - ( - "a_union_map".to_string(), - Value::Union(0, Box::new(Value::Map(value_map))), - ), - ( - "a_non_existing_union_map".to_string(), - Value::Union(0, Box::new(Value::Map(HashMap::new()))), - ), - ("an_enum".to_string(), Value::Enum(0, "Val1".to_owned())), - ( - "a_non_existing_enum".to_string(), - Value::Enum(0, "AnotherVariant".to_owned()), - ), - ]); - - let deserialized: StructWithMissingFields = crate::from_value(&record)?; - let reference = StructWithMissingFields { - a_string: "a valid message field".to_string(), - a_record: Some(RecordInUnion { - record_in_union: -2, - }), - an_array: Some([true, false]), - a_union_map: Some(raw_map), - an_enum: EnumInStruct::Val1, - }; - assert_eq!(deserialized, reference); - Ok(()) - } + ( + "a_time_micros".to_string(), + Value::Union(0, Box::new(Value::TimeMicros(123))), + ), + ( + "a_non_existing_time_micros".to_string(), + Value::Union(0, Box::new(Value::TimeMicros(-123))), + ), + ( + "a_timestamp_millis".to_string(), + Value::Union(0, Box::new(Value::TimestampMillis(234))), + ), + ( + "a_non_existing_timestamp_millis".to_string(), + Value::Union(0, Box::new(Value::TimestampMillis(-234))), + ), + ( + "a_timestamp_micros".to_string(), + Value::Union(0, Box::new(Value::TimestampMicros(345))), + ), + ( + "a_non_existing_timestamp_micros".to_string(), + Value::Union(0, Box::new(Value::TimestampMicros(-345))), + ), + ( + "a_timestamp_nanos".to_string(), + Value::Union(0, Box::new(Value::TimestampNanos(345))), + ), + ( + "a_non_existing_timestamp_nanos".to_string(), + Value::Union(0, Box::new(Value::TimestampNanos(-345))), + ), + ( + "a_local_timestamp_millis".to_string(), + Value::Union(0, Box::new(Value::LocalTimestampMillis(678))), + ), + ( + "a_non_existing_local_timestamp_millis".to_string(), + Value::Union(0, Box::new(Value::LocalTimestampMillis(-678))), + ), + ( + "a_local_timestamp_micros".to_string(), + Value::Union(0, Box::new(Value::LocalTimestampMicros(789))), + ), + ( + "a_non_existing_local_timestamp_micros".to_string(), + Value::Union(0, Box::new(Value::LocalTimestampMicros(-789))), + ), + ( + "a_local_timestamp_nanos".to_string(), + Value::Union(0, Box::new(Value::LocalTimestampNanos(789))), + ), + ( + "a_non_existing_local_timestamp_nanos".to_string(), + Value::Union(0, Box::new(Value::LocalTimestampNanos(-789))), + ), + ( + "a_record".to_string(), + Value::Union( + 0, + Box::new(Value::Record(vec![( + "record_in_union".to_string(), + Value::Int(-2), + )])), + ), + ), + ( + "a_non_existing_record".to_string(), + Value::Union( + 0, + Box::new(Value::Record(vec![("blah".to_string(), Value::Int(-22))])), + ), + ), + ( + "an_array".to_string(), + Value::Union( + 0, + Box::new(Value::Array(vec![ + Value::Boolean(true), + Value::Boolean(false), + ])), + ), + ), + ( + "a_non_existing_array".to_string(), + Value::Union( + 0, + Box::new(Value::Array(vec![ + Value::Boolean(false), + Value::Boolean(true), + ])), + ), + ), + ( + "a_union_map".to_string(), + Value::Union(0, Box::new(Value::Map(value_map))), + ), + ( + "a_non_existing_union_map".to_string(), + Value::Union(0, Box::new(Value::Map(HashMap::new()))), + ), + ("an_enum".to_string(), Value::Enum(0, "Val1".to_owned())), + ( + "a_non_existing_enum".to_string(), + Value::Enum(0, "AnotherVariant".to_owned()), + ), + ]); + + let deserialized: StructWithMissingFields = crate::from_value(&record)?; + let reference = StructWithMissingFields { + a_string: "a valid message field".to_string(), + a_record: Some(RecordInUnion { + record_in_union: -2, + }), + an_array: Some([true, false]), + a_union_map: Some(raw_map), + an_enum: EnumInStruct::Val1, + }; + assert_eq!(deserialized, reference); + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] - fn avro_3747_human_readable_false() -> TestResult { - use serde::de::Deserializer as SerdeDeserializer; + #[test] + #[serial(serde_is_human_readable)] + fn avro_3747_human_readable_false() -> TestResult { + use serde::de::Deserializer as SerdeDeserializer; - let is_human_readable = false; - crate::util::SERDE_HUMAN_READABLE.store(is_human_readable, Ordering::Release); + let is_human_readable = false; + crate::util::SERDE_HUMAN_READABLE.store(is_human_readable, Ordering::Release); - let deser = &Deserializer::new(&Value::Null); + let deser = &Deserializer::new(&Value::Null); - assert_eq!(deser.is_human_readable(), is_human_readable); + assert_eq!(deser.is_human_readable(), is_human_readable); - Ok(()) - } + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] - fn avro_3747_human_readable_true() -> TestResult { - use serde::de::Deserializer as SerdeDeserializer; + #[test] + #[serial(serde_is_human_readable)] + fn avro_3747_human_readable_true() -> TestResult { + use serde::de::Deserializer as SerdeDeserializer; - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let deser = &Deserializer::new(&Value::Null); + let deser = &Deserializer::new(&Value::Null); - assert!(deser.is_human_readable()); + assert!(deser.is_human_readable()); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3892_deserialize_string_from_bytes() -> TestResult { - let raw_value = vec![1, 2, 3, 4]; - let value = Value::Bytes(raw_value.clone()); - let result = from_value::(&value)?; - assert_eq!(result, String::from_utf8(raw_value)?); - Ok(()) - } + #[test] + fn test_avro_3892_deserialize_string_from_bytes() -> TestResult { + let raw_value = vec![1, 2, 3, 4]; + let value = Value::Bytes(raw_value.clone()); + let result = from_value::(&value)?; + assert_eq!(result, String::from_utf8(raw_value)?); + Ok(()) + } - #[test] - fn test_avro_3892_deserialize_str_from_bytes() -> TestResult { - let raw_value = &[1, 2, 3, 4]; - let value = Value::Bytes(raw_value.to_vec()); - let result = from_value::<&str>(&value)?; - assert_eq!(result, std::str::from_utf8(raw_value)?); - Ok(()) - } + #[test] + fn test_avro_3892_deserialize_str_from_bytes() -> TestResult { + let raw_value = &[1, 2, 3, 4]; + let value = Value::Bytes(raw_value.to_vec()); + let result = from_value::<&str>(&value)?; + assert_eq!(result, std::str::from_utf8(raw_value)?); + Ok(()) + } - #[derive(Debug)] - struct Bytes(Vec); + #[derive(Debug)] + struct Bytes(Vec); - impl<'de> Deserialize<'de> for Bytes { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct BytesVisitor; - impl Visitor<'_> for BytesVisitor { - type Value = Bytes; + impl<'de> Deserialize<'de> for Bytes { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct BytesVisitor; + impl Visitor<'_> for BytesVisitor { + type Value = Bytes; - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a byte array") - } + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a byte array") + } - fn visit_bytes(self, v: &[u8]) -> Result - where - E: serde::de::Error, - { - Ok(Bytes(v.to_vec())) + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + Ok(Bytes(v.to_vec())) + } } + deserializer.deserialize_bytes(BytesVisitor) } - deserializer.deserialize_bytes(BytesVisitor) } - } - #[test] - fn test_avro_3892_deserialize_bytes_from_decimal() -> TestResult { - let expected_bytes = BigInt::from(123456789).to_signed_bytes_be(); - let value = Value::Decimal(Decimal::from(&expected_bytes)); - let raw_bytes = from_value::(&value)?; - assert_eq!(raw_bytes.0, expected_bytes); - - let value = Value::Union(0, Box::new(Value::Decimal(Decimal::from(&expected_bytes)))); - let raw_bytes = from_value::>(&value)?; - assert_eq!(raw_bytes.unwrap().0, expected_bytes); - Ok(()) - } + #[test] + fn test_avro_3892_deserialize_bytes_from_decimal() -> TestResult { + let expected_bytes = BigInt::from(123456789).to_signed_bytes_be(); + let value = Value::Decimal(Decimal::from(&expected_bytes)); + let raw_bytes = from_value::(&value)?; + assert_eq!(raw_bytes.0, expected_bytes); + + let value = Value::Union(0, Box::new(Value::Decimal(Decimal::from(&expected_bytes)))); + let raw_bytes = from_value::>(&value)?; + assert_eq!(raw_bytes.unwrap().0, expected_bytes); + Ok(()) + } - #[test] - fn test_avro_3892_deserialize_bytes_from_uuid() -> TestResult { - let uuid_str = "10101010-2020-2020-2020-101010101010"; - let expected_bytes = Uuid::parse_str(uuid_str)?.as_bytes().to_vec(); - let value = Value::Uuid(Uuid::parse_str(uuid_str)?); - let raw_bytes = from_value::(&value)?; - assert_eq!(raw_bytes.0, expected_bytes); - - let value = Value::Union(0, Box::new(Value::Uuid(Uuid::parse_str(uuid_str)?))); - let raw_bytes = from_value::>(&value)?; - assert_eq!(raw_bytes.unwrap().0, expected_bytes); - Ok(()) - } + #[test] + fn test_avro_3892_deserialize_bytes_from_uuid() -> TestResult { + let uuid_str = "10101010-2020-2020-2020-101010101010"; + let expected_bytes = Uuid::parse_str(uuid_str)?.as_bytes().to_vec(); + let value = Value::Uuid(Uuid::parse_str(uuid_str)?); + let raw_bytes = from_value::(&value)?; + assert_eq!(raw_bytes.0, expected_bytes); + + let value = Value::Union(0, Box::new(Value::Uuid(Uuid::parse_str(uuid_str)?))); + let raw_bytes = from_value::>(&value)?; + assert_eq!(raw_bytes.unwrap().0, expected_bytes); + Ok(()) + } - #[test] - fn test_avro_3892_deserialize_bytes_from_fixed() -> TestResult { - let expected_bytes = vec![1, 2, 3, 4]; - let value = Value::Fixed(4, expected_bytes.clone()); - let raw_bytes = from_value::(&value)?; - assert_eq!(raw_bytes.0, expected_bytes); - - let value = Value::Union(0, Box::new(Value::Fixed(4, expected_bytes.clone()))); - let raw_bytes = from_value::>(&value)?; - assert_eq!(raw_bytes.unwrap().0, expected_bytes); - Ok(()) - } + #[test] + fn test_avro_3892_deserialize_bytes_from_fixed() -> TestResult { + let expected_bytes = vec![1, 2, 3, 4]; + let value = Value::Fixed(4, expected_bytes.clone()); + let raw_bytes = from_value::(&value)?; + assert_eq!(raw_bytes.0, expected_bytes); + + let value = Value::Union(0, Box::new(Value::Fixed(4, expected_bytes.clone()))); + let raw_bytes = from_value::>(&value)?; + assert_eq!(raw_bytes.unwrap().0, expected_bytes); + Ok(()) + } - #[test] - fn test_avro_3892_deserialize_bytes_from_bytes() -> TestResult { - let expected_bytes = vec![1, 2, 3, 4]; - let value = Value::Bytes(expected_bytes.clone()); - let raw_bytes = from_value::(&value)?; - assert_eq!(raw_bytes.0, expected_bytes); - - let value = Value::Union(0, Box::new(Value::Bytes(expected_bytes.clone()))); - let raw_bytes = from_value::>(&value)?; - assert_eq!(raw_bytes.unwrap().0, expected_bytes); - Ok(()) + #[test] + fn test_avro_3892_deserialize_bytes_from_bytes() -> TestResult { + let expected_bytes = vec![1, 2, 3, 4]; + let value = Value::Bytes(expected_bytes.clone()); + let raw_bytes = from_value::(&value)?; + assert_eq!(raw_bytes.0, expected_bytes); + + let value = Value::Union(0, Box::new(Value::Bytes(expected_bytes.clone()))); + let raw_bytes = from_value::>(&value)?; + assert_eq!(raw_bytes.unwrap().0, expected_bytes); + Ok(()) + } } } diff --git a/avro/src/decimal.rs b/avro/src/decimal.rs index c219b5d6..e215c8c5 100644 --- a/avro/src/decimal.rs +++ b/avro/src/decimal.rs @@ -15,179 +15,200 @@ // specific language governing permissions and limitations // under the License. -use crate::{AvroResult, Error, error::Details}; -use num_bigint::{BigInt, Sign}; -use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; - -#[derive(Debug, Clone, Eq)] -pub struct Decimal { - value: BigInt, - len: usize, -} - -impl Serialize for Decimal { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self.to_vec() { - Ok(ref bytes) => serializer.serialize_bytes(bytes), - Err(e) => Err(serde::ser::Error::custom(e)), - } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead + Unpin => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod decimal { + + use crate::{AvroResult, error::tokio::Details, error::tokio::Error}; + use num_bigint::{BigInt, Sign}; + use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; + + #[derive(Debug, Clone, Eq)] + pub struct Decimal { + value: BigInt, + len: usize, } -} -impl<'de> Deserialize<'de> for Decimal { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct DecimalVisitor; - impl<'de> serde::de::Visitor<'de> for DecimalVisitor { - type Value = Decimal; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a byte slice or seq of bytes") - } - fn visit_bytes(self, v: &[u8]) -> Result - where - E: serde::de::Error, - { - Ok(Decimal::from(v)) + impl Serialize for Decimal { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.to_vec() { + Ok(ref bytes) => serializer.serialize_bytes(bytes), + Err(e) => Err(serde::ser::Error::custom(e)), } - fn visit_seq(self, mut seq: A) -> Result - where - A: SeqAccess<'de>, - { - let mut bytes = Vec::new(); - while let Some(value) = seq.next_element::()? { - bytes.push(value); + } + } + impl<'de> Deserialize<'de> for Decimal { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct DecimalVisitor; + impl<'de> serde::de::Visitor<'de> for DecimalVisitor { + type Value = Decimal; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a byte slice or seq of bytes") } - Ok(Decimal::from(bytes)) + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + Ok(Decimal::from(v)) + } + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut bytes = Vec::new(); + while let Some(value) = seq.next_element::()? { + bytes.push(value); + } + + Ok(Decimal::from(bytes)) + } } + deserializer.deserialize_bytes(DecimalVisitor) } - deserializer.deserialize_bytes(DecimalVisitor) } -} -// We only care about value equality, not byte length. Can two equal `BigInt`s have two different -// byte lengths? -impl PartialEq for Decimal { - fn eq(&self, other: &Self) -> bool { - self.value == other.value + // We only care about value equality, not byte length. Can two equal `BigInt`s have two different + // byte lengths? + impl PartialEq for Decimal { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } } -} -impl Decimal { - pub(crate) fn len(&self) -> usize { - self.len - } + impl Decimal { + pub(crate) fn len(&self) -> usize { + self.len + } - pub(crate) fn to_vec(&self) -> AvroResult> { - self.to_sign_extended_bytes_with_len(self.len) - } + pub(crate) fn to_vec(&self) -> AvroResult> { + self.to_sign_extended_bytes_with_len(self.len) + } - pub(crate) fn to_sign_extended_bytes_with_len(&self, len: usize) -> AvroResult> { - let sign_byte = 0xFF * u8::from(self.value.sign() == Sign::Minus); - let mut decimal_bytes = vec![sign_byte; len]; - let raw_bytes = self.value.to_signed_bytes_be(); - let num_raw_bytes = raw_bytes.len(); - let start_byte_index = len.checked_sub(num_raw_bytes).ok_or(Details::SignExtend { - requested: len, - needed: num_raw_bytes, - })?; - decimal_bytes[start_byte_index..].copy_from_slice(&raw_bytes); - Ok(decimal_bytes) + pub(crate) fn to_sign_extended_bytes_with_len(&self, len: usize) -> AvroResult> { + let sign_byte = 0xFF * u8::from(self.value.sign() == Sign::Minus); + let mut decimal_bytes = vec![sign_byte; len]; + let raw_bytes = self.value.to_signed_bytes_be(); + let num_raw_bytes = raw_bytes.len(); + let start_byte_index = len.checked_sub(num_raw_bytes).ok_or(Details::SignExtend { + requested: len, + needed: num_raw_bytes, + })?; + decimal_bytes[start_byte_index..].copy_from_slice(&raw_bytes); + Ok(decimal_bytes) + } } -} -impl From for BigInt { - fn from(decimal: Decimal) -> Self { - decimal.value + impl From for BigInt { + fn from(decimal: Decimal) -> Self { + decimal.value + } } -} -/// Gets the internal byte array representation of a referenced decimal. -/// Usage: -/// ``` -/// use apache_avro::Decimal; -/// use std::convert::TryFrom; -/// -/// let decimal = Decimal::from(vec![1, 24]); -/// let maybe_bytes = >::try_from(&decimal); -/// ``` -impl std::convert::TryFrom<&Decimal> for Vec { - type Error = Error; - - fn try_from(decimal: &Decimal) -> Result { - decimal.to_vec() + /// Gets the internal byte array representation of a referenced decimal. + /// Usage: + /// ``` + /// use apache_avro::Decimal; + /// use std::convert::TryFrom; + /// + /// let decimal = Decimal::from(vec![1, 24]); + /// let maybe_bytes = >::try_from(&decimal); + /// ``` + impl std::convert::TryFrom<&Decimal> for Vec { + type Error = Error; + + fn try_from(decimal: &Decimal) -> Result { + decimal.to_vec() + } } -} -/// Gets the internal byte array representation of an owned decimal. -/// Usage: -/// ``` -/// use apache_avro::Decimal; -/// use std::convert::TryFrom; -/// -/// let decimal = Decimal::from(vec![1, 24]); -/// let maybe_bytes = >::try_from(decimal); -/// ``` -impl std::convert::TryFrom for Vec { - type Error = Error; - - fn try_from(decimal: Decimal) -> Result { - decimal.to_vec() + /// Gets the internal byte array representation of an owned decimal. + /// Usage: + /// ``` + /// use apache_avro::Decimal; + /// use std::convert::TryFrom; + /// + /// let decimal = Decimal::from(vec![1, 24]); + /// let maybe_bytes = >::try_from(decimal); + /// ``` + impl std::convert::TryFrom for Vec { + type Error = Error; + + fn try_from(decimal: Decimal) -> Result { + decimal.to_vec() + } } -} -impl> From for Decimal { - fn from(bytes: T) -> Self { - let bytes_ref = bytes.as_ref(); - Self { - value: BigInt::from_signed_bytes_be(bytes_ref), - len: bytes_ref.len(), + impl> From for Decimal { + fn from(bytes: T) -> Self { + let bytes_ref = bytes.as_ref(); + Self { + value: BigInt::from_signed_bytes_be(bytes_ref), + len: bytes_ref.len(), + } } } -} -#[cfg(test)] -mod tests { - use super::*; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; + #[cfg(test)] + mod tests { + use super::*; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; - #[test] - fn test_decimal_from_bytes_from_ref_decimal() -> TestResult { - let input = vec![1, 24]; - let d = Decimal::from(&input); + #[test] + fn test_decimal_from_bytes_from_ref_decimal() -> TestResult { + let input = vec![1, 24]; + let d = Decimal::from(&input); - let output = >::try_from(&d)?; - assert_eq!(output, input); + let output = >::try_from(&d)?; + assert_eq!(output, input); - Ok(()) - } + Ok(()) + } - #[test] - fn test_decimal_from_bytes_from_owned_decimal() -> TestResult { - let input = vec![1, 24]; - let d = Decimal::from(&input); + #[test] + fn test_decimal_from_bytes_from_owned_decimal() -> TestResult { + let input = vec![1, 24]; + let d = Decimal::from(&input); - let output = >::try_from(d)?; - assert_eq!(output, input); + let output = >::try_from(d)?; + assert_eq!(output, input); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3949_decimal_serde() -> TestResult { - let decimal = Decimal::from(&[1, 2, 3]); + #[test] + fn avro_3949_decimal_serde() -> TestResult { + let decimal = Decimal::from(&[1, 2, 3]); - let ser = serde_json::to_string(&decimal)?; - let de = serde_json::from_str(&ser)?; - std::assert_eq!(decimal, de); + let ser = serde_json::to_string(&decimal)?; + let de = serde_json::from_str(&ser)?; + std::assert_eq!(decimal, de); - Ok(()) + Ok(()) + } } } diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 0313d25c..1d0b29f1 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -22,48 +22,59 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decimal::tokio => decimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, #[tokio::test] => #[test] ); } )] mod decode { - #[synca::cfg(tokio)] + #[cfg(feature = "tokio")] + use futures::FutureExt; + #[cfg(feature = "tokio")] + use futures::TryFutureExt; + #[cfg(feature = "sync")] + use std::io::Read as AvroRead; + #[cfg(feature = "tokio")] + use tokio::io::AsyncRead as AvroRead; + #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; + use crate::util::safe_len; + use crate::util::tokio::{zag_i32, zag_i64}; use crate::{ AvroResult, Error, - bigdecimal::deserialize_big_decimal, - decimal::Decimal, + bigdecimal::tokio::deserialize_big_decimal, + decimal::tokio::Decimal, duration::Duration, - encode::encode_long, - error::Details, - schema::{ + encode::tokio::encode_long, + error::tokio::Details, + schema::tokio::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, }, - types::Value, + types::tokio::Value, }; use std::{borrow::Borrow, collections::HashMap, io::ErrorKind, str::FromStr}; use uuid::Uuid; - use crate::util::safe_len; - #[cfg(feature = "tokio")] - use crate::util::tokio::{zag_i32, zag_i64}; - #[cfg(feature = "sync")] - use crate::util::sync::{zag_i32, zag_i64}; #[inline] - pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { - zag_i64(reader).await.map(Value::Long) + pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { + zag_i64(reader).await?.map(Value::Long) } #[inline] - async fn decode_int(reader: &mut R) -> AvroResult { - zag_i32(reader).await.map(Value::Int) + async fn decode_int(reader: &mut R) -> AvroResult { + zag_i32(reader).await?.map(Value::Int) } #[inline] - pub(crate) async fn decode_len(reader: &mut R) -> AvroResult { + pub(crate) async fn decode_len(reader: &mut R) -> AvroResult { let len = zag_i64(reader).await?; safe_len(usize::try_from(len).map_err(|e| Details::ConvertI64ToUsize(e, len))?) } @@ -72,7 +83,7 @@ mod decode { /// /// Maps and arrays are 0-terminated, 0i64 is also encoded as 0 in Avro reading a length of 0 means /// the end of the map or array. - async fn decode_seq_len(reader: &mut R) -> AvroResult { + async fn decode_seq_len(reader: &mut R) -> AvroResult { let raw_len = zag_i64(reader).await?; safe_len( usize::try_from(match raw_len.cmp(&0) { @@ -88,15 +99,12 @@ mod decode { } /// Decode a `Value` from avro format given its `Schema`. - pub async fn decode( - schema: &Schema, - reader: &mut R, - ) -> AvroResult { + pub async fn decode(schema: &Schema, reader: &mut R) -> AvroResult { let rs = ResolvedSchema::try_from(schema)?; decode_internal(schema, rs.get_names(), &None, reader).await } - pub(crate) async fn decode_internal>( + pub(crate) async fn decode_internal>( schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, @@ -123,13 +131,17 @@ mod decode { } Schema::Decimal(DecimalSchema { ref inner, .. }) => match &**inner { Schema::Fixed { .. } => { - match Box::pin(decode_internal(inner, names, enclosing_namespace, reader)).await? { + match Box::pin(decode_internal(inner, names, enclosing_namespace, reader)) + .await? + { Value::Fixed(_, bytes) => Ok(Value::Decimal(Decimal::from(bytes))), value => Err(Details::FixedValue(value).into()), } } Schema::Bytes => { - match Box::pin(decode_internal(inner, names, enclosing_namespace, reader)).await? { + match Box::pin(decode_internal(inner, names, enclosing_namespace, reader)) + .await? + { Value::Bytes(bytes) => Ok(Value::Decimal(Decimal::from(bytes))), value => Err(Details::BytesValue(value).into()), } @@ -137,8 +149,17 @@ mod decode { schema => Err(Details::ResolveDecimalSchema(schema.into()).into()), }, Schema::BigDecimal => { - match Box::pin(decode_internal(&Schema::Bytes, names, enclosing_namespace, reader)).await? { - Value::Bytes(bytes) => deserialize_big_decimal(&bytes).await.map(Value::BigDecimal), + match Box::pin(decode_internal( + &Schema::Bytes, + names, + enclosing_namespace, + reader, + )) + .await? + { + Value::Bytes(bytes) => { + deserialize_big_decimal(&bytes).await.map(Value::BigDecimal) + } value => Err(Details::BytesValue(value).into()), } } @@ -206,16 +227,16 @@ mod decode { Ok(Value::Uuid(uuid)) } Schema::Int => decode_int(reader).await, - Schema::Date => zag_i32(reader).await.map(Value::Date), - Schema::TimeMillis => zag_i32(reader).await.map(Value::TimeMillis), + Schema::Date => zag_i32(reader).await?.map(Value::Date), + Schema::TimeMillis => zag_i32(reader).await?.map(Value::TimeMillis), Schema::Long => decode_long(reader).await, - Schema::TimeMicros => zag_i64(reader).await.map(Value::TimeMicros), - Schema::TimestampMillis => zag_i64(reader).await.map(Value::TimestampMillis), - Schema::TimestampMicros => zag_i64(reader).await.map(Value::TimestampMicros), - Schema::TimestampNanos => zag_i64(reader).await.map(Value::TimestampNanos), - Schema::LocalTimestampMillis => zag_i64(reader).await.map(Value::LocalTimestampMillis), - Schema::LocalTimestampMicros => zag_i64(reader).await.map(Value::LocalTimestampMicros), - Schema::LocalTimestampNanos => zag_i64(reader).await.map(Value::LocalTimestampNanos), + Schema::TimeMicros => zag_i64(reader).await?.map(Value::TimeMicros), + Schema::TimestampMillis => zag_i64(reader).await?.map(Value::TimestampMillis), + Schema::TimestampMicros => zag_i64(reader).await?.map(Value::TimestampMicros), + Schema::TimestampNanos => zag_i64(reader).await?.map(Value::TimestampNanos), + Schema::LocalTimestampMillis => zag_i64(reader).await?.map(Value::LocalTimestampMillis), + Schema::LocalTimestampMicros => zag_i64(reader).await?.map(Value::LocalTimestampMicros), + Schema::LocalTimestampNanos => zag_i64(reader).await?.map(Value::LocalTimestampNanos), Schema::Duration => { let mut buf = [0u8; 12]; reader @@ -243,7 +264,10 @@ mod decode { Schema::Bytes => { let len = decode_len(reader).await?; let mut buf = vec![0u8; len]; - reader.read_exact(&mut buf).await.map_err(Details::ReadBytes)?; + reader + .read_exact(&mut buf) + .await + .map_err(Details::ReadBytes)?; Ok(Value::Bytes(buf)) } Schema::String => { @@ -282,8 +306,13 @@ mod decode { items.reserve(len); for _ in 0..len { items.push( - Box::pin(decode_internal(&inner.items, names, enclosing_namespace, reader)) - .await?, + Box::pin(decode_internal( + &inner.items, + names, + enclosing_namespace, + reader, + )) + .await?, ); } } @@ -301,8 +330,13 @@ mod decode { items.reserve(len); for _ in 0..len { - match Box::pin(decode_internal(&Schema::String, names, enclosing_namespace, reader)) - .await? + match Box::pin(decode_internal( + &Schema::String, + names, + enclosing_namespace, + reader, + )) + .await? { Value::String(key) => { let value = Box::pin(decode_internal( @@ -334,7 +368,8 @@ mod decode { num_variants: variants.len(), })?; let value = - Box::pin(decode_internal(variant, names, enclosing_namespace, reader)).await?; + Box::pin(decode_internal(variant, names, enclosing_namespace, reader)) + .await?; Ok(Value::Union(index as u32, Box::new(value))) } Err(Details::ReadVariableIntegerBytes(io_err)) => { diff --git a/avro/src/encode.rs b/avro/src/encode.rs index ed41c44a..98c603e6 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -15,388 +15,403 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - AvroResult, - bigdecimal::serialize_big_decimal, - error::Details, - schema::{ - DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, - Schema, SchemaKind, UnionSchema, - }, - types::{Value, ValueKind}, -}; -#[cfg(feature = "tokio")] -use crate::util::tokio::{zig_i32, zig_i64}; -#[cfg(feature = "sync")] -use crate::util::sync::{zig_i32, zig_i64}; -use log::error; -use std::{borrow::Borrow, collections::HashMap, io::Write}; - -/// Encode a `Value` into avro format. -/// -/// **NOTE** This will not perform schema validation. The value is assumed to -/// be valid with regards to the schema. Schema are needed only to guide the -/// encoding for complex type values. -pub fn encode(value: &Value, schema: &Schema, writer: &mut W) -> AvroResult { - let rs = ResolvedSchema::try_from(schema)?; - encode_internal(value, schema, rs.get_names(), &None, writer) -} - -pub(crate) fn encode_bytes + ?Sized, W: Write>( - s: &B, - mut writer: W, -) -> AvroResult { - let bytes = s.as_ref(); - encode_long(bytes.len() as i64, &mut writer)?; - writer - .write(bytes) - .map_err(|e| Details::WriteBytes(e).into()) -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead + Unpin => std::io::Read, + crate::util::tokio => crate::util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod encode { + + use crate::util::tokio::{zig_i32, zig_i64}; + + use crate::{ + AvroResult, + bigdecimal::serialize_big_decimal, + error::Details, + schema::{ + DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, + Schema, SchemaKind, UnionSchema, + }, + types::{Value, ValueKind}, + }; + use log::error; + use std::{borrow::Borrow, collections::HashMap, io::Write}; + + /// Encode a `Value` into avro format. + /// + /// **NOTE** This will not perform schema validation. The value is assumed to + /// be valid with regards to the schema. Schema are needed only to guide the + /// encoding for complex type values. + pub fn encode(value: &Value, schema: &Schema, writer: &mut W) -> AvroResult { + let rs = ResolvedSchema::try_from(schema)?; + encode_internal(value, schema, rs.get_names(), &None, writer) + } -pub(crate) fn encode_long(i: i64, writer: W) -> AvroResult { - zig_i64(i, writer) -} + pub(crate) fn encode_bytes + ?Sized, W: Write>( + s: &B, + mut writer: W, + ) -> AvroResult { + let bytes = s.as_ref(); + encode_long(bytes.len() as i64, &mut writer)?; + writer + .write(bytes) + .map_err(|e| Details::WriteBytes(e).into()) + } -pub(crate) fn encode_int(i: i32, writer: W) -> AvroResult { - zig_i32(i, writer) -} + pub(crate) fn encode_long(i: i64, writer: W) -> AvroResult { + zig_i64(i, writer) + } -pub(crate) fn encode_internal>( - value: &Value, - schema: &Schema, - names: &HashMap, - enclosing_namespace: &Namespace, - writer: &mut W, -) -> AvroResult { - if let Schema::Ref { name } = schema { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - let resolved = names - .get(&fully_qualified_name) - .ok_or(Details::SchemaResolutionError(fully_qualified_name))?; - return encode_internal(value, resolved.borrow(), names, enclosing_namespace, writer); + pub(crate) fn encode_int(i: i32, writer: W) -> AvroResult { + zig_i32(i, writer) } - match value { - Value::Null => { - if let Schema::Union(union) = schema { - match union.schemas.iter().position(|sch| *sch == Schema::Null) { - None => Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Null, - supported_schema: vec![SchemaKind::Null, SchemaKind::Union], + pub(crate) fn encode_internal>( + value: &Value, + schema: &Schema, + names: &HashMap, + enclosing_namespace: &Namespace, + writer: &mut W, + ) -> AvroResult { + if let Schema::Ref { name } = schema { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + let resolved = names + .get(&fully_qualified_name) + .ok_or(Details::SchemaResolutionError(fully_qualified_name))?; + return encode_internal(value, resolved.borrow(), names, enclosing_namespace, writer); + } + + match value { + Value::Null => { + if let Schema::Union(union) = schema { + match union.schemas.iter().position(|sch| *sch == Schema::Null) { + None => Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Null, + supported_schema: vec![SchemaKind::Null, SchemaKind::Union], + } + .into()), + Some(p) => encode_long(p as i64, writer), } - .into()), - Some(p) => encode_long(p as i64, writer), + } else { + Ok(0) } - } else { - Ok(0) } - } - Value::Boolean(b) => writer - .write(&[u8::from(*b)]) - .map_err(|e| Details::WriteBytes(e).into()), - // Pattern | Pattern here to signify that these _must_ have the same encoding. - Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => encode_int(*i, writer), - Value::Long(i) - | Value::TimestampMillis(i) - | Value::TimestampMicros(i) - | Value::TimestampNanos(i) - | Value::LocalTimestampMillis(i) - | Value::LocalTimestampMicros(i) - | Value::LocalTimestampNanos(i) - | Value::TimeMicros(i) => encode_long(*i, writer), - Value::Float(x) => writer - .write(&x.to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Value::Double(x) => writer - .write(&x.to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Value::Decimal(decimal) => match schema { - Schema::Decimal(DecimalSchema { inner, .. }) => match *inner.clone() { - Schema::Fixed(FixedSchema { size, .. }) => { - let bytes = decimal.to_sign_extended_bytes_with_len(size).unwrap(); - let num_bytes = bytes.len(); - if num_bytes != size { - return Err(Details::EncodeDecimalAsFixedError(num_bytes, size).into()); + Value::Boolean(b) => writer + .write(&[u8::from(*b)]) + .map_err(|e| Details::WriteBytes(e).into()), + // Pattern | Pattern here to signify that these _must_ have the same encoding. + Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => encode_int(*i, writer), + Value::Long(i) + | Value::TimestampMillis(i) + | Value::TimestampMicros(i) + | Value::TimestampNanos(i) + | Value::LocalTimestampMillis(i) + | Value::LocalTimestampMicros(i) + | Value::LocalTimestampNanos(i) + | Value::TimeMicros(i) => encode_long(*i, writer), + Value::Float(x) => writer + .write(&x.to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Value::Double(x) => writer + .write(&x.to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Value::Decimal(decimal) => match schema { + Schema::Decimal(DecimalSchema { inner, .. }) => match *inner.clone() { + Schema::Fixed(FixedSchema { size, .. }) => { + let bytes = decimal.to_sign_extended_bytes_with_len(size).unwrap(); + let num_bytes = bytes.len(); + if num_bytes != size { + return Err(Details::EncodeDecimalAsFixedError(num_bytes, size).into()); + } + encode(&Value::Fixed(size, bytes), inner, writer) + } + Schema::Bytes => encode(&Value::Bytes(decimal.try_into()?), inner, writer), + _ => { + Err(Details::ResolveDecimalSchema(SchemaKind::from(*inner.clone())).into()) } - encode(&Value::Fixed(size, bytes), inner, writer) + }, + _ => Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Decimal, + supported_schema: vec![SchemaKind::Decimal], } - Schema::Bytes => encode(&Value::Bytes(decimal.try_into()?), inner, writer), - _ => Err(Details::ResolveDecimalSchema(SchemaKind::from(*inner.clone())).into()), + .into()), }, - _ => Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Decimal, - supported_schema: vec![SchemaKind::Decimal], + &Value::Duration(duration) => { + let slice: [u8; 12] = duration.into(); + writer + .write(&slice) + .map_err(|e| Details::WriteBytes(e).into()) } - .into()), - }, - &Value::Duration(duration) => { - let slice: [u8; 12] = duration.into(); - writer - .write(&slice) - .map_err(|e| Details::WriteBytes(e).into()) - } - Value::Uuid(uuid) => match *schema { - Schema::Uuid | Schema::String => encode_bytes( - // we need the call .to_string() to properly convert ASCII to UTF-8 - #[allow(clippy::unnecessary_to_owned)] - &uuid.to_string(), - writer, - ), - Schema::Fixed(FixedSchema { size, .. }) => { - if size != 16 { - return Err(Details::ConvertFixedToUuid(size).into()); - } + Value::Uuid(uuid) => match *schema { + Schema::Uuid | Schema::String => encode_bytes( + // we need the call .to_string() to properly convert ASCII to UTF-8 + #[allow(clippy::unnecessary_to_owned)] + &uuid.to_string(), + writer, + ), + Schema::Fixed(FixedSchema { size, .. }) => { + if size != 16 { + return Err(Details::ConvertFixedToUuid(size).into()); + } - let bytes = uuid.as_bytes(); - encode_bytes(bytes, writer) - } - _ => Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Uuid, - supported_schema: vec![SchemaKind::Uuid, SchemaKind::Fixed], + let bytes = uuid.as_bytes(); + encode_bytes(bytes, writer) + } + _ => Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Uuid, + supported_schema: vec![SchemaKind::Uuid, SchemaKind::Fixed], + } + .into()), + }, + Value::BigDecimal(bg) => { + let buf: Vec = serialize_big_decimal(bg)?; + writer + .write(buf.as_slice()) + .map_err(|e| Details::WriteBytes(e).into()) } - .into()), - }, - Value::BigDecimal(bg) => { - let buf: Vec = serialize_big_decimal(bg)?; - writer - .write(buf.as_slice()) - .map_err(|e| Details::WriteBytes(e).into()) - } - Value::Bytes(bytes) => match *schema { - Schema::Bytes => encode_bytes(bytes, writer), - Schema::Fixed { .. } => writer + Value::Bytes(bytes) => match *schema { + Schema::Bytes => encode_bytes(bytes, writer), + Schema::Fixed { .. } => writer + .write(bytes.as_slice()) + .map_err(|e| Details::WriteBytes(e).into()), + _ => Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Bytes, + supported_schema: vec![SchemaKind::Bytes, SchemaKind::Fixed], + } + .into()), + }, + Value::String(s) => match *schema { + Schema::String | Schema::Uuid => encode_bytes(s, writer), + Schema::Enum(EnumSchema { ref symbols, .. }) => { + if let Some(index) = symbols.iter().position(|item| item == s) { + encode_int(index as i32, writer) + } else { + error!("Invalid symbol string {:?}.", &s[..]); + Err(Details::GetEnumSymbol(s.clone()).into()) + } + } + _ => Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::String, + supported_schema: vec![SchemaKind::String, SchemaKind::Enum], + } + .into()), + }, + Value::Fixed(_, bytes) => writer .write(bytes.as_slice()) .map_err(|e| Details::WriteBytes(e).into()), - _ => Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Bytes, - supported_schema: vec![SchemaKind::Bytes, SchemaKind::Fixed], - } - .into()), - }, - Value::String(s) => match *schema { - Schema::String | Schema::Uuid => encode_bytes(s, writer), - Schema::Enum(EnumSchema { ref symbols, .. }) => { - if let Some(index) = symbols.iter().position(|item| item == s) { - encode_int(index as i32, writer) + Value::Enum(i, _) => encode_int(*i as i32, writer), + Value::Union(idx, item) => { + if let Schema::Union(ref inner) = *schema { + let inner_schema = inner + .schemas + .get(*idx as usize) + .expect("Invalid Union validation occurred"); + encode_long(*idx as i64, &mut *writer)?; + encode_internal(item, inner_schema, names, enclosing_namespace, &mut *writer) } else { - error!("Invalid symbol string {:?}.", &s[..]); - Err(Details::GetEnumSymbol(s.clone()).into()) - } - } - _ => Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::String, - supported_schema: vec![SchemaKind::String, SchemaKind::Enum], - } - .into()), - }, - Value::Fixed(_, bytes) => writer - .write(bytes.as_slice()) - .map_err(|e| Details::WriteBytes(e).into()), - Value::Enum(i, _) => encode_int(*i as i32, writer), - Value::Union(idx, item) => { - if let Schema::Union(ref inner) = *schema { - let inner_schema = inner - .schemas - .get(*idx as usize) - .expect("Invalid Union validation occurred"); - encode_long(*idx as i64, &mut *writer)?; - encode_internal(item, inner_schema, names, enclosing_namespace, &mut *writer) - } else { - error!("invalid schema type for Union: {schema:?}"); - Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Union, - supported_schema: vec![SchemaKind::Union], + error!("invalid schema type for Union: {schema:?}"); + Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Union, + supported_schema: vec![SchemaKind::Union], + } + .into()) } - .into()) } - } - Value::Array(items) => { - if let Schema::Array(ref inner) = *schema { - if !items.is_empty() { - encode_long(items.len() as i64, &mut *writer)?; - for item in items.iter() { - encode_internal( - item, - &inner.items, - names, - enclosing_namespace, - &mut *writer, - )?; + Value::Array(items) => { + if let Schema::Array(ref inner) = *schema { + if !items.is_empty() { + encode_long(items.len() as i64, &mut *writer)?; + for item in items.iter() { + encode_internal( + item, + &inner.items, + names, + enclosing_namespace, + &mut *writer, + )?; + } } + writer + .write(&[0u8]) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + error!("invalid schema type for Array: {schema:?}"); + Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Array, + supported_schema: vec![SchemaKind::Array], + } + .into()) } - writer - .write(&[0u8]) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - error!("invalid schema type for Array: {schema:?}"); - Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Array, - supported_schema: vec![SchemaKind::Array], - } - .into()) } - } - Value::Map(items) => { - if let Schema::Map(ref inner) = *schema { - if !items.is_empty() { - encode_long(items.len() as i64, &mut *writer)?; - for (key, value) in items { - encode_bytes(key, &mut *writer)?; - encode_internal( - value, - &inner.types, - names, - enclosing_namespace, - &mut *writer, - )?; + Value::Map(items) => { + if let Schema::Map(ref inner) = *schema { + if !items.is_empty() { + encode_long(items.len() as i64, &mut *writer)?; + for (key, value) in items { + encode_bytes(key, &mut *writer)?; + encode_internal( + value, + &inner.types, + names, + enclosing_namespace, + &mut *writer, + )?; + } } + writer + .write(&[0u8]) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + error!("invalid schema type for Map: {schema:?}"); + Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Map, + supported_schema: vec![SchemaKind::Map], + } + .into()) } - writer - .write(&[0u8]) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - error!("invalid schema type for Map: {schema:?}"); - Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Map, - supported_schema: vec![SchemaKind::Map], - } - .into()) } - } - Value::Record(value_fields) => { - if let Schema::Record(RecordSchema { - ref name, - fields: ref schema_fields, - .. - }) = *schema - { - let record_namespace = name.fully_qualified_name(enclosing_namespace).namespace; - - let mut lookup = HashMap::new(); - value_fields.iter().for_each(|(name, field)| { - lookup.insert(name, field); - }); - - let mut written_bytes = 0; - for schema_field in schema_fields.iter() { - let name = &schema_field.name; - let value_opt = lookup.get(name).or_else(|| { - if let Some(aliases) = &schema_field.aliases { - aliases.iter().find_map(|alias| lookup.get(alias)) - } else { - None - } + Value::Record(value_fields) => { + if let Schema::Record(RecordSchema { + ref name, + fields: ref schema_fields, + .. + }) = *schema + { + let record_namespace = name.fully_qualified_name(enclosing_namespace).namespace; + + let mut lookup = HashMap::new(); + value_fields.iter().for_each(|(name, field)| { + lookup.insert(name, field); }); - if let Some(value) = value_opt { - written_bytes += encode_internal( + let mut written_bytes = 0; + for schema_field in schema_fields.iter() { + let name = &schema_field.name; + let value_opt = lookup.get(name).or_else(|| { + if let Some(aliases) = &schema_field.aliases { + aliases.iter().find_map(|alias| lookup.get(alias)) + } else { + None + } + }); + + if let Some(value) = value_opt { + written_bytes += encode_internal( + value, + &schema_field.schema, + names, + &record_namespace, + writer, + )?; + } else { + return Err(Details::NoEntryInLookupTable( + name.clone(), + format!("{lookup:?}"), + ) + .into()); + } + } + Ok(written_bytes) + } else if let Schema::Union(UnionSchema { schemas, .. }) = schema { + let mut union_buffer: Vec = Vec::new(); + for (index, schema) in schemas.iter().enumerate() { + encode_long(index as i64, &mut union_buffer)?; + let encode_res = encode_internal( value, - &schema_field.schema, + schema, names, - &record_namespace, - writer, - )?; - } else { - return Err(Details::NoEntryInLookupTable( - name.clone(), - format!("{lookup:?}"), - ) - .into()); - } - } - Ok(written_bytes) - } else if let Schema::Union(UnionSchema { schemas, .. }) = schema { - let mut union_buffer: Vec = Vec::new(); - for (index, schema) in schemas.iter().enumerate() { - encode_long(index as i64, &mut union_buffer)?; - let encode_res = encode_internal( - value, - schema, - names, - enclosing_namespace, - &mut union_buffer, - ); - match encode_res { - Ok(_) => { - return writer - .write(union_buffer.as_slice()) - .map_err(|e| Details::WriteBytes(e).into()); - } - Err(_) => { - union_buffer.clear(); //undo any partial encoding + enclosing_namespace, + &mut union_buffer, + ); + match encode_res { + Ok(_) => { + return writer + .write(union_buffer.as_slice()) + .map_err(|e| Details::WriteBytes(e).into()); + } + Err(_) => { + union_buffer.clear(); //undo any partial encoding + } } } + Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Record, + supported_schema: vec![SchemaKind::Record, SchemaKind::Union], + } + .into()) + } else { + error!("invalid schema type for Record: {schema:?}"); + Err(Details::EncodeValueAsSchemaError { + value_kind: ValueKind::Record, + supported_schema: vec![SchemaKind::Record, SchemaKind::Union], + } + .into()) } - Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Record, - supported_schema: vec![SchemaKind::Record, SchemaKind::Union], - } - .into()) - } else { - error!("invalid schema type for Record: {schema:?}"); - Err(Details::EncodeValueAsSchemaError { - value_kind: ValueKind::Record, - supported_schema: vec![SchemaKind::Record, SchemaKind::Union], - } - .into()) } } } -} -pub fn encode_to_vec(value: &Value, schema: &Schema) -> AvroResult> { - let mut buffer = Vec::new(); - encode(value, schema, &mut buffer)?; - Ok(buffer) -} - -#[cfg(test)] -#[allow(clippy::expect_fun_call)] -pub(crate) mod tests { - use super::*; - use crate::error::{Details, Error}; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; - use uuid::Uuid; - - pub(crate) fn success(value: &Value, schema: &Schema) -> String { - format!( - "Value: {:?}\n should encode with schema:\n{:?}", - &value, &schema - ) + pub fn encode_to_vec(value: &Value, schema: &Schema) -> AvroResult> { + let mut buffer = Vec::new(); + encode(value, schema, &mut buffer)?; + Ok(buffer) } - #[test] - fn test_encode_empty_array() { - let mut buf = Vec::new(); - let empty: Vec = Vec::new(); - encode( - &Value::Array(empty.clone()), - &Schema::array(Schema::Int), - &mut buf, - ) - .expect(&success(&Value::Array(empty), &Schema::array(Schema::Int))); - assert_eq!(vec![0u8], buf); - } + #[cfg(test)] + #[allow(clippy::expect_fun_call)] + pub(crate) mod tests { + use super::*; + use crate::error::{Details, Error}; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; + use uuid::Uuid; + + pub(crate) fn success(value: &Value, schema: &Schema) -> String { + format!( + "Value: {:?}\n should encode with schema:\n{:?}", + &value, &schema + ) + } - #[test] - fn test_encode_empty_map() { - let mut buf = Vec::new(); - let empty: HashMap = HashMap::new(); - encode( - &Value::Map(empty.clone()), - &Schema::map(Schema::Int), - &mut buf, - ) - .expect(&success(&Value::Map(empty), &Schema::map(Schema::Int))); - assert_eq!(vec![0u8], buf); - } + #[test] + fn test_encode_empty_array() { + let mut buf = Vec::new(); + let empty: Vec = Vec::new(); + encode( + &Value::Array(empty.clone()), + &Schema::array(Schema::Int), + &mut buf, + ) + .expect(&success(&Value::Array(empty), &Schema::array(Schema::Int))); + assert_eq!(vec![0u8], buf); + } + + #[test] + fn test_encode_empty_map() { + let mut buf = Vec::new(); + let empty: HashMap = HashMap::new(); + encode( + &Value::Map(empty.clone()), + &Schema::map(Schema::Int), + &mut buf, + ) + .expect(&success(&Value::Map(empty), &Schema::map(Schema::Int))); + assert_eq!(vec![0u8], buf); + } - #[test] - fn test_avro_3433_recursive_definition_encode_record() { - let mut buf = Vec::new(); - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3433_recursive_definition_encode_record() { + let mut buf = Vec::new(); + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -418,22 +433,22 @@ pub(crate) mod tests { } ] }"#, - ) - .unwrap(); - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = - Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - assert!(!buf.is_empty()); - } + ) + .unwrap(); + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = + Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3433_recursive_definition_encode_array() { - let mut buf = Vec::new(); - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3433_recursive_definition_encode_array() { + let mut buf = Vec::new(); + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -461,27 +476,27 @@ pub(crate) mod tests { } ] }"#, - ) - .unwrap(); - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ("a".into(), Value::Array(vec![inner_value1])), - ( - "b".into(), - Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), - ), - ]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - assert!(!buf.is_empty()); - } + ) + .unwrap(); + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ("a".into(), Value::Array(vec![inner_value1])), + ( + "b".into(), + Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), + ), + ]); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3433_recursive_definition_encode_map() { - let mut buf = Vec::new(); - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3433_recursive_definition_encode_map() { + let mut buf = Vec::new(); + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -506,27 +521,27 @@ pub(crate) mod tests { } ] }"#, - ) - .unwrap(); - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ("a".into(), inner_value1), - ( - "b".into(), - Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), - ), - ]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - assert!(!buf.is_empty()); - } + ) + .unwrap(); + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ("a".into(), inner_value1), + ( + "b".into(), + Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), + ), + ]); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3433_recursive_definition_encode_record_wrapper() { - let mut buf = Vec::new(); - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3433_recursive_definition_encode_record_wrapper() { + let mut buf = Vec::new(); + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -555,25 +570,25 @@ pub(crate) mod tests { } ] }"#, - ) - .unwrap(); - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![( - "j".into(), - Value::Record(vec![("z".into(), Value::Int(6))]), - )]); - let outer_value = - Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - assert!(!buf.is_empty()); - } + ) + .unwrap(); + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![( + "j".into(), + Value::Record(vec![("z".into(), Value::Int(6))]), + )]); + let outer_value = + Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3433_recursive_definition_encode_map_and_array() { - let mut buf = Vec::new(); - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3433_recursive_definition_encode_map_and_array() { + let mut buf = Vec::new(); + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -601,27 +616,27 @@ pub(crate) mod tests { } ] }"#, - ) - .unwrap(); - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ( - "a".into(), - Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), - ), - ("b".into(), Value::Array(vec![inner_value1])), - ]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); - assert!(!buf.is_empty()); - } + ) + .unwrap(); + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ( + "a".into(), + Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), + ), + ("b".into(), Value::Array(vec![inner_value1])), + ]); + encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3433_recursive_definition_encode_union() { - let mut buf = Vec::new(); - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3433_recursive_definition_encode_union() { + let mut buf = Vec::new(); + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -643,30 +658,30 @@ pub(crate) mod tests { } ] }"#, - ) - .unwrap(); - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value1 = Value::Record(vec![ - ("a".into(), Value::Union(1, Box::new(inner_value1))), - ("b".into(), inner_value2.clone()), - ]); - encode(&outer_value1, &schema, &mut buf).expect(&success(&outer_value1, &schema)); - assert!(!buf.is_empty()); - - buf.drain(..); - let outer_value2 = Value::Record(vec![ - ("a".into(), Value::Union(0, Box::new(Value::Null))), - ("b".into(), inner_value2), - ]); - encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value1, &schema)); - assert!(!buf.is_empty()); - } + ) + .unwrap(); + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value1 = Value::Record(vec![ + ("a".into(), Value::Union(1, Box::new(inner_value1))), + ("b".into(), inner_value2.clone()), + ]); + encode(&outer_value1, &schema, &mut buf).expect(&success(&outer_value1, &schema)); + assert!(!buf.is_empty()); + + buf.drain(..); + let outer_value2 = Value::Record(vec![ + ("a".into(), Value::Union(0, Box::new(Value::Null))), + ("b".into(), inner_value2), + ]); + encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value1, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3448_proper_multi_level_encoding_outer_namespace() { - let schema = r#" + #[test] + fn test_avro_3448_proper_multi_level_encoding_outer_namespace() { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -707,55 +722,55 @@ pub(crate) mod tests { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema).unwrap(); + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) - .expect(&success(&outer_record_variation_1, &schema)); - assert!(!buf.is_empty()); - buf.drain(..); - encode(&outer_record_variation_2, &schema, &mut buf) - .expect(&success(&outer_record_variation_2, &schema)); - assert!(!buf.is_empty()); - buf.drain(..); - encode(&outer_record_variation_3, &schema, &mut buf) - .expect(&success(&outer_record_variation_3, &schema)); - assert!(!buf.is_empty()); - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + let mut buf = Vec::new(); + encode(&outer_record_variation_1, &schema, &mut buf) + .expect(&success(&outer_record_variation_1, &schema)); + assert!(!buf.is_empty()); + buf.drain(..); + encode(&outer_record_variation_2, &schema, &mut buf) + .expect(&success(&outer_record_variation_2, &schema)); + assert!(!buf.is_empty()); + buf.drain(..); + encode(&outer_record_variation_3, &schema, &mut buf) + .expect(&success(&outer_record_variation_3, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3448_proper_multi_level_encoding_middle_namespace() { - let schema = r#" + #[test] + fn test_avro_3448_proper_multi_level_encoding_middle_namespace() { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -797,55 +812,55 @@ pub(crate) mod tests { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema).unwrap(); + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) - .expect(&success(&outer_record_variation_1, &schema)); - assert!(!buf.is_empty()); - buf.drain(..); - encode(&outer_record_variation_2, &schema, &mut buf) - .expect(&success(&outer_record_variation_2, &schema)); - assert!(!buf.is_empty()); - buf.drain(..); - encode(&outer_record_variation_3, &schema, &mut buf) - .expect(&success(&outer_record_variation_3, &schema)); - assert!(!buf.is_empty()); - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + let mut buf = Vec::new(); + encode(&outer_record_variation_1, &schema, &mut buf) + .expect(&success(&outer_record_variation_1, &schema)); + assert!(!buf.is_empty()); + buf.drain(..); + encode(&outer_record_variation_2, &schema, &mut buf) + .expect(&success(&outer_record_variation_2, &schema)); + assert!(!buf.is_empty()); + buf.drain(..); + encode(&outer_record_variation_3, &schema, &mut buf) + .expect(&success(&outer_record_variation_3, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn test_avro_3448_proper_multi_level_encoding_inner_namespace() { - let schema = r#" + #[test] + fn test_avro_3448_proper_multi_level_encoding_inner_namespace() { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -888,82 +903,83 @@ pub(crate) mod tests { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema).unwrap(); + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) - .expect(&success(&outer_record_variation_1, &schema)); - assert!(!buf.is_empty()); - buf.drain(..); - encode(&outer_record_variation_2, &schema, &mut buf) - .expect(&success(&outer_record_variation_2, &schema)); - assert!(!buf.is_empty()); - buf.drain(..); - encode(&outer_record_variation_3, &schema, &mut buf) - .expect(&success(&outer_record_variation_3, &schema)); - assert!(!buf.is_empty()); - } - - #[test] - fn test_avro_3585_encode_uuids() { - let value = Value::String(String::from("00000000-0000-0000-0000-000000000000")); - let schema = Schema::Uuid; - let mut buffer = Vec::new(); - let encoded = encode(&value, &schema, &mut buffer); - assert!(encoded.is_ok()); - assert!(!buffer.is_empty()); - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + let mut buf = Vec::new(); + encode(&outer_record_variation_1, &schema, &mut buf) + .expect(&success(&outer_record_variation_1, &schema)); + assert!(!buf.is_empty()); + buf.drain(..); + encode(&outer_record_variation_2, &schema, &mut buf) + .expect(&success(&outer_record_variation_2, &schema)); + assert!(!buf.is_empty()); + buf.drain(..); + encode(&outer_record_variation_3, &schema, &mut buf) + .expect(&success(&outer_record_variation_3, &schema)); + assert!(!buf.is_empty()); + } - #[test] - fn avro_3926_encode_decode_uuid_to_fixed_wrong_schema_size() -> TestResult { - let schema = Schema::Fixed(FixedSchema { - size: 15, - name: "uuid".into(), - aliases: None, - doc: None, - default: None, - attributes: Default::default(), - }); - let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); + #[test] + fn test_avro_3585_encode_uuids() { + let value = Value::String(String::from("00000000-0000-0000-0000-000000000000")); + let schema = Schema::Uuid; + let mut buffer = Vec::new(); + let encoded = encode(&value, &schema, &mut buffer); + assert!(encoded.is_ok()); + assert!(!buffer.is_empty()); + } - let mut buffer = Vec::new(); - match encode(&value, &schema, &mut buffer).map_err(Error::into_details) { - Err(Details::ConvertFixedToUuid(actual)) => { - assert_eq!(actual, 15); + #[test] + fn avro_3926_encode_decode_uuid_to_fixed_wrong_schema_size() -> TestResult { + let schema = Schema::Fixed(FixedSchema { + size: 15, + name: "uuid".into(), + aliases: None, + doc: None, + default: None, + attributes: Default::default(), + }); + let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); + + let mut buffer = Vec::new(); + match encode(&value, &schema, &mut buffer).map_err(Error::into_details) { + Err(Details::ConvertFixedToUuid(actual)) => { + assert_eq!(actual, 15); + } + _ => panic!("Expected Details::ConvertFixedToUuid"), } - _ => panic!("Expected Details::ConvertFixedToUuid"), - } - Ok(()) + Ok(()) + } } } diff --git a/avro/src/error.rs b/avro/src/error.rs index 95aeb2b9..b01c967e 100644 --- a/avro/src/error.rs +++ b/avro/src/error.rs @@ -15,644 +15,671 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - schema::{Name, Schema, SchemaKind, UnionSchema}, - types::{Value, ValueKind}, -}; -use std::{error::Error as _, fmt}; - -/// Errors encounterd by Avro. -/// -/// To inspect the details of the error use [`details`](Self::details) or [`into_details`](Self::into_details) -/// to get a [`Details`] which contains more precise error information. -/// -/// See [`Details`] for all possible errors. -#[derive(thiserror::Error, Debug)] -#[repr(transparent)] -#[error(transparent)] -pub struct Error { - details: Box
, -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + #[tokio::test] => #[test] + ); + } +)] +mod error { + use crate::{ + schema::tokio::{Name, Schema, SchemaKind, UnionSchema}, + types::tokio::{Value, ValueKind}, + }; + use std::{error::Error as _, fmt}; + + /// Errors encounterd by Avro. + /// + /// To inspect the details of the error use [`details`](Self::details) or [`into_details`](Self::into_details) + /// to get a [`Details`] which contains more precise error information. + /// + /// See [`Details`] for all possible errors. + #[derive(thiserror::Error, Debug)] + #[repr(transparent)] + #[error(transparent)] + pub struct Error { + details: Box
, + } -impl Error { - pub fn new(details: Details) -> Self { - Self { - details: Box::new(details), + impl Error { + pub fn new(details: Details) -> Self { + Self { + details: Box::new(details), + } } - } - pub fn details(&self) -> &Details { - &self.details - } + pub fn details(&self) -> &Details { + &self.details + } - pub fn into_details(self) -> Details { - *self.details + pub fn into_details(self) -> Details { + *self.details + } } -} -impl From
for Error { - fn from(details: Details) -> Self { - Self::new(details) + impl From
for Error { + fn from(details: Details) -> Self { + Self::new(details) + } } -} -impl serde::ser::Error for Error { - fn custom(msg: T) -> Self { - Self::new(
::custom(msg)) + impl serde::ser::Error for Error { + fn custom(msg: T) -> Self { + Self::new(
::custom(msg)) + } } -} -impl serde::de::Error for Error { - fn custom(msg: T) -> Self { - Self::new(
::custom(msg)) + impl serde::de::Error for Error { + fn custom(msg: T) -> Self { + Self::new(
::custom(msg)) + } } -} -#[derive(thiserror::Error)] -pub enum Details { - #[error("Bad Snappy CRC32; expected {expected:x} but got {actual:x}")] - SnappyCrc32 { expected: u32, actual: u32 }, + #[derive(thiserror::Error)] + pub enum Details { + #[error("Bad Snappy CRC32; expected {expected:x} but got {actual:x}")] + SnappyCrc32 { expected: u32, actual: u32 }, - #[error("Invalid u8 for bool: {0}")] - BoolValue(u8), + #[error("Invalid u8 for bool: {0}")] + BoolValue(u8), - #[error("Not a fixed value, required for decimal with fixed schema: {0:?}")] - FixedValue(Value), + #[error("Not a fixed value, required for decimal with fixed schema: {0:?}")] + FixedValue(Value), - #[error("Not a bytes value, required for decimal with bytes schema: {0:?}")] - BytesValue(Value), + #[error("Not a bytes value, required for decimal with bytes schema: {0:?}")] + BytesValue(Value), - #[error("Not a string value, required for uuid: {0:?}")] - GetUuidFromStringValue(Value), + #[error("Not a string value, required for uuid: {0:?}")] + GetUuidFromStringValue(Value), - #[error("Two schemas with the same fullname were given: {0:?}")] - NameCollision(String), + #[error("Two schemas with the same fullname were given: {0:?}")] + NameCollision(String), - #[error("Not a fixed or bytes type, required for decimal schema, got: {0:?}")] - ResolveDecimalSchema(SchemaKind), + #[error("Not a fixed or bytes type, required for decimal schema, got: {0:?}")] + ResolveDecimalSchema(SchemaKind), - #[error("Invalid utf-8 string")] - ConvertToUtf8(#[source] std::string::FromUtf8Error), + #[error("Invalid utf-8 string")] + ConvertToUtf8(#[source] std::string::FromUtf8Error), - #[error("Invalid utf-8 string")] - ConvertToUtf8Error(#[source] std::str::Utf8Error), + #[error("Invalid utf-8 string")] + ConvertToUtf8Error(#[source] std::str::Utf8Error), - /// Describes errors happened while validating Avro data. - #[error("Value does not match schema")] - Validation, + /// Describes errors happened while validating Avro data. + #[error("Value does not match schema")] + Validation, - /// Describes errors happened while validating Avro data. - #[error("Value {value:?} does not match schema {schema:?}: Reason: {reason}")] - ValidationWithReason { - value: Value, - schema: Schema, - reason: String, - }, + /// Describes errors happened while validating Avro data. + #[error("Value {value:?} does not match schema {schema:?}: Reason: {reason}")] + ValidationWithReason { + value: Value, + schema: Schema, + reason: String, + }, - #[error("Unable to allocate {desired} bytes (maximum allowed: {maximum})")] - MemoryAllocation { desired: usize, maximum: usize }, + #[error("Unable to allocate {desired} bytes (maximum allowed: {maximum})")] + MemoryAllocation { desired: usize, maximum: usize }, - /// Describe a specific error happening with decimal representation - #[error( - "Number of bytes requested for decimal sign extension {requested} is less than the number of bytes needed to decode {needed}" - )] - SignExtend { requested: usize, needed: usize }, + /// Describe a specific error happening with decimal representation + #[error( + "Number of bytes requested for decimal sign extension {requested} is less than the number of bytes needed to decode {needed}" + )] + SignExtend { requested: usize, needed: usize }, - #[error("Failed to read boolean bytes: {0}")] - ReadBoolean(#[source] std::io::Error), + #[error("Failed to read boolean bytes: {0}")] + ReadBoolean(#[source] std::io::Error), - #[error("Failed to read bytes: {0}")] - ReadBytes(#[source] std::io::Error), + #[error("Failed to read bytes: {0}")] + ReadBytes(#[source] std::io::Error), - #[error("Failed to read string: {0}")] - ReadString(#[source] std::io::Error), + #[error("Failed to read string: {0}")] + ReadString(#[source] std::io::Error), - #[error("Failed to read double: {0}")] - ReadDouble(#[source] std::io::Error), + #[error("Failed to read double: {0}")] + ReadDouble(#[source] std::io::Error), - #[error("Failed to read float: {0}")] - ReadFloat(#[source] std::io::Error), + #[error("Failed to read float: {0}")] + ReadFloat(#[source] std::io::Error), - #[error("Failed to read duration: {0}")] - ReadDuration(#[source] std::io::Error), + #[error("Failed to read duration: {0}")] + ReadDuration(#[source] std::io::Error), - #[error("Failed to read fixed number of bytes '{1}': : {0}")] - ReadFixed(#[source] std::io::Error, usize), + #[error("Failed to read fixed number of bytes '{1}': : {0}")] + ReadFixed(#[source] std::io::Error, usize), - #[error("Failed to convert &str to UUID: {0}")] - ConvertStrToUuid(#[source] uuid::Error), + #[error("Failed to convert &str to UUID: {0}")] + ConvertStrToUuid(#[source] uuid::Error), - #[error("Failed to convert Fixed bytes to UUID. It must be exactly 16 bytes, got {0}")] - ConvertFixedToUuid(usize), + #[error("Failed to convert Fixed bytes to UUID. It must be exactly 16 bytes, got {0}")] + ConvertFixedToUuid(usize), - #[error("Failed to convert Fixed bytes to UUID: {0}")] - ConvertSliceToUuid(#[source] uuid::Error), + #[error("Failed to convert Fixed bytes to UUID: {0}")] + ConvertSliceToUuid(#[source] uuid::Error), - #[error("Map key is not a string; key type is {0:?}")] - MapKeyType(ValueKind), + #[error("Map key is not a string; key type is {0:?}")] + MapKeyType(ValueKind), - #[error("Union index {index} out of bounds: {num_variants}")] - GetUnionVariant { index: i64, num_variants: usize }, + #[error("Union index {index} out of bounds: {num_variants}")] + GetUnionVariant { index: i64, num_variants: usize }, - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Enum symbol index out of bounds: {num_variants}")] - EnumSymbolIndex { index: usize, num_variants: usize }, + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Enum symbol index out of bounds: {num_variants}")] + EnumSymbolIndex { index: usize, num_variants: usize }, - #[error("Enum symbol not found {0}")] - GetEnumSymbol(String), + #[error("Enum symbol not found {0}")] + GetEnumSymbol(String), - #[error("Unable to decode enum index")] - GetEnumUnknownIndexValue, + #[error("Unable to decode enum index")] + GetEnumUnknownIndexValue, - #[error("Scale {scale} is greater than precision {precision}")] - GetScaleAndPrecision { scale: usize, precision: usize }, + #[error("Scale {scale} is greater than precision {precision}")] + GetScaleAndPrecision { scale: usize, precision: usize }, - #[error( - "Fixed type number of bytes {size} is not large enough to hold decimal values of precision {precision}" - )] - GetScaleWithFixedSize { size: usize, precision: usize }, + #[error( + "Fixed type number of bytes {size} is not large enough to hold decimal values of precision {precision}" + )] + GetScaleWithFixedSize { size: usize, precision: usize }, - #[error("Expected Value::Uuid, got: {0:?}")] - GetUuid(Value), + #[error("Expected Value::Uuid, got: {0:?}")] + GetUuid(Value), - #[error("Expected Value::BigDecimal, got: {0:?}")] - GetBigDecimal(Value), + #[error("Expected Value::BigDecimal, got: {0:?}")] + GetBigDecimal(Value), - #[error("Fixed bytes of size 12 expected, got Fixed of size {0}")] - GetDecimalFixedBytes(usize), + #[error("Fixed bytes of size 12 expected, got Fixed of size {0}")] + GetDecimalFixedBytes(usize), - #[error("Expected Value::Duration or Value::Fixed(12), got: {0:?}")] - ResolveDuration(Value), + #[error("Expected Value::Duration or Value::Fixed(12), got: {0:?}")] + ResolveDuration(Value), - #[error("Expected Value::Decimal, Value::Bytes or Value::Fixed, got: {0:?}")] - ResolveDecimal(Value), + #[error("Expected Value::Decimal, Value::Bytes or Value::Fixed, got: {0:?}")] + ResolveDecimal(Value), - #[error("Missing field in record: {0:?}")] - GetField(String), + #[error("Missing field in record: {0:?}")] + GetField(String), - #[error("Unable to convert to u8, got {0:?}")] - GetU8(Value), + #[error("Unable to convert to u8, got {0:?}")] + GetU8(Value), - #[error("Precision {precision} too small to hold decimal values with {num_bytes} bytes")] - ComparePrecisionAndSize { precision: usize, num_bytes: usize }, + #[error("Precision {precision} too small to hold decimal values with {num_bytes} bytes")] + ComparePrecisionAndSize { precision: usize, num_bytes: usize }, - #[error("Cannot convert length to i32: {1}")] - ConvertLengthToI32(#[source] std::num::TryFromIntError, usize), + #[error("Cannot convert length to i32: {1}")] + ConvertLengthToI32(#[source] std::num::TryFromIntError, usize), - #[error("Expected Value::Date or Value::Int, got: {0:?}")] - GetDate(Value), + #[error("Expected Value::Date or Value::Int, got: {0:?}")] + GetDate(Value), - #[error("Expected Value::TimeMillis or Value::Int, got: {0:?}")] - GetTimeMillis(Value), + #[error("Expected Value::TimeMillis or Value::Int, got: {0:?}")] + GetTimeMillis(Value), - #[error("Expected Value::TimeMicros, Value::Long or Value::Int, got: {0:?}")] - GetTimeMicros(Value), + #[error("Expected Value::TimeMicros, Value::Long or Value::Int, got: {0:?}")] + GetTimeMicros(Value), - #[error("Expected Value::TimestampMillis, Value::Long or Value::Int, got: {0:?}")] - GetTimestampMillis(Value), + #[error("Expected Value::TimestampMillis, Value::Long or Value::Int, got: {0:?}")] + GetTimestampMillis(Value), - #[error("Expected Value::TimestampMicros, Value::Long or Value::Int, got: {0:?}")] - GetTimestampMicros(Value), + #[error("Expected Value::TimestampMicros, Value::Long or Value::Int, got: {0:?}")] + GetTimestampMicros(Value), - #[error("Expected Value::TimestampNanos, Value::Long or Value::Int, got: {0:?}")] - GetTimestampNanos(Value), + #[error("Expected Value::TimestampNanos, Value::Long or Value::Int, got: {0:?}")] + GetTimestampNanos(Value), - #[error("Expected Value::LocalTimestampMillis, Value::Long or Value::Int, got: {0:?}")] - GetLocalTimestampMillis(Value), + #[error("Expected Value::LocalTimestampMillis, Value::Long or Value::Int, got: {0:?}")] + GetLocalTimestampMillis(Value), - #[error("Expected Value::LocalTimestampMicros, Value::Long or Value::Int, got: {0:?}")] - GetLocalTimestampMicros(Value), + #[error("Expected Value::LocalTimestampMicros, Value::Long or Value::Int, got: {0:?}")] + GetLocalTimestampMicros(Value), - #[error("Expected Value::LocalTimestampNanos, Value::Long or Value::Int, got: {0:?}")] - GetLocalTimestampNanos(Value), + #[error("Expected Value::LocalTimestampNanos, Value::Long or Value::Int, got: {0:?}")] + GetLocalTimestampNanos(Value), - #[error("Expected Value::Null, got: {0:?}")] - GetNull(Value), + #[error("Expected Value::Null, got: {0:?}")] + GetNull(Value), - #[error("Expected Value::Boolean, got: {0:?}")] - GetBoolean(Value), + #[error("Expected Value::Boolean, got: {0:?}")] + GetBoolean(Value), - #[error("Expected Value::Int, got: {0:?}")] - GetInt(Value), + #[error("Expected Value::Int, got: {0:?}")] + GetInt(Value), - #[error("Expected Value::Long or Value::Int, got: {0:?}")] - GetLong(Value), + #[error("Expected Value::Long or Value::Int, got: {0:?}")] + GetLong(Value), - #[error(r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"#)] - GetDouble(Value), + #[error(r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"# + )] + GetDouble(Value), - #[error(r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"#)] - GetFloat(Value), + #[error(r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"# + )] + GetFloat(Value), - #[error("Expected Value::Bytes, got: {0:?}")] - GetBytes(Value), + #[error("Expected Value::Bytes, got: {0:?}")] + GetBytes(Value), - #[error("Expected Value::String, Value::Bytes or Value::Fixed, got: {0:?}")] - GetString(Value), + #[error("Expected Value::String, Value::Bytes or Value::Fixed, got: {0:?}")] + GetString(Value), - #[error("Expected Value::Enum, got: {0:?}")] - GetEnum(Value), + #[error("Expected Value::Enum, got: {0:?}")] + GetEnum(Value), - #[error("Fixed size mismatch, expected: {size}, got: {n}")] - CompareFixedSizes { size: usize, n: usize }, + #[error("Fixed size mismatch, expected: {size}, got: {n}")] + CompareFixedSizes { size: usize, n: usize }, - #[error("String expected for fixed, got: {0:?}")] - GetStringForFixed(Value), + #[error("String expected for fixed, got: {0:?}")] + GetStringForFixed(Value), - #[error("Enum default {symbol:?} is not among allowed symbols {symbols:?}")] - GetEnumDefault { - symbol: String, - symbols: Vec, - }, + #[error("Enum default {symbol:?} is not among allowed symbols {symbols:?}")] + GetEnumDefault { + symbol: String, + symbols: Vec, + }, - #[error("Enum value index {index} is out of bounds {nsymbols}")] - GetEnumValue { index: usize, nsymbols: usize }, + #[error("Enum value index {index} is out of bounds {nsymbols}")] + GetEnumValue { index: usize, nsymbols: usize }, - #[error("Key {0} not found in decimal metadata JSON")] - GetDecimalMetadataFromJson(&'static str), + #[error("Key {0} not found in decimal metadata JSON")] + GetDecimalMetadataFromJson(&'static str), - #[error("Could not find matching type in {schema:?} for {value:?}")] - FindUnionVariant { schema: UnionSchema, value: Value }, + #[error("Could not find matching type in {schema:?} for {value:?}")] + FindUnionVariant { schema: UnionSchema, value: Value }, - #[error("Union type should not be empty")] - EmptyUnion, + #[error("Union type should not be empty")] + EmptyUnion, - #[error("Array({expected:?}) expected, got {other:?}")] - GetArray { expected: SchemaKind, other: Value }, + #[error("Array({expected:?}) expected, got {other:?}")] + GetArray { expected: SchemaKind, other: Value }, - #[error("Map({expected:?}) expected, got {other:?}")] - GetMap { expected: SchemaKind, other: Value }, + #[error("Map({expected:?}) expected, got {other:?}")] + GetMap { expected: SchemaKind, other: Value }, - #[error("Record with fields {expected:?} expected, got {other:?}")] - GetRecord { - expected: Vec<(String, SchemaKind)>, - other: Value, - }, + #[error("Record with fields {expected:?} expected, got {other:?}")] + GetRecord { + expected: Vec<(String, SchemaKind)>, + other: Value, + }, - #[error("No `name` field")] - GetNameField, + #[error("No `name` field")] + GetNameField, - #[error("No `name` in record field")] - GetNameFieldFromRecord, + #[error("No `name` in record field")] + GetNameFieldFromRecord, - #[error("Unions may not directly contain a union")] - GetNestedUnion, + #[error("Unions may not directly contain a union")] + GetNestedUnion, - #[error("Unions cannot contain duplicate types")] - GetUnionDuplicate, + #[error("Unions cannot contain duplicate types")] + GetUnionDuplicate, - #[error("One union type {0:?} must match the `default`'s value type {1:?}")] - GetDefaultUnion(SchemaKind, ValueKind), + #[error("One union type {0:?} must match the `default`'s value type {1:?}")] + GetDefaultUnion(SchemaKind, ValueKind), - #[error("`default`'s value type of field {0:?} in {1:?} must be {2:?}")] - GetDefaultRecordField(String, String, String), + #[error("`default`'s value type of field {0:?} in {1:?} must be {2:?}")] + GetDefaultRecordField(String, String, String), - #[error("JSON value {0} claims to be u64 but cannot be converted")] - GetU64FromJson(serde_json::Number), + #[error("JSON value {0} claims to be u64 but cannot be converted")] + GetU64FromJson(serde_json::Number), - #[error("JSON value {0} claims to be i64 but cannot be converted")] - GetI64FromJson(serde_json::Number), + #[error("JSON value {0} claims to be i64 but cannot be converted")] + GetI64FromJson(serde_json::Number), - #[error("Cannot convert u64 to usize: {1}")] - ConvertU64ToUsize(#[source] std::num::TryFromIntError, u64), + #[error("Cannot convert u64 to usize: {1}")] + ConvertU64ToUsize(#[source] std::num::TryFromIntError, u64), - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Cannot convert u32 to usize: {1}")] - ConvertU32ToUsize(#[source] std::num::TryFromIntError, u32), + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Cannot convert u32 to usize: {1}")] + ConvertU32ToUsize(#[source] std::num::TryFromIntError, u32), - #[error("Cannot convert i64 to usize: {1}")] - ConvertI64ToUsize(#[source] std::num::TryFromIntError, i64), + #[error("Cannot convert i64 to usize: {1}")] + ConvertI64ToUsize(#[source] std::num::TryFromIntError, i64), - #[error("Cannot convert i32 to usize: {1}")] - ConvertI32ToUsize(#[source] std::num::TryFromIntError, i32), + #[error("Cannot convert i32 to usize: {1}")] + ConvertI32ToUsize(#[source] std::num::TryFromIntError, i32), - #[error("Invalid JSON value for decimal precision/scale integer: {0}")] - GetPrecisionOrScaleFromJson(serde_json::Number), + #[error("Invalid JSON value for decimal precision/scale integer: {0}")] + GetPrecisionOrScaleFromJson(serde_json::Number), - #[error("Failed to parse schema from JSON")] - ParseSchemaJson(#[source] serde_json::Error), + #[error("Failed to parse schema from JSON")] + ParseSchemaJson(#[source] serde_json::Error), - #[error("Failed to read schema")] - ReadSchemaFromReader(#[source] std::io::Error), + #[error("Failed to read schema")] + ReadSchemaFromReader(#[source] std::io::Error), - #[error("Must be a JSON string, object or array")] - ParseSchemaFromValidJson, + #[error("Must be a JSON string, object or array")] + ParseSchemaFromValidJson, - #[error("Unknown primitive type: {0}")] - ParsePrimitive(String), + #[error("Unknown primitive type: {0}")] + ParsePrimitive(String), - #[error("invalid JSON for {key:?}: {value:?}")] - GetDecimalMetadataValueFromJson { - key: String, - value: serde_json::Value, - }, + #[error("invalid JSON for {key:?}: {value:?}")] + GetDecimalMetadataValueFromJson { + key: String, + value: serde_json::Value, + }, - #[error("The decimal precision ({precision}) must be bigger or equal to the scale ({scale})")] - DecimalPrecisionLessThanScale { precision: usize, scale: usize }, + #[error( + "The decimal precision ({precision}) must be bigger or equal to the scale ({scale})" + )] + DecimalPrecisionLessThanScale { precision: usize, scale: usize }, - #[error("The decimal precision ({precision}) must be a positive number")] - DecimalPrecisionMuBePositive { precision: usize }, + #[error("The decimal precision ({precision}) must be a positive number")] + DecimalPrecisionMuBePositive { precision: usize }, - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Unreadable big decimal sign")] - BigDecimalSign, + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Unreadable big decimal sign")] + BigDecimalSign, - #[error("Unreadable length for big decimal inner bytes: {0}")] - BigDecimalLen(#[source] Box), + #[error("Unreadable length for big decimal inner bytes: {0}")] + BigDecimalLen(#[source] Box), - #[error("Unreadable big decimal scale")] - BigDecimalScale, + #[error("Unreadable big decimal scale")] + BigDecimalScale, - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Unexpected `type` {0} variant for `logicalType`")] - GetLogicalTypeVariant(serde_json::Value), + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Unexpected `type` {0} variant for `logicalType`")] + GetLogicalTypeVariant(serde_json::Value), - #[error("No `type` field found for `logicalType`")] - GetLogicalTypeField, + #[error("No `type` field found for `logicalType`")] + GetLogicalTypeField, - #[error("logicalType must be a string, but is {0:?}")] - GetLogicalTypeFieldType(serde_json::Value), + #[error("logicalType must be a string, but is {0:?}")] + GetLogicalTypeFieldType(serde_json::Value), - #[error("Unknown complex type: {0}")] - GetComplexType(serde_json::Value), + #[error("Unknown complex type: {0}")] + GetComplexType(serde_json::Value), - #[error("No `type` in complex type")] - GetComplexTypeField, + #[error("No `type` in complex type")] + GetComplexTypeField, - #[error("No `fields` in record")] - GetRecordFieldsJson, + #[error("No `fields` in record")] + GetRecordFieldsJson, - #[error("No `symbols` field in enum")] - GetEnumSymbolsField, + #[error("No `symbols` field in enum")] + GetEnumSymbolsField, - #[error("Unable to parse `symbols` in enum")] - GetEnumSymbols, + #[error("Unable to parse `symbols` in enum")] + GetEnumSymbols, - #[error("Invalid enum symbol name {0}")] - EnumSymbolName(String), + #[error("Invalid enum symbol name {0}")] + EnumSymbolName(String), - #[error("Invalid field name {0}")] - FieldName(String), + #[error("Invalid field name {0}")] + FieldName(String), - #[error("Duplicate field name {0}")] - FieldNameDuplicate(String), + #[error("Duplicate field name {0}")] + FieldNameDuplicate(String), - #[error("Invalid schema name {0}. It must match the regex '{1}'")] - InvalidSchemaName(String, &'static str), + #[error("Invalid schema name {0}. It must match the regex '{1}'")] + InvalidSchemaName(String, &'static str), - #[error("Invalid namespace {0}. It must match the regex '{1}'")] - InvalidNamespace(String, &'static str), + #[error("Invalid namespace {0}. It must match the regex '{1}'")] + InvalidNamespace(String, &'static str), - #[error( - "Invalid schema: There is no type called '{0}', if you meant to define a non-primitive schema, it should be defined inside `type` attribute. Please review the specification" - )] - InvalidSchemaRecord(String), + #[error( + "Invalid schema: There is no type called '{0}', if you meant to define a non-primitive schema, it should be defined inside `type` attribute. Please review the specification" + )] + InvalidSchemaRecord(String), - #[error("Duplicate enum symbol {0}")] - EnumSymbolDuplicate(String), + #[error("Duplicate enum symbol {0}")] + EnumSymbolDuplicate(String), - #[error("Default value for enum must be a string! Got: {0}")] - EnumDefaultWrongType(serde_json::Value), + #[error("Default value for enum must be a string! Got: {0}")] + EnumDefaultWrongType(serde_json::Value), - #[error("No `items` in array")] - GetArrayItemsField, + #[error("No `items` in array")] + GetArrayItemsField, - #[error("No `values` in map")] - GetMapValuesField, + #[error("No `values` in map")] + GetMapValuesField, - #[error("Fixed schema `size` value must be a positive integer: {0}")] - GetFixedSizeFieldPositive(serde_json::Value), + #[error("Fixed schema `size` value must be a positive integer: {0}")] + GetFixedSizeFieldPositive(serde_json::Value), - #[error("Fixed schema has no `size`")] - GetFixedSizeField, + #[error("Fixed schema has no `size`")] + GetFixedSizeField, - #[error("Fixed schema's default value length ({0}) does not match its size ({1})")] - FixedDefaultLenSizeMismatch(usize, u64), + #[error("Fixed schema's default value length ({0}) does not match its size ({1})")] + FixedDefaultLenSizeMismatch(usize, u64), - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Failed to compress with flate: {0}")] - DeflateCompress(#[source] std::io::Error), + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Failed to compress with flate: {0}")] + DeflateCompress(#[source] std::io::Error), - // no longer possible after migration from libflate to miniz_oxide - #[deprecated(since = "0.19.0", note = "This error can no longer occur")] - #[error("Failed to finish flate compressor: {0}")] - DeflateCompressFinish(#[source] std::io::Error), + // no longer possible after migration from libflate to miniz_oxide + #[deprecated(since = "0.19.0", note = "This error can no longer occur")] + #[error("Failed to finish flate compressor: {0}")] + DeflateCompressFinish(#[source] std::io::Error), - #[error("Failed to decompress with flate: {0}")] - DeflateDecompress(#[source] std::io::Error), + #[error("Failed to decompress with flate: {0}")] + DeflateDecompress(#[source] std::io::Error), - #[cfg(feature = "snappy")] - #[error("Failed to compress with snappy: {0}")] - SnappyCompress(#[source] snap::Error), + #[cfg(feature = "snappy")] + #[error("Failed to compress with snappy: {0}")] + SnappyCompress(#[source] snap::Error), - #[cfg(feature = "snappy")] - #[error("Failed to get snappy decompression length: {0}")] - GetSnappyDecompressLen(#[source] snap::Error), + #[cfg(feature = "snappy")] + #[error("Failed to get snappy decompression length: {0}")] + GetSnappyDecompressLen(#[source] snap::Error), - #[cfg(feature = "snappy")] - #[error("Failed to decompress with snappy: {0}")] - SnappyDecompress(#[source] snap::Error), + #[cfg(feature = "snappy")] + #[error("Failed to decompress with snappy: {0}")] + SnappyDecompress(#[source] snap::Error), - #[error("Failed to compress with zstd: {0}")] - ZstdCompress(#[source] std::io::Error), + #[error("Failed to compress with zstd: {0}")] + ZstdCompress(#[source] std::io::Error), - #[error("Failed to decompress with zstd: {0}")] - ZstdDecompress(#[source] std::io::Error), + #[error("Failed to decompress with zstd: {0}")] + ZstdDecompress(#[source] std::io::Error), - #[error("Failed to read header: {0}")] - ReadHeader(#[source] std::io::Error), + #[error("Failed to read header: {0}")] + ReadHeader(#[source] std::io::Error), - #[error("wrong magic in header")] - HeaderMagic, + #[error("wrong magic in header")] + HeaderMagic, - #[error("Message Header mismatch. Expected: {0:?}. Actual: {1:?}")] - SingleObjectHeaderMismatch(Vec, Vec), + #[error("Message Header mismatch. Expected: {0:?}. Actual: {1:?}")] + SingleObjectHeaderMismatch(Vec, Vec), - #[error("Failed to get JSON from avro.schema key in map")] - GetAvroSchemaFromMap, + #[error("Failed to get JSON from avro.schema key in map")] + GetAvroSchemaFromMap, - #[error("no metadata in header")] - GetHeaderMetadata, + #[error("no metadata in header")] + GetHeaderMetadata, - #[error("Failed to read marker bytes: {0}")] - ReadMarker(#[source] std::io::Error), + #[error("Failed to read marker bytes: {0}")] + ReadMarker(#[source] std::io::Error), - #[error("Failed to read block marker bytes: {0}")] - ReadBlockMarker(#[source] std::io::Error), + #[error("Failed to read block marker bytes: {0}")] + ReadBlockMarker(#[source] std::io::Error), - #[error("Read into buffer failed: {0}")] - ReadIntoBuf(#[source] std::io::Error), + #[error("Read into buffer failed: {0}")] + ReadIntoBuf(#[source] std::io::Error), - #[error( - "Invalid sync marker! The sync marker in the data block \ + #[error( + "Invalid sync marker! The sync marker in the data block \ doesn't match the file header's sync marker. This likely \ indicates data corruption, truncated file, or incorrectly \ concatenated Avro files. Verify file integrity and ensure \ proper file transmission or creation." - )] - GetBlockMarker, + )] + GetBlockMarker, - #[error("Overflow when decoding integer value")] - IntegerOverflow, + #[error("Overflow when decoding integer value")] + IntegerOverflow, - #[error("Failed to read bytes for decoding variable length integer: {0}")] - ReadVariableIntegerBytes(#[source] std::io::Error), + #[error("Failed to read bytes for decoding variable length integer: {0}")] + ReadVariableIntegerBytes(#[source] std::io::Error), - #[error("Decoded integer out of range for i32: {1}: {0}")] - ZagI32(#[source] std::num::TryFromIntError, i64), + #[error("Decoded integer out of range for i32: {1}: {0}")] + ZagI32(#[source] std::num::TryFromIntError, i64), - #[error("unable to read block")] - ReadBlock, + #[error("unable to read block")] + ReadBlock, - #[error("Failed to serialize value into Avro value: {0}")] - SerializeValue(String), + #[error("Failed to serialize value into Avro value: {0}")] + SerializeValue(String), - #[error("Failed to serialize value of type {value_type} using schema {schema:?}: {value}")] - SerializeValueWithSchema { - value_type: &'static str, - value: String, - schema: Schema, - }, + #[error("Failed to serialize value of type {value_type} using schema {schema:?}: {value}")] + SerializeValueWithSchema { + value_type: &'static str, + value: String, + schema: Schema, + }, - #[error("Failed to serialize field '{field_name}' for record {record_schema:?}: {error}")] - SerializeRecordFieldWithSchema { - field_name: &'static str, - record_schema: Schema, - error: Box, - }, + #[error("Failed to serialize field '{field_name}' for record {record_schema:?}: {error}")] + SerializeRecordFieldWithSchema { + field_name: &'static str, + record_schema: Schema, + error: Box, + }, - #[error("Failed to deserialize Avro value into value: {0}")] - DeserializeValue(String), + #[error("Failed to deserialize Avro value into value: {0}")] + DeserializeValue(String), - #[error("Failed to write buffer bytes during flush: {0}")] - WriteBytes(#[source] std::io::Error), + #[error("Failed to write buffer bytes during flush: {0}")] + WriteBytes(#[source] std::io::Error), - #[error("Failed to flush inner writer during flush: {0}")] - FlushWriter(#[source] std::io::Error), + #[error("Failed to flush inner writer during flush: {0}")] + FlushWriter(#[source] std::io::Error), - #[error("Failed to write marker: {0}")] - WriteMarker(#[source] std::io::Error), + #[error("Failed to write marker: {0}")] + WriteMarker(#[source] std::io::Error), - #[error("Failed to convert JSON to string: {0}")] - ConvertJsonToString(#[source] serde_json::Error), + #[error("Failed to convert JSON to string: {0}")] + ConvertJsonToString(#[source] serde_json::Error), - /// Error while converting float to json value - #[error("failed to convert avro float to json: {0}")] - ConvertF64ToJson(f64), + /// Error while converting float to json value + #[error("failed to convert avro float to json: {0}")] + ConvertF64ToJson(f64), - /// Error while resolving Schema::Ref - #[error("Unresolved schema reference: {0}")] - SchemaResolutionError(Name), + /// Error while resolving Schema::Ref + #[error("Unresolved schema reference: {0}")] + SchemaResolutionError(Name), - #[error("The file metadata is already flushed.")] - FileHeaderAlreadyWritten, + #[error("The file metadata is already flushed.")] + FileHeaderAlreadyWritten, - #[error("Metadata keys starting with 'avro.' are reserved for internal usage: {0}.")] - InvalidMetadataKey(String), + #[error("Metadata keys starting with 'avro.' are reserved for internal usage: {0}.")] + InvalidMetadataKey(String), - /// Error when two named schema have the same fully qualified name - #[error("Two named schema defined for same fullname: {0}.")] - AmbiguousSchemaDefinition(Name), + /// Error when two named schema have the same fully qualified name + #[error("Two named schema defined for same fullname: {0}.")] + AmbiguousSchemaDefinition(Name), - #[error("Signed decimal bytes length {0} not equal to fixed schema size {1}.")] - EncodeDecimalAsFixedError(usize, usize), + #[error("Signed decimal bytes length {0} not equal to fixed schema size {1}.")] + EncodeDecimalAsFixedError(usize, usize), - #[error("There is no entry for '{0}' in the lookup table: {1}.")] - NoEntryInLookupTable(String, String), + #[error("There is no entry for '{0}' in the lookup table: {1}.")] + NoEntryInLookupTable(String, String), - #[error("Can only encode value type {value_kind:?} as one of {supported_schema:?}")] - EncodeValueAsSchemaError { - value_kind: ValueKind, - supported_schema: Vec, - }, - #[error("Internal buffer not drained properly. Re-initialize the single object writer struct!")] - IllegalSingleObjectWriterState, + #[error("Can only encode value type {value_kind:?} as one of {supported_schema:?}")] + EncodeValueAsSchemaError { + value_kind: ValueKind, + supported_schema: Vec, + }, + #[error( + "Internal buffer not drained properly. Re-initialize the single object writer struct!" + )] + IllegalSingleObjectWriterState, - #[error("Codec '{0}' is not supported/enabled")] - CodecNotSupported(String), + #[error("Codec '{0}' is not supported/enabled")] + CodecNotSupported(String), - #[error("Invalid Avro data! Cannot read codec type from value that is not Value::Bytes.")] - BadCodecMetadata, + #[error("Invalid Avro data! Cannot read codec type from value that is not Value::Bytes.")] + BadCodecMetadata, - #[error("Cannot convert a slice to Uuid: {0}")] - UuidFromSlice(#[source] uuid::Error), -} + #[error("Cannot convert a slice to Uuid: {0}")] + UuidFromSlice(#[source] uuid::Error), + } -#[derive(thiserror::Error, PartialEq)] -pub enum CompatibilityError { - #[error( - "Incompatible schema types! Writer schema is '{writer_schema_type}', but reader schema is '{reader_schema_type}'" - )] - WrongType { - writer_schema_type: String, - reader_schema_type: String, - }, - - #[error("Incompatible schema types! The {schema_type} should have been {expected_type:?}")] - TypeExpected { - schema_type: String, - expected_type: Vec, - }, - - #[error( - "Incompatible schemata! Field '{0}' in reader schema does not match the type in the writer schema" - )] - FieldTypeMismatch(String, #[source] Box), - - #[error("Incompatible schemata! Field '{0}' in reader schema must have a default value")] - MissingDefaultValue(String), - - #[error("Incompatible schemata! Reader's symbols must contain all writer's symbols")] - MissingSymbols, - - #[error("Incompatible schemata! All elements in union must match for both schemas")] - MissingUnionElements, - - #[error("Incompatible schemata! Name and size don't match for fixed")] - FixedMismatch, - - #[error( - "Incompatible schemata! The name must be the same for both schemas. Writer's name {writer_name} and reader's name {reader_name}" - )] - NameMismatch { - writer_name: String, - reader_name: String, - }, - - #[error( - "Incompatible schemata! Unknown type for '{0}'. Make sure that the type is a valid one" - )] - Inconclusive(String), -} + #[derive(thiserror::Error, PartialEq)] + pub enum CompatibilityError { + #[error( + "Incompatible schema types! Writer schema is '{writer_schema_type}', but reader schema is '{reader_schema_type}'" + )] + WrongType { + writer_schema_type: String, + reader_schema_type: String, + }, + + #[error("Incompatible schema types! The {schema_type} should have been {expected_type:?}")] + TypeExpected { + schema_type: String, + expected_type: Vec, + }, + + #[error( + "Incompatible schemata! Field '{0}' in reader schema does not match the type in the writer schema" + )] + FieldTypeMismatch(String, #[source] Box), + + #[error("Incompatible schemata! Field '{0}' in reader schema must have a default value")] + MissingDefaultValue(String), + + #[error("Incompatible schemata! Reader's symbols must contain all writer's symbols")] + MissingSymbols, + + #[error("Incompatible schemata! All elements in union must match for both schemas")] + MissingUnionElements, + + #[error("Incompatible schemata! Name and size don't match for fixed")] + FixedMismatch, + + #[error( + "Incompatible schemata! The name must be the same for both schemas. Writer's name {writer_name} and reader's name {reader_name}" + )] + NameMismatch { + writer_name: String, + reader_name: String, + }, + + #[error( + "Incompatible schemata! Unknown type for '{0}'. Make sure that the type is a valid one" + )] + Inconclusive(String), + } -impl serde::ser::Error for Details { - fn custom(msg: T) -> Self { - Details::SerializeValue(msg.to_string()) + impl serde::ser::Error for Details { + fn custom(msg: T) -> Self { + Details::SerializeValue(msg.to_string()) + } } -} -impl serde::de::Error for Details { - fn custom(msg: T) -> Self { - Details::DeserializeValue(msg.to_string()) + impl serde::de::Error for Details { + fn custom(msg: T) -> Self { + Details::DeserializeValue(msg.to_string()) + } } -} -impl fmt::Debug for Details { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut msg = self.to_string(); - if let Some(e) = self.source() { - msg.extend([": ", &e.to_string()]); + impl fmt::Debug for Details { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut msg = self.to_string(); + if let Some(e) = self.source() { + msg.extend([": ", &e.to_string()]); + } + write!(f, "{msg}") } - write!(f, "{msg}") } -} -impl fmt::Debug for CompatibilityError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut msg = self.to_string(); - if let Some(e) = self.source() { - msg.extend([": ", &e.to_string()]); + impl fmt::Debug for CompatibilityError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut msg = self.to_string(); + if let Some(e) = self.source() { + msg.extend([": ", &e.to_string()]); + } + write!(f, "{msg}") } - write!(f, "{msg}") } } diff --git a/avro/src/headers.rs b/avro/src/headers.rs index dce134f1..149fcf80 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -16,109 +16,130 @@ // under the License. //! Handling of Avro magic headers -use uuid::Uuid; -use crate::{AvroResult, Schema, rabin::Rabin, schema::SchemaFingerprint}; - -/// This trait represents that an object is able to construct an Avro message header. It is -/// implemented for some known header types already. If you need a header type that is not already -/// included here, then you can create your own struct and implement this trait. -pub trait HeaderBuilder { - fn build_header(&self) -> Vec; -} - -/// HeaderBuilder based on the Rabin schema fingerprint -/// -/// This is the default and will be used automatically by the `new` impls in -/// [crate::reader::GenericSingleObjectReader] and [crate::writer::GenericSingleObjectWriter]. -pub struct RabinFingerprintHeader { - fingerprint: SchemaFingerprint, -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + #[tokio::test] => #[test] + ); + } +)] +mod headers { + use uuid::Uuid; + + use crate::{ + AvroResult, error::tokio::Details, rabin::Rabin, schema::tokio::Schema, + schema::tokio::SchemaFingerprint, + }; + + /// This trait represents that an object is able to construct an Avro message header. It is + /// implemented for some known header types already. If you need a header type that is not already + /// included here, then you can create your own struct and implement this trait. + pub trait HeaderBuilder { + fn build_header(&self) -> Vec; + } -impl RabinFingerprintHeader { - /// Use this helper to build an instance from an existing Avro `Schema`. - pub fn from_schema(schema: &Schema) -> Self { - let fingerprint = schema.fingerprint::(); - RabinFingerprintHeader { fingerprint } + /// HeaderBuilder based on the Rabin schema fingerprint + /// + /// This is the default and will be used automatically by the `new` impls in + /// [crate::reader::GenericSingleObjectReader] and [crate::writer::GenericSingleObjectWriter]. + pub struct RabinFingerprintHeader { + fingerprint: SchemaFingerprint, } -} -impl HeaderBuilder for RabinFingerprintHeader { - fn build_header(&self) -> Vec { - let bytes = &self.fingerprint.bytes; - vec![ - 0xC3, 0x01, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], - bytes[7], - ] + impl RabinFingerprintHeader { + /// Use this helper to build an instance from an existing Avro `Schema`. + pub fn from_schema(schema: &Schema) -> Self { + let fingerprint = schema.fingerprint::(); + RabinFingerprintHeader { fingerprint } + } } -} -/// HeaderBuilder based on -/// [Glue](https://docs.aws.amazon.com/glue/latest/dg/what-is-glue.html) schema UUID -/// -/// See the function docs for usage details -pub struct GlueSchemaUuidHeader { - schema_uuid: Uuid, -} + impl HeaderBuilder for RabinFingerprintHeader { + fn build_header(&self) -> Vec { + let bytes = &self.fingerprint.bytes; + vec![ + 0xC3, 0x01, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], + bytes[7], + ] + } + } -impl GlueSchemaUuidHeader { - /// Create an instance of the struct from a Glue Schema UUID + /// HeaderBuilder based on + /// [Glue](https://docs.aws.amazon.com/glue/latest/dg/what-is-glue.html) schema UUID /// - /// Code for writing messages will most likely want to use this. You will need to determine - /// via other means the correct Glue schema UUID and use it with this method to be able to - /// create Avro-encoded messages with the correct headers. - pub fn from_uuid(schema_uuid: Uuid) -> Self { - GlueSchemaUuidHeader { schema_uuid } + /// See the function docs for usage details + pub struct GlueSchemaUuidHeader { + schema_uuid: Uuid, } - /// The minimum length of a Glue header. - /// 2 bytes for the special prefix (3, 0) plus - /// 16 bytes for the Uuid - const GLUE_HEADER_LENGTH: usize = 18; + impl GlueSchemaUuidHeader { + /// Create an instance of the struct from a Glue Schema UUID + /// + /// Code for writing messages will most likely want to use this. You will need to determine + /// via other means the correct Glue schema UUID and use it with this method to be able to + /// create Avro-encoded messages with the correct headers. + pub fn from_uuid(schema_uuid: Uuid) -> Self { + GlueSchemaUuidHeader { schema_uuid } + } - /// Create an instance of the struct based on parsing the UUID out of the header of a raw - /// message - /// - /// Code for reading messages will most likely want to use this. Once you receive the raw bytes - /// of a message, use this function to build the struct from it. That struct can then be used - /// with the below `schema_uuid` function to retrieve the UUID in order to retrieve the correct - /// schema for the message. You can then use the raw message, the schema, and the struct - /// instance to read the message. - pub fn parse_from_raw_avro(message_payload: &[u8]) -> AvroResult { - if message_payload.len() < Self::GLUE_HEADER_LENGTH { - return Err(crate::error::Details::HeaderMagic.into()); + /// The minimum length of a Glue header. + /// 2 bytes for the special prefix (3, 0) plus + /// 16 bytes for the Uuid + const GLUE_HEADER_LENGTH: usize = 18; + + /// Create an instance of the struct based on parsing the UUID out of the header of a raw + /// message + /// + /// Code for reading messages will most likely want to use this. Once you receive the raw bytes + /// of a message, use this function to build the struct from it. That struct can then be used + /// with the below `schema_uuid` function to retrieve the UUID in order to retrieve the correct + /// schema for the message. You can then use the raw message, the schema, and the struct + /// instance to read the message. + pub fn parse_from_raw_avro(message_payload: &[u8]) -> AvroResult { + if message_payload.len() < Self::GLUE_HEADER_LENGTH { + return Err(Details::HeaderMagic.into()); + } + let schema_uuid = + Uuid::from_slice(&message_payload[2..18]).map_err(Details::UuidFromSlice)?; + Ok(GlueSchemaUuidHeader { schema_uuid }) } - let schema_uuid = Uuid::from_slice(&message_payload[2..18]) - .map_err(crate::error::Details::UuidFromSlice)?; - Ok(GlueSchemaUuidHeader { schema_uuid }) - } - /// Retrieve the UUID from the object - /// - /// This is most useful in conjunction with the `parse_from_raw_avro` function to retrieve the - /// actual UUID from the raw data of a received message. - pub fn schema_uuid(&self) -> Uuid { - self.schema_uuid + /// Retrieve the UUID from the object + /// + /// This is most useful in conjunction with the `parse_from_raw_avro` function to retrieve the + /// actual UUID from the raw data of a received message. + pub fn schema_uuid(&self) -> Uuid { + self.schema_uuid + } } -} -impl HeaderBuilder for GlueSchemaUuidHeader { - fn build_header(&self) -> Vec { - let mut output_vec: Vec = vec![3, 0]; - output_vec.extend_from_slice(self.schema_uuid.as_bytes()); - output_vec + impl HeaderBuilder for GlueSchemaUuidHeader { + fn build_header(&self) -> Vec { + let mut output_vec: Vec = vec![3, 0]; + output_vec.extend_from_slice(self.schema_uuid.as_bytes()); + output_vec + } } -} -#[cfg(test)] -mod test { - use super::*; - use crate::{Error, error::Details}; - use apache_avro_test_helper::TestResult; + #[cfg(test)] + mod test { + use super::*; + use crate::{Error, error::tokio::Details}; + use apache_avro_test_helper::TestResult; - #[test] - fn test_rabin_fingerprint_header() -> TestResult { - let schema_str = r#" + #[test] + fn test_rabin_fingerprint_header() -> TestResult { + let schema_str = r#" { "type": "record", "name": "test", @@ -135,43 +156,44 @@ mod test { ] } "#; - let schema = Schema::parse_str(schema_str)?; - let header_builder = RabinFingerprintHeader::from_schema(&schema); - let computed_header = header_builder.build_header(); - let expected_header: Vec = vec![195, 1, 232, 198, 194, 12, 97, 95, 44, 71]; - assert_eq!(computed_header, expected_header); - Ok(()) - } + let schema = Schema::parse_str(schema_str)?; + let header_builder = RabinFingerprintHeader::from_schema(&schema); + let computed_header = header_builder.build_header(); + let expected_header: Vec = vec![195, 1, 232, 198, 194, 12, 97, 95, 44, 71]; + assert_eq!(computed_header, expected_header); + Ok(()) + } - #[test] - fn test_glue_schema_header() -> TestResult { - let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; - let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); - let computed_header = header_builder.build_header(); - let expected_header: Vec = vec![ - 3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, - ]; - assert_eq!(computed_header, expected_header); - Ok(()) - } + #[test] + fn test_glue_schema_header() -> TestResult { + let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; + let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); + let computed_header = header_builder.build_header(); + let expected_header: Vec = vec![ + 3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, + ]; + assert_eq!(computed_header, expected_header); + Ok(()) + } - #[test] - fn test_glue_header_parse() -> TestResult { - let incoming_avro_message: Vec = vec![ - 3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, 65, 65, 65, - ]; - let header_builder = GlueSchemaUuidHeader::parse_from_raw_avro(&incoming_avro_message)?; - let expected_schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; - assert_eq!(header_builder.schema_uuid(), expected_schema_uuid); - Ok(()) - } + #[test] + fn test_glue_header_parse() -> TestResult { + let incoming_avro_message: Vec = vec![ + 3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, 65, 65, 65, + ]; + let header_builder = GlueSchemaUuidHeader::parse_from_raw_avro(&incoming_avro_message)?; + let expected_schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; + assert_eq!(header_builder.schema_uuid(), expected_schema_uuid); + Ok(()) + } - #[test] - fn test_glue_header_parse_err_on_message_too_short() -> TestResult { - let incoming_message: Vec = vec![3, 0, 178, 241, 207, 0, 4, 52, 1]; - let header_builder_res = GlueSchemaUuidHeader::parse_from_raw_avro(&incoming_message) - .map_err(Error::into_details); - assert!(matches!(header_builder_res, Err(Details::HeaderMagic))); - Ok(()) + #[test] + fn test_glue_header_parse_err_on_message_too_short() -> TestResult { + let incoming_message: Vec = vec![3, 0, 178, 241, 207, 0, 4, 52, 1]; + let header_builder_res = GlueSchemaUuidHeader::parse_from_raw_avro(&incoming_message) + .map_err(Error::into_details); + assert!(matches!(header_builder_res, Err(Details::HeaderMagic))); + Ok(()) + } } } diff --git a/avro/src/lib.rs b/avro/src/lib.rs index a93d661c..7016db89 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -882,13 +882,11 @@ pub mod schema_equality; pub mod types; pub mod validator; -pub use crate::{ - bigdecimal::BigDecimal, - bytes::{ - serde_avro_bytes, serde_avro_bytes_opt, serde_avro_fixed, serde_avro_fixed_opt, - serde_avro_slice, serde_avro_slice_opt, - }, +pub use crate::bytes::{ + serde_avro_bytes, serde_avro_bytes_opt, serde_avro_fixed, serde_avro_fixed_opt, + serde_avro_slice, serde_avro_slice_opt, }; +pub use crate::tokio::bigdecimal::BigDecimal; #[cfg(feature = "bzip")] pub use codec::bzip::Bzip2Settings; #[cfg(feature = "xz")] @@ -901,12 +899,6 @@ pub use decimal::Decimal; pub use duration::{Days, Duration, Millis, Months}; pub use error::Error; -#[cfg(feature = "sync")] -pub use reader::sync::{ - GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, - from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, -}; -#[cfg(feature = "tokio")] pub use reader::tokio::{ GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, @@ -926,18 +918,6 @@ pub use apache_avro_derive::*; /// A convenience type alias for `Result`s with `Error`s. pub type AvroResult = Result; -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tests_tokio { }, - #[cfg(feature = "sync")] - pub mod tests_sync { - sync!(); - replace!( - tokio::io::AsyncRead => std::io::Read, - #[tokio::test] => #[test] - ); - } -)] #[cfg(test)] mod tests { use crate::{ @@ -950,34 +930,34 @@ mod tests { #[tokio::test] async fn test_enum_default() { let writer_raw_schema = r#" - { - "type": "record", - "name": "test", - "fields": [ - {"name": "a", "type": "long", "default": 42}, - {"name": "b", "type": "string"} - ] - } - "#; + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } + "#; let reader_raw_schema = r#" - { - "type": "record", - "name": "test", - "fields": [ - {"name": "a", "type": "long", "default": 42}, - {"name": "b", "type": "string"}, - { - "name": "c", - "type": { - "type": "enum", - "name": "suit", - "symbols": ["diamonds", "spades", "clubs", "hearts"] - }, - "default": "spades" - } - ] - } - "#; + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"}, + { + "name": "c", + "type": { + "type": "enum", + "name": "suit", + "symbols": ["diamonds", "spades", "clubs", "hearts"] + }, + "default": "spades" + } + ] + } + "#; let writer_schema = Schema::parse_str(writer_raw_schema).unwrap(); let reader_schema = Schema::parse_str(reader_raw_schema).unwrap(); let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); @@ -1004,24 +984,24 @@ mod tests { #[test] fn test_enum_string_value() { let raw_schema = r#" - { - "type": "record", - "name": "test", - "fields": [ - {"name": "a", "type": "long", "default": 42}, - {"name": "b", "type": "string"}, - { - "name": "c", - "type": { - "type": "enum", - "name": "suit", - "symbols": ["diamonds", "spades", "clubs", "hearts"] - }, - "default": "spades" - } - ] - } - "#; + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"}, + { + "name": "c", + "type": { + "type": "enum", + "name": "suit", + "symbols": ["diamonds", "spades", "clubs", "hearts"] + }, + "default": "spades" + } + ] + } + "#; let schema = Schema::parse_str(raw_schema).unwrap(); let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); @@ -1046,24 +1026,24 @@ mod tests { #[tokio::test] async fn test_enum_no_reader_schema() { let writer_raw_schema = r#" - { - "type": "record", - "name": "test", - "fields": [ - {"name": "a", "type": "long", "default": 42}, - {"name": "b", "type": "string"}, - { - "name": "c", - "type": { - "type": "enum", - "name": "suit", - "symbols": ["diamonds", "spades", "clubs", "hearts"] - }, - "default": "spades" - } - ] - } - "#; + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"}, + { + "name": "c", + "type": { + "type": "enum", + "name": "suit", + "symbols": ["diamonds", "spades", "clubs", "hearts"] + }, + "default": "spades" + } + ] + } + "#; let writer_schema = Schema::parse_str(writer_raw_schema).unwrap(); let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); @@ -1086,15 +1066,15 @@ mod tests { #[test] fn test_illformed_length() { let raw_schema = r#" - { - "type": "record", - "name": "test", - "fields": [ - {"name": "a", "type": "long", "default": 42}, - {"name": "b", "type": "string"} - ] - } - "#; + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } + "#; let schema = Schema::parse_str(raw_schema).unwrap(); diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 2c8e96b8..1a274535 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -22,43 +22,47 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, - decode::tokio::decode => decode::sync::decode, - decode::tokio::decode_internal => decode::sync::decode_internal, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, util::tokio => util::sync, #[tokio::test] => #[test] ); } )] mod reader { - #[synca::cfg(tokio)] + #[cfg(feature = "sync")] + use std::io::Read as AvroRead; + #[cfg(feature = "tokio")] + use tokio::io::AsyncRead as AvroRead; + #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; + use crate::decode::tokio::{decode, decode_internal}; use crate::{ AvroResult, Codec, Error, - decode::tokio::{decode, decode_internal}, - error::Details, + error::tokio::Details, from_value, - headers::{HeaderBuilder, RabinFingerprintHeader}, - schema::{ + headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, + schema::tokio::{ AvroSchema, Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, resolve_names_with_schemata, }, - types::Value, - util, + types::tokio::Value, + util::tokio::read_long, }; + #[cfg(feature = "tokio")] use futures::Stream; use log::warn; use serde::de::DeserializeOwned; use serde_json::from_slice; + #[cfg(feature = "tokio")] use std::pin::Pin; + #[cfg(feature = "tokio")] use std::task::Poll; - use std::{ - collections::HashMap, - io::ErrorKind, - marker::PhantomData, - str::FromStr, - }; + use std::{collections::HashMap, io::ErrorKind, marker::PhantomData}; /// Internal Block reader. #[derive(Debug, Clone)] @@ -77,7 +81,7 @@ mod reader { names_refs: Names, } - impl<'r, R: tokio::io::AsyncRead + Unpin> Block<'r, R> { + impl<'r, R: AvroRead + Unpin> Block<'r, R> { async fn new(reader: R, schemata: Vec<&'r Schema>) -> AvroResult> { let mut block = Block { reader, @@ -152,7 +156,7 @@ mod reader { // We need to resize to ensure that the buffer len is safe to read `n` elements. // // TODO: Figure out a way to avoid having to truncate for the second case. - self.buf.resize(util::safe_len(n)?, 0); + self.buf.resize(crate::util::safe_len(n)?, 0); self.reader .read_exact(&mut self.buf) .await @@ -165,17 +169,18 @@ mod reader { /// the block. The objects are stored in an internal buffer to the `Reader`. async fn read_block_next(&mut self) -> AvroResult<()> { assert!(self.is_empty(), "Expected self to be empty!"); - match util::tokio::read_long(&mut self.reader) + match read_long(&mut self.reader) .await .map_err(Error::into_details) { Ok(block_len) => { self.message_count = block_len as usize; - let block_bytes = util::tokio::read_long(&mut self.reader).await?; + let block_bytes = read_long(&mut self.reader).await?; self.fill_buf(block_bytes as usize).await?; let mut marker = [0u8; 16]; self.reader - .read_exact(&mut marker).await + .read_exact(&mut marker) + .await .map_err(Details::ReadBlockMarker)?; if marker != self.marker { @@ -364,7 +369,7 @@ mod reader { should_resolve_schema: bool, } - impl<'a, R: tokio::io::AsyncRead + Unpin> Reader<'a, R> { + impl<'a, R: AvroRead + Unpin> Reader<'a, R> { /// Creates a `Reader` given something implementing the `io::Read` trait to read from. /// No reader `Schema` will be set. /// @@ -449,28 +454,30 @@ mod reader { } #[cfg(feature = "tokio")] - impl Stream for Reader<'_, R> { + impl Stream for Reader<'_, R> { type Item = AvroResult; - async fn poll_next( + fn poll_next( mut self: Pin<&mut Self>, _cx: &mut futures::task::Context<'_>, - ) -> Poll>> { + ) -> Poll> { // to prevent keep on reading after the first error occurs if self.errored { return Poll::Ready(None); }; - match self.read_next().await { - Ok(opt) => Poll::Ready(opt.map(Ok)), - Err(e) => { - self.errored = true; - Poll::Ready(Some(Err(e))) + async { + match self.read_next().await { + Ok(opt) => Poll::Ready(opt.map(Ok)), + Err(e) => { + self.errored = true; + Poll::Ready(Some(Err(e))) + } } } } } #[cfg(feature = "sync")] - impl Iterator for Reader<'_, R> { + impl Iterator for Reader<'_, R> { type Item = AvroResult; fn next(&mut self) -> Option { @@ -496,7 +503,7 @@ mod reader { /// **NOTE** This function has a quite small niche of usage and does NOT take care of reading the /// header and consecutive data blocks; use [`Reader`](struct.Reader.html) if you don't know what /// you are doing, instead. - pub async fn from_avro_datum( + pub async fn from_avro_datum( writer_schema: &Schema, reader: &mut R, reader_schema: Option<&Schema>, @@ -514,7 +521,7 @@ mod reader { /// schemata to resolve any dependencies. /// /// In case a reader `Schema` is provided, schema resolution will also be performed. - pub async fn from_avro_datum_schemata( + pub async fn from_avro_datum_schemata( writer_schema: &Schema, writer_schemata: Vec<&Schema>, reader: &mut R, @@ -526,7 +533,8 @@ mod reader { reader, reader_schema, Vec::with_capacity(0), - ).await + ) + .await } /// Decode a `Value` encoded in Avro format given the provided `Schema` and anything implementing `io::Read` @@ -535,7 +543,7 @@ mod reader { /// schemata to resolve any dependencies. /// /// In case a reader `Schema` is provided, schema resolution will also be performed. - pub async fn from_avro_datum_reader_schemata( + pub async fn from_avro_datum_reader_schemata( writer_schema: &Schema, writer_schemata: Vec<&Schema>, reader: &mut R, @@ -578,7 +586,7 @@ mod reader { }) } - pub async fn read_value(&self, reader: &mut R) -> AvroResult { + pub async fn read_value(&self, reader: &mut R) -> AvroResult { let mut header = vec![0; self.expected_header.len()]; match reader.read_exact(&mut header).await { Ok(_) => { @@ -588,7 +596,8 @@ mod reader { self.write_schema.get_names(), &None, reader, - ).await + ) + .await } else { Err(Details::SingleObjectHeaderMismatch( self.expected_header.clone(), @@ -626,7 +635,7 @@ mod reader { where T: AvroSchema + From, { - pub async fn read_from_value(&self, reader: &mut R) -> AvroResult { + pub async fn read_from_value(&self, reader: &mut R) -> AvroResult { self.inner.read_value(reader).await.map(|v| v.into()) } } @@ -635,7 +644,7 @@ mod reader { where T: AvroSchema + DeserializeOwned, { - pub async fn read(&self, reader: &mut R) -> AvroResult { + pub async fn read(&self, reader: &mut R) -> AvroResult { from_value::(&self.inner.read_value(reader).await?) } } diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 33894372..cad51d63 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -16,2511 +16,2569 @@ // under the License. //! Logic for parsing and interacting with schemas in Avro format. -use crate::{ - AvroResult, - error::{Details, Error}, - schema_equality, types, - validator::{ - validate_enum_symbol_name, validate_namespace, validate_record_field_name, - validate_schema_name, - }, -}; -use digest::Digest; -use log::{debug, error, warn}; -use serde::{ - Deserialize, Serialize, Serializer, - ser::{SerializeMap, SerializeSeq}, -}; -use serde_json::{Map, Value}; -use std::{ - borrow::Borrow, - collections::{BTreeMap, HashMap, HashSet}, - fmt, - fmt::Debug, - hash::Hash, - io::Read, - str::FromStr, -}; -use strum_macros::{Display, EnumDiscriminants, EnumString}; -#[cfg(feature = "tokio")] -use crate::util::tokio::MapHelper; -#[cfg(feature = "sync")] -use crate::util::sync::MapHelper; - -/// Represents an Avro schema fingerprint -/// More information about Avro schema fingerprints can be found in the -/// [Avro Schema Fingerprint documentation](https://avro.apache.org/docs/current/specification/#schema-fingerprints) -pub struct SchemaFingerprint { - pub bytes: Vec, -} - -impl fmt::Display for SchemaFingerprint { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "{}", - self.bytes - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::>() - .join("") - ) - } -} - -/// Represents any valid Avro schema -/// More information about Avro schemas can be found in the -/// [Avro Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) -#[derive(Clone, Debug, EnumDiscriminants, Display)] -#[strum_discriminants(name(SchemaKind), derive(Hash, Ord, PartialOrd))] -pub enum Schema { - /// A `null` Avro schema. - Null, - /// A `boolean` Avro schema. - Boolean, - /// An `int` Avro schema. - Int, - /// A `long` Avro schema. - Long, - /// A `float` Avro schema. - Float, - /// A `double` Avro schema. - Double, - /// A `bytes` Avro schema. - /// `Bytes` represents a sequence of 8-bit unsigned bytes. - Bytes, - /// A `string` Avro schema. - /// `String` represents a unicode character sequence. - String, - /// A `array` Avro schema. Avro arrays are required to have the same type for each element. - /// This variant holds the `Schema` for the array element type. - Array(ArraySchema), - /// A `map` Avro schema. - /// `Map` holds a pointer to the `Schema` of its values, which must all be the same schema. - /// `Map` keys are assumed to be `string`. - Map(MapSchema), - /// A `union` Avro schema. - Union(UnionSchema), - /// A `record` Avro schema. - Record(RecordSchema), - /// An `enum` Avro schema. - Enum(EnumSchema), - /// A `fixed` Avro schema. - Fixed(FixedSchema), - /// Logical type which represents `Decimal` values. The underlying type is serialized and - /// deserialized as `Schema::Bytes` or `Schema::Fixed`. - Decimal(DecimalSchema), - /// Logical type which represents `Decimal` values without predefined scale. - /// The underlying type is serialized and deserialized as `Schema::Bytes` - BigDecimal, - /// A universally unique identifier, annotating a string. - Uuid, - /// Logical type which represents the number of days since the unix epoch. - /// Serialization format is `Schema::Int`. - Date, - /// The time of day in number of milliseconds after midnight with no reference any calendar, - /// time zone or date in particular. - TimeMillis, - /// The time of day in number of microseconds after midnight with no reference any calendar, - /// time zone or date in particular. - TimeMicros, - /// An instant in time represented as the number of milliseconds after the UNIX epoch. - TimestampMillis, - /// An instant in time represented as the number of microseconds after the UNIX epoch. - TimestampMicros, - /// An instant in time represented as the number of nanoseconds after the UNIX epoch. - TimestampNanos, - /// An instant in localtime represented as the number of milliseconds after the UNIX epoch. - LocalTimestampMillis, - /// An instant in local time represented as the number of microseconds after the UNIX epoch. - LocalTimestampMicros, - /// An instant in local time represented as the number of nanoseconds after the UNIX epoch. - LocalTimestampNanos, - /// An amount of time defined by a number of months, days and milliseconds. - Duration, - /// A reference to another schema. - Ref { name: Name }, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct MapSchema { - pub types: Box, - pub attributes: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ArraySchema { - pub items: Box, - pub attributes: BTreeMap, -} -impl PartialEq for Schema { - /// Assess equality of two `Schema` based on [Parsing Canonical Form]. - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas - fn eq(&self, other: &Self) -> bool { - schema_equality::compare_schemata(self, other) - } -} - -impl SchemaKind { - pub fn is_primitive(self) -> bool { - matches!( - self, - SchemaKind::Null - | SchemaKind::Boolean - | SchemaKind::Int - | SchemaKind::Long - | SchemaKind::Double - | SchemaKind::Float - | SchemaKind::Bytes - | SchemaKind::String, - ) - } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead + Unpin => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + schema_equality::tokio => schema_equality::sync, + util::tokio => util::sync, + types::tokio => types::sync, + validator::tokio => validator::sync, + #[tokio::test] => #[test] + ); + } +)] +mod schema { + + use crate::{ + AvroResult, + error::tokio::{Details, Error}, + schema_equality::tokio as schema_equality, + types::tokio as types, + util::tokio::MapHelper, + validator::tokio::{ + validate_enum_symbol_name, validate_namespace, validate_record_field_name, + validate_schema_name, + }, + }; + use digest::Digest; + use log::{debug, error, warn}; + use serde::{ + Deserialize, Serialize, Serializer, + ser::{SerializeMap, SerializeSeq}, + }; + use serde_json::{Map, Value}; + use std::{ + borrow::Borrow, + collections::{BTreeMap, HashMap, HashSet}, + fmt, + fmt::Debug, + hash::Hash, + io::Read, + str::FromStr, + }; + use strum_macros::{Display, EnumDiscriminants, EnumString}; - pub fn is_named(self) -> bool { - matches!( - self, - SchemaKind::Record | SchemaKind::Enum | SchemaKind::Fixed | SchemaKind::Ref - ) + /// Represents an Avro schema fingerprint + /// More information about Avro schema fingerprints can be found in the + /// [Avro Schema Fingerprint documentation](https://avro.apache.org/docs/current/specification/#schema-fingerprints) + pub struct SchemaFingerprint { + pub bytes: Vec, } -} -impl From<&types::Value> for SchemaKind { - fn from(value: &types::Value) -> Self { - use crate::types::Value; - match value { - Value::Null => Self::Null, - Value::Boolean(_) => Self::Boolean, - Value::Int(_) => Self::Int, - Value::Long(_) => Self::Long, - Value::Float(_) => Self::Float, - Value::Double(_) => Self::Double, - Value::Bytes(_) => Self::Bytes, - Value::String(_) => Self::String, - Value::Array(_) => Self::Array, - Value::Map(_) => Self::Map, - Value::Union(_, _) => Self::Union, - Value::Record(_) => Self::Record, - Value::Enum(_, _) => Self::Enum, - Value::Fixed(_, _) => Self::Fixed, - Value::Decimal { .. } => Self::Decimal, - Value::BigDecimal(_) => Self::BigDecimal, - Value::Uuid(_) => Self::Uuid, - Value::Date(_) => Self::Date, - Value::TimeMillis(_) => Self::TimeMillis, - Value::TimeMicros(_) => Self::TimeMicros, - Value::TimestampMillis(_) => Self::TimestampMillis, - Value::TimestampMicros(_) => Self::TimestampMicros, - Value::TimestampNanos(_) => Self::TimestampNanos, - Value::LocalTimestampMillis(_) => Self::LocalTimestampMillis, - Value::LocalTimestampMicros(_) => Self::LocalTimestampMicros, - Value::LocalTimestampNanos(_) => Self::LocalTimestampNanos, - Value::Duration { .. } => Self::Duration, + impl fmt::Display for SchemaFingerprint { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + self.bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join("") + ) + } + } + + /// Represents any valid Avro schema + /// More information about Avro schemas can be found in the + /// [Avro Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) + #[derive(Clone, Debug, EnumDiscriminants, Display)] + #[strum_discriminants(name(SchemaKind), derive(Hash, Ord, PartialOrd))] + pub enum Schema { + /// A `null` Avro schema. + Null, + /// A `boolean` Avro schema. + Boolean, + /// An `int` Avro schema. + Int, + /// A `long` Avro schema. + Long, + /// A `float` Avro schema. + Float, + /// A `double` Avro schema. + Double, + /// A `bytes` Avro schema. + /// `Bytes` represents a sequence of 8-bit unsigned bytes. + Bytes, + /// A `string` Avro schema. + /// `String` represents a unicode character sequence. + String, + /// A `array` Avro schema. Avro arrays are required to have the same type for each element. + /// This variant holds the `Schema` for the array element type. + Array(ArraySchema), + /// A `map` Avro schema. + /// `Map` holds a pointer to the `Schema` of its values, which must all be the same schema. + /// `Map` keys are assumed to be `string`. + Map(MapSchema), + /// A `union` Avro schema. + Union(UnionSchema), + /// A `record` Avro schema. + Record(RecordSchema), + /// An `enum` Avro schema. + Enum(EnumSchema), + /// A `fixed` Avro schema. + Fixed(FixedSchema), + /// Logical type which represents `Decimal` values. The underlying type is serialized and + /// deserialized as `Schema::Bytes` or `Schema::Fixed`. + Decimal(DecimalSchema), + /// Logical type which represents `Decimal` values without predefined scale. + /// The underlying type is serialized and deserialized as `Schema::Bytes` + BigDecimal, + /// A universally unique identifier, annotating a string. + Uuid, + /// Logical type which represents the number of days since the unix epoch. + /// Serialization format is `Schema::Int`. + Date, + /// The time of day in number of milliseconds after midnight with no reference any calendar, + /// time zone or date in particular. + TimeMillis, + /// The time of day in number of microseconds after midnight with no reference any calendar, + /// time zone or date in particular. + TimeMicros, + /// An instant in time represented as the number of milliseconds after the UNIX epoch. + TimestampMillis, + /// An instant in time represented as the number of microseconds after the UNIX epoch. + TimestampMicros, + /// An instant in time represented as the number of nanoseconds after the UNIX epoch. + TimestampNanos, + /// An instant in localtime represented as the number of milliseconds after the UNIX epoch. + LocalTimestampMillis, + /// An instant in local time represented as the number of microseconds after the UNIX epoch. + LocalTimestampMicros, + /// An instant in local time represented as the number of nanoseconds after the UNIX epoch. + LocalTimestampNanos, + /// An amount of time defined by a number of months, days and milliseconds. + Duration, + /// A reference to another schema. + Ref { name: Name }, + } + + #[derive(Clone, Debug, PartialEq)] + pub struct MapSchema { + pub types: Box, + pub attributes: BTreeMap, + } + + #[derive(Clone, Debug, PartialEq)] + pub struct ArraySchema { + pub items: Box, + pub attributes: BTreeMap, + } + + impl PartialEq for Schema { + /// Assess equality of two `Schema` based on [Parsing Canonical Form]. + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas + fn eq(&self, other: &Self) -> bool { + schema_equality::compare_schemata(self, other) + } + } + + impl SchemaKind { + pub fn is_primitive(self) -> bool { + matches!( + self, + SchemaKind::Null + | SchemaKind::Boolean + | SchemaKind::Int + | SchemaKind::Long + | SchemaKind::Double + | SchemaKind::Float + | SchemaKind::Bytes + | SchemaKind::String, + ) + } + + pub fn is_named(self) -> bool { + matches!( + self, + SchemaKind::Record | SchemaKind::Enum | SchemaKind::Fixed | SchemaKind::Ref + ) + } + } + + impl From<&types::Value> for SchemaKind { + fn from(value: &types::Value) -> Self { + use crate::types::tokio::Value; + match value { + Value::Null => Self::Null, + Value::Boolean(_) => Self::Boolean, + Value::Int(_) => Self::Int, + Value::Long(_) => Self::Long, + Value::Float(_) => Self::Float, + Value::Double(_) => Self::Double, + Value::Bytes(_) => Self::Bytes, + Value::String(_) => Self::String, + Value::Array(_) => Self::Array, + Value::Map(_) => Self::Map, + Value::Union(_, _) => Self::Union, + Value::Record(_) => Self::Record, + Value::Enum(_, _) => Self::Enum, + Value::Fixed(_, _) => Self::Fixed, + Value::Decimal { .. } => Self::Decimal, + Value::BigDecimal(_) => Self::BigDecimal, + Value::Uuid(_) => Self::Uuid, + Value::Date(_) => Self::Date, + Value::TimeMillis(_) => Self::TimeMillis, + Value::TimeMicros(_) => Self::TimeMicros, + Value::TimestampMillis(_) => Self::TimestampMillis, + Value::TimestampMicros(_) => Self::TimestampMicros, + Value::TimestampNanos(_) => Self::TimestampNanos, + Value::LocalTimestampMillis(_) => Self::LocalTimestampMillis, + Value::LocalTimestampMicros(_) => Self::LocalTimestampMicros, + Value::LocalTimestampNanos(_) => Self::LocalTimestampNanos, + Value::Duration { .. } => Self::Duration, + } + } + } + + /// Represents names for `record`, `enum` and `fixed` Avro schemas. + /// + /// Each of these `Schema`s have a `fullname` composed of two parts: + /// * a name + /// * a namespace + /// + /// `aliases` can also be defined, to facilitate schema evolution. + /// + /// More information about schema names can be found in the + /// [Avro specification](https://avro.apache.org/docs/current/specification/#names) + #[derive(Clone, Debug, Hash, PartialEq, Eq)] + pub struct Name { + pub name: String, + pub namespace: Namespace, + } + + /// Represents documentation for complex Avro schemas. + pub type Documentation = Option; + /// Represents the aliases for Named Schema + pub type Aliases = Option>; + /// Represents Schema lookup within a schema env + pub(crate) type Names = HashMap; + /// Represents Schema lookup within a schema + pub type NamesRef<'a> = HashMap; + /// Represents the namespace for Named Schema + pub type Namespace = Option; + + impl Name { + /// Create a new `Name`. + /// Parses the optional `namespace` from the `name` string. + /// `aliases` will not be defined. + pub fn new(name: &str) -> AvroResult { + let (name, namespace) = Name::get_name_and_namespace(name)?; + Ok(Self { + name, + namespace: namespace.filter(|ns| !ns.is_empty()), + }) } - } -} -/// Represents names for `record`, `enum` and `fixed` Avro schemas. -/// -/// Each of these `Schema`s have a `fullname` composed of two parts: -/// * a name -/// * a namespace -/// -/// `aliases` can also be defined, to facilitate schema evolution. -/// -/// More information about schema names can be found in the -/// [Avro specification](https://avro.apache.org/docs/current/specification/#names) -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct Name { - pub name: String, - pub namespace: Namespace, -} + fn get_name_and_namespace(name: &str) -> AvroResult<(String, Namespace)> { + validate_schema_name(name) + } -/// Represents documentation for complex Avro schemas. -pub type Documentation = Option; -/// Represents the aliases for Named Schema -pub type Aliases = Option>; -/// Represents Schema lookup within a schema env -pub(crate) type Names = HashMap; -/// Represents Schema lookup within a schema -pub type NamesRef<'a> = HashMap; -/// Represents the namespace for Named Schema -pub type Namespace = Option; - -impl Name { - /// Create a new `Name`. - /// Parses the optional `namespace` from the `name` string. - /// `aliases` will not be defined. - pub fn new(name: &str) -> AvroResult { - let (name, namespace) = Name::get_name_and_namespace(name)?; - Ok(Self { - name, - namespace: namespace.filter(|ns| !ns.is_empty()), - }) - } + /// Parse a `serde_json::Value` into a `Name`. + pub(crate) fn parse( + complex: &Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + let (name, namespace_from_name) = complex + .name() + .map(|name| Name::get_name_and_namespace(name.as_str()).unwrap()) + .ok_or(Details::GetNameField)?; + // FIXME Reading name from the type is wrong ! The name there is just a metadata (AVRO-3430) + let type_name = match complex.get("type") { + Some(Value::Object(complex_type)) => complex_type.name().or(None), + _ => None, + }; - fn get_name_and_namespace(name: &str) -> AvroResult<(String, Namespace)> { - validate_schema_name(name) - } + let namespace = namespace_from_name + .or_else(|| { + complex + .string("namespace") + .or_else(|| enclosing_namespace.clone()) + }) + .filter(|ns| !ns.is_empty()); - /// Parse a `serde_json::Value` into a `Name`. - pub(crate) fn parse( - complex: &Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - let (name, namespace_from_name) = complex - .name() - .map(|name| Name::get_name_and_namespace(name.as_str()).unwrap()) - .ok_or(Details::GetNameField)?; - // FIXME Reading name from the type is wrong ! The name there is just a metadata (AVRO-3430) - let type_name = match complex.get("type") { - Some(Value::Object(complex_type)) => complex_type.name().or(None), - _ => None, - }; + if let Some(ref ns) = namespace { + validate_namespace(ns)?; + } - let namespace = namespace_from_name - .or_else(|| { - complex - .string("namespace") - .or_else(|| enclosing_namespace.clone()) + Ok(Self { + name: type_name.unwrap_or(name), + namespace, }) - .filter(|ns| !ns.is_empty()); - - if let Some(ref ns) = namespace { - validate_namespace(ns)?; } - Ok(Self { - name: type_name.unwrap_or(name), - namespace, - }) - } - - /// Return the `fullname` of this `Name` - /// - /// More information about fullnames can be found in the - /// [Avro specification](https://avro.apache.org/docs/current/specification/#names) - pub fn fullname(&self, default_namespace: Namespace) -> String { - if self.name.contains('.') { - self.name.clone() - } else { - let namespace = self.namespace.clone().or(default_namespace); + /// Return the `fullname` of this `Name` + /// + /// More information about fullnames can be found in the + /// [Avro specification](https://avro.apache.org/docs/current/specification/#names) + pub fn fullname(&self, default_namespace: Namespace) -> String { + if self.name.contains('.') { + self.name.clone() + } else { + let namespace = self.namespace.clone().or(default_namespace); - match namespace { - Some(ref namespace) if !namespace.is_empty() => { - format!("{}.{}", namespace, self.name) + match namespace { + Some(ref namespace) if !namespace.is_empty() => { + format!("{}.{}", namespace, self.name) + } + _ => self.name.clone(), } - _ => self.name.clone(), } } - } - /// Return the fully qualified name needed for indexing or searching for the schema within a schema/schema env context. Puts the enclosing namespace into the name's namespace for clarity in schema/schema env parsing - /// ```ignore - /// use apache_avro::schema::Name; - /// - /// assert_eq!( - /// Name::new("some_name")?.fully_qualified_name(&Some("some_namespace".into())), - /// Name::new("some_namespace.some_name")? - /// ); - /// assert_eq!( - /// Name::new("some_namespace.some_name")?.fully_qualified_name(&Some("other_namespace".into())), - /// Name::new("some_namespace.some_name")? - /// ); - /// ``` - pub fn fully_qualified_name(&self, enclosing_namespace: &Namespace) -> Name { - Name { - name: self.name.clone(), - namespace: self - .namespace - .clone() - .or_else(|| enclosing_namespace.clone().filter(|ns| !ns.is_empty())), + /// Return the fully qualified name needed for indexing or searching for the schema within a schema/schema env context. Puts the enclosing namespace into the name's namespace for clarity in schema/schema env parsing + /// ```ignore + /// use apache_avro::schema::Name; + /// + /// assert_eq!( + /// Name::new("some_name")?.fully_qualified_name(&Some("some_namespace".into())), + /// Name::new("some_namespace.some_name")? + /// ); + /// assert_eq!( + /// Name::new("some_namespace.some_name")?.fully_qualified_name(&Some("other_namespace".into())), + /// Name::new("some_namespace.some_name")? + /// ); + /// ``` + pub fn fully_qualified_name(&self, enclosing_namespace: &Namespace) -> Name { + Name { + name: self.name.clone(), + namespace: self + .namespace + .clone() + .or_else(|| enclosing_namespace.clone().filter(|ns| !ns.is_empty())), + } } } -} -impl From<&str> for Name { - fn from(name: &str) -> Self { - Name::new(name).unwrap() + impl From<&str> for Name { + fn from(name: &str) -> Self { + Name::new(name).unwrap() + } } -} -impl fmt::Display for Name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.fullname(None)[..]) + impl fmt::Display for Name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.fullname(None)[..]) + } } -} -impl<'de> Deserialize<'de> for Name { - fn deserialize(deserializer: D) -> Result - where - D: serde::de::Deserializer<'de>, - { - Value::deserialize(deserializer).and_then(|value| { - use serde::de::Error; - if let Value::Object(json) = value { - Name::parse(&json, &None).map_err(Error::custom) - } else { - Err(Error::custom(format!("Expected a JSON object: {value:?}"))) - } - }) + impl<'de> Deserialize<'de> for Name { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + Value::deserialize(deserializer).and_then(|value| { + use serde::de::Error; + if let Value::Object(json) = value { + Name::parse(&json, &None).map_err(Error::custom) + } else { + Err(Error::custom(format!("Expected a JSON object: {value:?}"))) + } + }) + } } -} -/// Newtype pattern for `Name` to better control the `serde_json::Value` representation. -/// Aliases are serialized as an array of plain strings in the JSON representation. -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct Alias(Name); + /// Newtype pattern for `Name` to better control the `serde_json::Value` representation. + /// Aliases are serialized as an array of plain strings in the JSON representation. + #[derive(Clone, Debug, Hash, PartialEq, Eq)] + pub struct Alias(Name); -impl Alias { - pub fn new(name: &str) -> AvroResult { - Name::new(name).map(Self) - } + impl Alias { + pub fn new(name: &str) -> AvroResult { + Name::new(name).map(Self) + } - pub fn name(&self) -> String { - self.0.name.clone() - } + pub fn name(&self) -> String { + self.0.name.clone() + } - pub fn namespace(&self) -> Namespace { - self.0.namespace.clone() - } + pub fn namespace(&self) -> Namespace { + self.0.namespace.clone() + } - pub fn fullname(&self, default_namespace: Namespace) -> String { - self.0.fullname(default_namespace) - } + pub fn fullname(&self, default_namespace: Namespace) -> String { + self.0.fullname(default_namespace) + } - pub fn fully_qualified_name(&self, default_namespace: &Namespace) -> Name { - self.0.fully_qualified_name(default_namespace) + pub fn fully_qualified_name(&self, default_namespace: &Namespace) -> Name { + self.0.fully_qualified_name(default_namespace) + } } -} -impl From<&str> for Alias { - fn from(name: &str) -> Self { - Alias::new(name).unwrap() + impl From<&str> for Alias { + fn from(name: &str) -> Self { + Alias::new(name).unwrap() + } } -} -impl Serialize for Alias { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.fullname(None)) + impl Serialize for Alias { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.fullname(None)) + } } -} -#[derive(Debug)] -pub struct ResolvedSchema<'s> { - names_ref: NamesRef<'s>, - schemata: Vec<&'s Schema>, -} + #[derive(Debug)] + pub struct ResolvedSchema<'s> { + names_ref: NamesRef<'s>, + schemata: Vec<&'s Schema>, + } -impl<'s> TryFrom<&'s Schema> for ResolvedSchema<'s> { - type Error = Error; + impl<'s> TryFrom<&'s Schema> for ResolvedSchema<'s> { + type Error = Error; - fn try_from(schema: &'s Schema) -> AvroResult { - let names = HashMap::new(); - let mut rs = ResolvedSchema { - names_ref: names, - schemata: vec![schema], - }; - rs.resolve(rs.get_schemata(), &None, None)?; - Ok(rs) + fn try_from(schema: &'s Schema) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedSchema { + names_ref: names, + schemata: vec![schema], + }; + rs.resolve(rs.get_schemata(), &None, None)?; + Ok(rs) + } } -} -impl<'s> TryFrom> for ResolvedSchema<'s> { - type Error = Error; + impl<'s> TryFrom> for ResolvedSchema<'s> { + type Error = Error; - fn try_from(schemata: Vec<&'s Schema>) -> AvroResult { - let names = HashMap::new(); - let mut rs = ResolvedSchema { - names_ref: names, - schemata, - }; - rs.resolve(rs.get_schemata(), &None, None)?; - Ok(rs) + fn try_from(schemata: Vec<&'s Schema>) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedSchema { + names_ref: names, + schemata, + }; + rs.resolve(rs.get_schemata(), &None, None)?; + Ok(rs) + } } -} -impl<'s> ResolvedSchema<'s> { - pub fn get_schemata(&self) -> Vec<&'s Schema> { - self.schemata.clone() - } + impl<'s> ResolvedSchema<'s> { + pub fn get_schemata(&self) -> Vec<&'s Schema> { + self.schemata.clone() + } - pub fn get_names(&self) -> &NamesRef<'s> { - &self.names_ref - } + pub fn get_names(&self) -> &NamesRef<'s> { + &self.names_ref + } - /// Creates `ResolvedSchema` with some already known schemas. - /// - /// Those schemata would be used to resolve references if needed. - pub fn new_with_known_schemata<'n>( - schemata_to_resolve: Vec<&'s Schema>, - enclosing_namespace: &Namespace, - known_schemata: &'n NamesRef<'n>, - ) -> AvroResult { - let names = HashMap::new(); - let mut rs = ResolvedSchema { - names_ref: names, - schemata: schemata_to_resolve, - }; - rs.resolve(rs.get_schemata(), enclosing_namespace, Some(known_schemata))?; - Ok(rs) - } + /// Creates `ResolvedSchema` with some already known schemas. + /// + /// Those schemata would be used to resolve references if needed. + pub fn new_with_known_schemata<'n>( + schemata_to_resolve: Vec<&'s Schema>, + enclosing_namespace: &Namespace, + known_schemata: &'n NamesRef<'n>, + ) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedSchema { + names_ref: names, + schemata: schemata_to_resolve, + }; + rs.resolve(rs.get_schemata(), enclosing_namespace, Some(known_schemata))?; + Ok(rs) + } - fn resolve<'n>( - &mut self, - schemata: Vec<&'s Schema>, - enclosing_namespace: &Namespace, - known_schemata: Option<&'n NamesRef<'n>>, - ) -> AvroResult<()> { - for schema in schemata { - match schema { - Schema::Array(schema) => { - self.resolve(vec![&schema.items], enclosing_namespace, known_schemata)? - } - Schema::Map(schema) => { - self.resolve(vec![&schema.types], enclosing_namespace, known_schemata)? - } - Schema::Union(UnionSchema { schemas, .. }) => { - for schema in schemas { - self.resolve(vec![schema], enclosing_namespace, known_schemata)? + fn resolve<'n>( + &mut self, + schemata: Vec<&'s Schema>, + enclosing_namespace: &Namespace, + known_schemata: Option<&'n NamesRef<'n>>, + ) -> AvroResult<()> { + for schema in schemata { + match schema { + Schema::Array(schema) => { + self.resolve(vec![&schema.items], enclosing_namespace, known_schemata)? } - } - Schema::Enum(EnumSchema { name, .. }) | Schema::Fixed(FixedSchema { name, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if self - .names_ref - .insert(fully_qualified_name.clone(), schema) - .is_some() - { - return Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()); + Schema::Map(schema) => { + self.resolve(vec![&schema.types], enclosing_namespace, known_schemata)? } - } - Schema::Record(RecordSchema { name, fields, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if self - .names_ref - .insert(fully_qualified_name.clone(), schema) - .is_some() - { - return Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()); - } else { - let record_namespace = fully_qualified_name.namespace; - for field in fields { - self.resolve(vec![&field.schema], &record_namespace, known_schemata)? + Schema::Union(UnionSchema { schemas, .. }) => { + for schema in schemas { + self.resolve(vec![schema], enclosing_namespace, known_schemata)? } } - } - Schema::Ref { name } => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - // first search for reference in current schemata, then look into external references. - if !self.names_ref.contains_key(&fully_qualified_name) { - let is_resolved_with_known_schemas = known_schemata - .as_ref() - .map(|names| names.contains_key(&fully_qualified_name)) - .unwrap_or(false); - if !is_resolved_with_known_schemas { - return Err(Details::SchemaResolutionError(fully_qualified_name).into()); + Schema::Enum(EnumSchema { name, .. }) + | Schema::Fixed(FixedSchema { name, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if self + .names_ref + .insert(fully_qualified_name.clone(), schema) + .is_some() + { + return Err( + Details::AmbiguousSchemaDefinition(fully_qualified_name).into() + ); + } + } + Schema::Record(RecordSchema { name, fields, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if self + .names_ref + .insert(fully_qualified_name.clone(), schema) + .is_some() + { + return Err( + Details::AmbiguousSchemaDefinition(fully_qualified_name).into() + ); + } else { + let record_namespace = fully_qualified_name.namespace; + for field in fields { + self.resolve( + vec![&field.schema], + &record_namespace, + known_schemata, + )? + } } } + Schema::Ref { name } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + // first search for reference in current schemata, then look into external references. + if !self.names_ref.contains_key(&fully_qualified_name) { + let is_resolved_with_known_schemas = known_schemata + .as_ref() + .map(|names| names.contains_key(&fully_qualified_name)) + .unwrap_or(false); + if !is_resolved_with_known_schemas { + return Err( + Details::SchemaResolutionError(fully_qualified_name).into() + ); + } + } + } + _ => (), } - _ => (), } + Ok(()) } - Ok(()) } -} - -pub(crate) struct ResolvedOwnedSchema { - names: Names, - root_schema: Schema, -} - -impl TryFrom for ResolvedOwnedSchema { - type Error = Error; - fn try_from(schema: Schema) -> AvroResult { - let names = HashMap::new(); - let mut rs = ResolvedOwnedSchema { - names, - root_schema: schema, - }; - resolve_names(&rs.root_schema, &mut rs.names, &None)?; - Ok(rs) + pub(crate) struct ResolvedOwnedSchema { + names: Names, + root_schema: Schema, } -} -impl ResolvedOwnedSchema { - pub(crate) fn get_root_schema(&self) -> &Schema { - &self.root_schema - } - pub(crate) fn get_names(&self) -> &Names { - &self.names - } -} + impl TryFrom for ResolvedOwnedSchema { + type Error = Error; -pub(crate) fn resolve_names( - schema: &Schema, - names: &mut Names, - enclosing_namespace: &Namespace, -) -> AvroResult<()> { - match schema { - Schema::Array(schema) => resolve_names(&schema.items, names, enclosing_namespace), - Schema::Map(schema) => resolve_names(&schema.types, names, enclosing_namespace), - Schema::Union(UnionSchema { schemas, .. }) => { - for schema in schemas { - resolve_names(schema, names, enclosing_namespace)? - } - Ok(()) - } - Schema::Enum(EnumSchema { name, .. }) | Schema::Fixed(FixedSchema { name, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if names - .insert(fully_qualified_name.clone(), schema.clone()) - .is_some() - { - Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) - } else { - Ok(()) - } - } - Schema::Record(RecordSchema { name, fields, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if names - .insert(fully_qualified_name.clone(), schema.clone()) - .is_some() - { - Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) - } else { - let record_namespace = fully_qualified_name.namespace; - for field in fields { - resolve_names(&field.schema, names, &record_namespace)? - } - Ok(()) - } - } - Schema::Ref { name } => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - names - .get(&fully_qualified_name) - .map(|_| ()) - .ok_or_else(|| Details::SchemaResolutionError(fully_qualified_name).into()) + fn try_from(schema: Schema) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedOwnedSchema { + names, + root_schema: schema, + }; + resolve_names(&rs.root_schema, &mut rs.names, &None)?; + Ok(rs) } - _ => Ok(()), } -} - -pub(crate) fn resolve_names_with_schemata( - schemata: &Vec<&Schema>, - names: &mut Names, - enclosing_namespace: &Namespace, -) -> AvroResult<()> { - for schema in schemata { - resolve_names(schema, names, enclosing_namespace)?; - } - Ok(()) -} -/// Represents a `field` in a `record` Avro schema. -#[derive(bon::Builder, Clone, Debug, PartialEq)] -pub struct RecordField { - /// Name of the field. - pub name: String, - /// Documentation of the field. - #[builder(default)] - pub doc: Documentation, - /// Aliases of the field's name. They have no namespace. - pub aliases: Option>, - /// Default value of the field. - /// This value will be used when reading Avro datum if schema resolution - /// is enabled. - pub default: Option, - /// Schema of the field. - pub schema: Schema, - /// Order of the field. - /// - /// **NOTE** This currently has no effect. - #[builder(default = RecordFieldOrder::Ignore)] - pub order: RecordFieldOrder, - /// Position of the field in the list of `field` of its parent `Schema` - #[builder(default)] - pub position: usize, - /// A collection of all unknown fields in the record field. - #[builder(default = BTreeMap::new())] - pub custom_attributes: BTreeMap, -} - -/// Represents any valid order for a `field` in a `record` Avro schema. -#[derive(Clone, Debug, Eq, PartialEq, EnumString)] -#[strum(serialize_all = "kebab_case")] -pub enum RecordFieldOrder { - Ascending, - Descending, - Ignore, -} - -impl RecordField { - /// Parse a `serde_json::Value` into a `RecordField`. - async fn parse( - field: &Map, - position: usize, - parser: &mut Parser, - enclosing_record: &Name, - ) -> AvroResult { - let name = field.name().ok_or(Details::GetNameFieldFromRecord)?; - - validate_record_field_name(&name)?; - - // TODO: "type" = "" - let schema = parser.parse_complex( - field, - &enclosing_record.namespace, - RecordSchemaParseLocation::FromField, - )?; - - let default = field.get("default").cloned(); - Self::resolve_default_value( - &schema, - &name, - &enclosing_record.fullname(None), - &parser.parsed_schemas, - &default, - ).await?; - - let aliases = field.get("aliases").and_then(|aliases| { - aliases.as_array().map(|aliases| { - aliases - .iter() - .flat_map(|alias| alias.as_str()) - .map(|alias| alias.to_string()) - .collect::>() - }) - }); - - let order = field - .get("order") - .and_then(|order| order.as_str()) - .and_then(|order| RecordFieldOrder::from_str(order).ok()) - .unwrap_or(RecordFieldOrder::Ascending); - - Ok(RecordField { - name, - doc: field.doc(), - default, - aliases, - order, - position, - custom_attributes: RecordField::get_field_custom_attributes(field, &schema), - schema, - }) + impl ResolvedOwnedSchema { + pub(crate) fn get_root_schema(&self) -> &Schema { + &self.root_schema + } + pub(crate) fn get_names(&self) -> &Names { + &self.names + } } - async fn resolve_default_value( - field_schema: &Schema, - field_name: &str, - record_name: &str, - names: &Names, - default: &Option, + pub(crate) fn resolve_names( + schema: &Schema, + names: &mut Names, + enclosing_namespace: &Namespace, ) -> AvroResult<()> { - if let Some(value) = default { - let avro_value = types::Value::from(value.clone()); - match field_schema { - Schema::Union(union_schema) => { - let schemas = &union_schema.schemas; - let resolved = schemas.iter().any(async |schema| { - avro_value - .to_owned() - .resolve_internal(schema, names, &schema.namespace(), &None).await - .is_ok() - }); - - if !resolved { - let schema: Option<&Schema> = schemas.first(); - return match schema { - Some(first_schema) => Err(Details::GetDefaultUnion( - SchemaKind::from(first_schema), - types::ValueKind::from(avro_value), - ) - .into()), - None => Err(Details::EmptyUnion.into()), - }; - } + match schema { + Schema::Array(schema) => resolve_names(&schema.items, names, enclosing_namespace), + Schema::Map(schema) => resolve_names(&schema.types, names, enclosing_namespace), + Schema::Union(UnionSchema { schemas, .. }) => { + for schema in schemas { + resolve_names(schema, names, enclosing_namespace)? } - _ => { - let resolved = avro_value - .resolve_internal(field_schema, names, &field_schema.namespace(), &None).await - .is_ok(); - - if !resolved { - return Err(Details::GetDefaultRecordField( - field_name.to_string(), - record_name.to_string(), - field_schema.canonical_form(), - ) - .into()); + Ok(()) + } + Schema::Enum(EnumSchema { name, .. }) | Schema::Fixed(FixedSchema { name, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if names + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) + } else { + Ok(()) + } + } + Schema::Record(RecordSchema { name, fields, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if names + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) + } else { + let record_namespace = fully_qualified_name.namespace; + for field in fields { + resolve_names(&field.schema, names, &record_namespace)? } + Ok(()) } - }; - } - - Ok(()) - } - - fn get_field_custom_attributes( - field: &Map, - schema: &Schema, - ) -> BTreeMap { - let mut custom_attributes: BTreeMap = BTreeMap::new(); - for (key, value) in field { - match key.as_str() { - "type" | "name" | "doc" | "default" | "order" | "position" | "aliases" - | "logicalType" => continue, - key if key == "symbols" && matches!(schema, Schema::Enum(_)) => continue, - key if key == "size" && matches!(schema, Schema::Fixed(_)) => continue, - _ => custom_attributes.insert(key.clone(), value.clone()), - }; + } + Schema::Ref { name } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + names + .get(&fully_qualified_name) + .map(|_| ()) + .ok_or_else(|| Details::SchemaResolutionError(fully_qualified_name).into()) + } + _ => Ok(()), } - custom_attributes } - /// Returns true if this `RecordField` is nullable, meaning the schema is a `UnionSchema` where the first variant is `Null`. - pub fn is_nullable(&self) -> bool { - match self.schema { - Schema::Union(ref inner) => inner.is_nullable(), - _ => false, + pub(crate) fn resolve_names_with_schemata( + schemata: &Vec<&Schema>, + names: &mut Names, + enclosing_namespace: &Namespace, + ) -> AvroResult<()> { + for schema in schemata { + resolve_names(schema, names, enclosing_namespace)?; } + Ok(()) } -} -/// A description of an Enum schema. -#[derive(bon::Builder, Debug, Clone)] -pub struct RecordSchema { - /// The name of the schema - pub name: Name, - /// The aliases of the schema - #[builder(default)] - pub aliases: Aliases, - /// The documentation of the schema - #[builder(default)] - pub doc: Documentation, - /// The set of fields of the schema - pub fields: Vec, - /// The `lookup` table maps field names to their position in the `Vec` - /// of `fields`. - pub lookup: BTreeMap, - /// The custom attributes of the schema - #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, -} + /// Represents a `field` in a `record` Avro schema. + #[derive(bon::Builder, Clone, Debug, PartialEq)] + pub struct RecordField { + /// Name of the field. + pub name: String, + /// Documentation of the field. + #[builder(default)] + pub doc: Documentation, + /// Aliases of the field's name. They have no namespace. + pub aliases: Option>, + /// Default value of the field. + /// This value will be used when reading Avro datum if schema resolution + /// is enabled. + pub default: Option, + /// Schema of the field. + pub schema: Schema, + /// Order of the field. + /// + /// **NOTE** This currently has no effect. + #[builder(default = RecordFieldOrder::Ignore)] + pub order: RecordFieldOrder, + /// Position of the field in the list of `field` of its parent `Schema` + #[builder(default)] + pub position: usize, + /// A collection of all unknown fields in the record field. + #[builder(default = BTreeMap::new())] + pub custom_attributes: BTreeMap, + } + + /// Represents any valid order for a `field` in a `record` Avro schema. + #[derive(Clone, Debug, Eq, PartialEq, EnumString)] + #[strum(serialize_all = "kebab_case")] + pub enum RecordFieldOrder { + Ascending, + Descending, + Ignore, + } + + impl RecordField { + /// Parse a `serde_json::Value` into a `RecordField`. + async fn parse( + field: &Map, + position: usize, + parser: &mut Parser, + enclosing_record: &Name, + ) -> AvroResult { + let name = field.name().ok_or(Details::GetNameFieldFromRecord)?; -/// A description of an Enum schema. -#[derive(bon::Builder, Debug, Clone)] -pub struct EnumSchema { - /// The name of the schema - pub name: Name, - /// The aliases of the schema - #[builder(default)] - pub aliases: Aliases, - /// The documentation of the schema - #[builder(default)] - pub doc: Documentation, - /// The set of symbols of the schema - pub symbols: Vec, - /// An optional default symbol used for compatibility - pub default: Option, - /// The custom attributes of the schema - #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, -} + validate_record_field_name(&name)?; -/// A description of a Union schema. -#[derive(bon::Builder, Debug, Clone)] -pub struct FixedSchema { - /// The name of the schema - pub name: Name, - /// The aliases of the schema - #[builder(default)] - pub aliases: Aliases, - /// The documentation of the schema - #[builder(default)] - pub doc: Documentation, - /// The size of the fixed schema - pub size: usize, - /// An optional default symbol used for compatibility - pub default: Option, - /// The custom attributes of the schema - #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, -} - -impl FixedSchema { - fn serialize_to_map(&self, mut map: S::SerializeMap) -> Result - where - S: Serializer, - { - map.serialize_entry("type", "fixed")?; - if let Some(ref n) = self.name.namespace { - map.serialize_entry("namespace", n)?; - } - map.serialize_entry("name", &self.name.name)?; - if let Some(ref docstr) = self.doc { - map.serialize_entry("doc", docstr)?; - } - map.serialize_entry("size", &self.size)?; + // TODO: "type" = "" + let schema = parser.parse_complex( + field, + &enclosing_record.namespace, + RecordSchemaParseLocation::FromField, + )?; - if let Some(ref aliases) = self.aliases { - map.serialize_entry("aliases", aliases)?; - } + let default = field.get("default").cloned(); + Self::resolve_default_value( + &schema, + &name, + &enclosing_record.fullname(None), + &parser.parsed_schemas, + &default, + ) + .await?; + + let aliases = field.get("aliases").and_then(|aliases| { + aliases.as_array().map(|aliases| { + aliases + .iter() + .flat_map(|alias| alias.as_str()) + .map(|alias| alias.to_string()) + .collect::>() + }) + }); - for attr in &self.attributes { - map.serialize_entry(attr.0, attr.1)?; + let order = field + .get("order") + .and_then(|order| order.as_str()) + .and_then(|order| RecordFieldOrder::from_str(order).ok()) + .unwrap_or(RecordFieldOrder::Ascending); + + Ok(RecordField { + name, + doc: field.doc(), + default, + aliases, + order, + position, + custom_attributes: RecordField::get_field_custom_attributes(field, &schema), + schema, + }) } - Ok(map) - } -} + async fn resolve_default_value( + field_schema: &Schema, + field_name: &str, + record_name: &str, + names: &Names, + default: &Option, + ) -> AvroResult<()> { + if let Some(value) = default { + let avro_value = types::Value::from(value.clone()); + match field_schema { + Schema::Union(union_schema) => { + let schemas = &union_schema.schemas; + + let mut resolved = false; + for schema in schemas { + if avro_value + .to_owned() + .resolve_internal(schema, names, &schema.namespace(), &None) + .await + .is_ok() + { + resolved = true; + break; + } + } -/// A description of a Union schema. -/// -/// `scale` defaults to 0 and is an integer greater than or equal to 0 and `precision` is an -/// integer greater than 0. -#[derive(Debug, Clone)] -pub struct DecimalSchema { - /// The number of digits in the unscaled value - pub precision: DecimalMetadata, - /// The number of digits to the right of the decimal point - pub scale: DecimalMetadata, - /// The inner schema of the decimal (fixed or bytes) - pub inner: Box, -} + if !resolved { + let schema: Option<&Schema> = schemas.first(); + return match schema { + Some(first_schema) => Err(Details::GetDefaultUnion( + SchemaKind::from(first_schema), + types::ValueKind::from(avro_value), + ) + .into()), + None => Err(Details::EmptyUnion.into()), + }; + } + } + _ => { + let resolved = avro_value + .resolve_internal(field_schema, names, &field_schema.namespace(), &None) + .await + .is_ok(); + + if !resolved { + return Err(Details::GetDefaultRecordField( + field_name.to_string(), + record_name.to_string(), + field_schema.canonical_form(), + ) + .into()); + } + } + }; + } -/// A description of a Union schema -#[derive(Debug, Clone)] -pub struct UnionSchema { - /// The schemas that make up this union - pub(crate) schemas: Vec, - // Used to ensure uniqueness of schema inputs, and provide constant time finding of the - // schema index given a value. - // **NOTE** that this approach does not work for named types, and will have to be modified - // to support that. A simple solution is to also keep a mapping of the names used. - variant_index: BTreeMap, -} + Ok(()) + } -impl UnionSchema { - /// Creates a new UnionSchema from a vector of schemas. - pub fn new(schemas: Vec) -> AvroResult { - let mut vindex = BTreeMap::new(); - for (i, schema) in schemas.iter().enumerate() { - if let Schema::Union(_) = schema { - return Err(Details::GetNestedUnion.into()); + fn get_field_custom_attributes( + field: &Map, + schema: &Schema, + ) -> BTreeMap { + let mut custom_attributes: BTreeMap = BTreeMap::new(); + for (key, value) in field { + match key.as_str() { + "type" | "name" | "doc" | "default" | "order" | "position" | "aliases" + | "logicalType" => continue, + key if key == "symbols" && matches!(schema, Schema::Enum(_)) => continue, + key if key == "size" && matches!(schema, Schema::Fixed(_)) => continue, + _ => custom_attributes.insert(key.clone(), value.clone()), + }; } - let kind = SchemaKind::from(schema); - if !kind.is_named() && vindex.insert(kind, i).is_some() { - return Err(Details::GetUnionDuplicate.into()); + custom_attributes + } + + /// Returns true if this `RecordField` is nullable, meaning the schema is a `UnionSchema` where the first variant is `Null`. + pub fn is_nullable(&self) -> bool { + match self.schema { + Schema::Union(ref inner) => inner.is_nullable(), + _ => false, + } + } + } + + /// A description of an Enum schema. + #[derive(bon::Builder, Debug, Clone)] + pub struct RecordSchema { + /// The name of the schema + pub name: Name, + /// The aliases of the schema + #[builder(default)] + pub aliases: Aliases, + /// The documentation of the schema + #[builder(default)] + pub doc: Documentation, + /// The set of fields of the schema + pub fields: Vec, + /// The `lookup` table maps field names to their position in the `Vec` + /// of `fields`. + pub lookup: BTreeMap, + /// The custom attributes of the schema + #[builder(default = BTreeMap::new())] + pub attributes: BTreeMap, + } + + /// A description of an Enum schema. + #[derive(bon::Builder, Debug, Clone)] + pub struct EnumSchema { + /// The name of the schema + pub name: Name, + /// The aliases of the schema + #[builder(default)] + pub aliases: Aliases, + /// The documentation of the schema + #[builder(default)] + pub doc: Documentation, + /// The set of symbols of the schema + pub symbols: Vec, + /// An optional default symbol used for compatibility + pub default: Option, + /// The custom attributes of the schema + #[builder(default = BTreeMap::new())] + pub attributes: BTreeMap, + } + + /// A description of a Union schema. + #[derive(bon::Builder, Debug, Clone)] + pub struct FixedSchema { + /// The name of the schema + pub name: Name, + /// The aliases of the schema + #[builder(default)] + pub aliases: Aliases, + /// The documentation of the schema + #[builder(default)] + pub doc: Documentation, + /// The size of the fixed schema + pub size: usize, + /// An optional default symbol used for compatibility + pub default: Option, + /// The custom attributes of the schema + #[builder(default = BTreeMap::new())] + pub attributes: BTreeMap, + } + + impl FixedSchema { + fn serialize_to_map(&self, mut map: S::SerializeMap) -> Result + where + S: Serializer, + { + map.serialize_entry("type", "fixed")?; + if let Some(ref n) = self.name.namespace { + map.serialize_entry("namespace", n)?; } - } - Ok(UnionSchema { - schemas, - variant_index: vindex, - }) - } + map.serialize_entry("name", &self.name.name)?; + if let Some(ref docstr) = self.doc { + map.serialize_entry("doc", docstr)?; + } + map.serialize_entry("size", &self.size)?; - /// Returns a slice to all variants of this schema. - pub fn variants(&self) -> &[Schema] { - &self.schemas - } + if let Some(ref aliases) = self.aliases { + map.serialize_entry("aliases", aliases)?; + } + + for attr in &self.attributes { + map.serialize_entry(attr.0, attr.1)?; + } - /// Returns true if the any of the variants of this `UnionSchema` is `Null`. - pub fn is_nullable(&self) -> bool { - self.schemas.iter().any(|x| matches!(x, Schema::Null)) + Ok(map) + } } - /// Optionally returns a reference to the schema matched by this value, as well as its position - /// within this union. + /// A description of a Union schema. /// - /// Extra arguments: - /// - `known_schemata` - mapping between `Name` and `Schema` - if passed, additional external schemas would be used to resolve references. - pub fn find_schema_with_known_schemata + Debug>( - &self, - value: &types::Value, - known_schemata: Option<&HashMap>, - enclosing_namespace: &Namespace, - ) -> Option<(usize, &Schema)> { - let schema_kind = SchemaKind::from(value); - if let Some(&i) = self.variant_index.get(&schema_kind) { - // fast path - Some((i, &self.schemas[i])) - } else { - // slow path (required for matching logical or named types) - - // first collect what schemas we already know - let mut collected_names: HashMap = known_schemata - .map(|names| { - names - .iter() - .map(|(name, schema)| (name.clone(), schema.borrow())) - .collect() - }) - .unwrap_or_default(); - - self.schemas.iter().enumerate().find(async |(_, schema)| { - let resolved_schema = ResolvedSchema::new_with_known_schemata( - vec![*schema], - enclosing_namespace, - &collected_names, - ) - .expect("Schema didn't successfully parse"); - let resolved_names = resolved_schema.names_ref; - - // extend known schemas with just resolved names - collected_names.extend(resolved_names); - let namespace = &schema.namespace().or_else(|| enclosing_namespace.clone()); - - value - .clone() - .resolve_internal(schema, &collected_names, namespace, &None).await - .is_ok() + /// `scale` defaults to 0 and is an integer greater than or equal to 0 and `precision` is an + /// integer greater than 0. + #[derive(Debug, Clone)] + pub struct DecimalSchema { + /// The number of digits in the unscaled value + pub precision: DecimalMetadata, + /// The number of digits to the right of the decimal point + pub scale: DecimalMetadata, + /// The inner schema of the decimal (fixed or bytes) + pub inner: Box, + } + + /// A description of a Union schema + #[derive(Debug, Clone)] + pub struct UnionSchema { + /// The schemas that make up this union + pub(crate) schemas: Vec, + // Used to ensure uniqueness of schema inputs, and provide constant time finding of the + // schema index given a value. + // **NOTE** that this approach does not work for named types, and will have to be modified + // to support that. A simple solution is to also keep a mapping of the names used. + variant_index: BTreeMap, + } + + impl UnionSchema { + /// Creates a new UnionSchema from a vector of schemas. + pub fn new(schemas: Vec) -> AvroResult { + let mut vindex = BTreeMap::new(); + for (i, schema) in schemas.iter().enumerate() { + if let Schema::Union(_) = schema { + return Err(Details::GetNestedUnion.into()); + } + let kind = SchemaKind::from(schema); + if !kind.is_named() && vindex.insert(kind, i).is_some() { + return Err(Details::GetUnionDuplicate.into()); + } + } + Ok(UnionSchema { + schemas, + variant_index: vindex, }) } - } -} - -// No need to compare variant_index, it is derivative of schemas. -impl PartialEq for UnionSchema { - fn eq(&self, other: &UnionSchema) -> bool { - self.schemas.eq(&other.schemas) - } -} - -type DecimalMetadata = usize; -pub(crate) type Precision = DecimalMetadata; -pub(crate) type Scale = DecimalMetadata; - -fn parse_json_integer_for_decimal(value: &serde_json::Number) -> Result { - Ok(if value.is_u64() { - let num = value - .as_u64() - .ok_or_else(|| Details::GetU64FromJson(value.clone()))?; - num.try_into() - .map_err(|e| Details::ConvertU64ToUsize(e, num))? - } else if value.is_i64() { - let num = value - .as_i64() - .ok_or_else(|| Details::GetI64FromJson(value.clone()))?; - num.try_into() - .map_err(|e| Details::ConvertI64ToUsize(e, num))? - } else { - return Err(Details::GetPrecisionOrScaleFromJson(value.clone()).into()); - }) -} - -#[derive(Debug, Default)] -enum RecordSchemaParseLocation { - /// When the parse is happening at root level - #[default] - Root, - - /// When the parse is happening inside a record field - FromField, -} -#[derive(Default)] -struct Parser { - input_schemas: HashMap, - /// A map of name -> Schema::Ref - /// Used to resolve cyclic references, i.e. when a - /// field's type is a reference to its record's type - resolving_schemas: Names, - input_order: Vec, - /// A map of name -> fully parsed Schema - /// Used to avoid parsing the same schema twice - parsed_schemas: Names, -} + /// Returns a slice to all variants of this schema. + pub fn variants(&self) -> &[Schema] { + &self.schemas + } -impl Schema { - /// Converts `self` into its [Parsing Canonical Form]. - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - pub fn canonical_form(&self) -> String { - let json = serde_json::to_value(self) - .unwrap_or_else(|e| panic!("Cannot parse Schema from JSON: {e}")); - let mut defined_names = HashSet::new(); - parsing_canonical_form(&json, &mut defined_names) - } + /// Returns true if the any of the variants of this `UnionSchema` is `Null`. + pub fn is_nullable(&self) -> bool { + self.schemas.iter().any(|x| matches!(x, Schema::Null)) + } - /// Returns the [Parsing Canonical Form] of `self` that is self contained (not dependent on - /// any definitions in `schemata`) - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - pub fn independent_canonical_form(&self, schemata: &[Schema]) -> Result { - let mut this = self.clone(); - this.denormalize(schemata)?; - Ok(this.canonical_form()) + /// Optionally returns a reference to the schema matched by this value, as well as its position + /// within this union. + /// + /// Extra arguments: + /// - `known_schemata` - mapping between `Name` and `Schema` - if passed, additional external schemas would be used to resolve references. + pub async fn find_schema_with_known_schemata + Debug>( + &self, + value: &types::Value, + known_schemata: Option<&HashMap>, + enclosing_namespace: &Namespace, + ) -> Option<(usize, &Schema)> { + let schema_kind = SchemaKind::from(value); + if let Some(&i) = self.variant_index.get(&schema_kind) { + // fast path + Some((i, &self.schemas[i])) + } else { + // slow path (required for matching logical or named types) + + // first collect what schemas we already know + let mut collected_names: HashMap = known_schemata + .map(|names| { + names + .iter() + .map(|(name, schema)| (name.clone(), schema.borrow())) + .collect() + }) + .unwrap_or_default(); + + let mut i: usize = 0; + for schema in self.schemas.iter() { + let resolved_schema = ResolvedSchema::new_with_known_schemata( + vec![schema], + enclosing_namespace, + &collected_names, + ) + .expect("Schema didn't successfully parse"); + let resolved_names = resolved_schema.names_ref; + + // extend known schemas with just resolved names + collected_names.extend(resolved_names); + let namespace = &schema.namespace().or_else(|| enclosing_namespace.clone()); + + if value + .clone() + .resolve_internal(schema, &collected_names, namespace, &None) + .await + .is_ok() + { + return Some((i, schema)); + } else { + i = i + 1; + } + } + None + } + } } - /// Generate [fingerprint] of Schema's [Parsing Canonical Form]. - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - /// [fingerprint]: - /// https://avro.apache.org/docs/current/specification/#schema-fingerprints - pub fn fingerprint(&self) -> SchemaFingerprint { - let mut d = D::new(); - d.update(self.canonical_form()); - SchemaFingerprint { - bytes: d.finalize().to_vec(), + // No need to compare variant_index, it is derivative of schemas. + impl PartialEq for UnionSchema { + fn eq(&self, other: &UnionSchema) -> bool { + self.schemas.eq(&other.schemas) } } - /// Create a `Schema` from a string representing a JSON Avro schema. - pub fn parse_str(input: &str) -> Result { - let mut parser = Parser::default(); - parser.parse_str(input) + type DecimalMetadata = usize; + pub(crate) type Precision = DecimalMetadata; + pub(crate) type Scale = DecimalMetadata; + + fn parse_json_integer_for_decimal( + value: &serde_json::Number, + ) -> Result { + Ok(if value.is_u64() { + let num = value + .as_u64() + .ok_or_else(|| Details::GetU64FromJson(value.clone()))?; + num.try_into() + .map_err(|e| Details::ConvertU64ToUsize(e, num))? + } else if value.is_i64() { + let num = value + .as_i64() + .ok_or_else(|| Details::GetI64FromJson(value.clone()))?; + num.try_into() + .map_err(|e| Details::ConvertI64ToUsize(e, num))? + } else { + return Err(Details::GetPrecisionOrScaleFromJson(value.clone()).into()); + }) } - /// Create an array of `Schema`'s from a list of named JSON Avro schemas (Record, Enum, and - /// Fixed). - /// - /// It is allowed that the schemas have cross-dependencies; these will be resolved - /// during parsing. - /// - /// If two of the input schemas have the same fullname, an Error will be returned. - pub fn parse_list(input: impl IntoIterator>) -> AvroResult> { - let input = input.into_iter(); - let input_len = input.size_hint().0; - let mut input_schemas: HashMap = HashMap::with_capacity(input_len); - let mut input_order: Vec = Vec::with_capacity(input_len); - for json in input { - let json = json.as_ref(); - let schema: Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; - if let Value::Object(inner) = &schema { - let name = Name::parse(inner, &None)?; - let previous_value = input_schemas.insert(name.clone(), schema); - if previous_value.is_some() { - return Err(Details::NameCollision(name.fullname(None)).into()); + #[derive(Debug, Default)] + enum RecordSchemaParseLocation { + /// When the parse is happening at root level + #[default] + Root, + + /// When the parse is happening inside a record field + FromField, + } + + #[derive(Default)] + struct Parser { + input_schemas: HashMap, + /// A map of name -> Schema::Ref + /// Used to resolve cyclic references, i.e. when a + /// field's type is a reference to its record's type + resolving_schemas: Names, + input_order: Vec, + /// A map of name -> fully parsed Schema + /// Used to avoid parsing the same schema twice + parsed_schemas: Names, + } + + impl Schema { + /// Converts `self` into its [Parsing Canonical Form]. + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + pub fn canonical_form(&self) -> String { + let json = serde_json::to_value(self) + .unwrap_or_else(|e| panic!("Cannot parse Schema from JSON: {e}")); + let mut defined_names = HashSet::new(); + parsing_canonical_form(&json, &mut defined_names) + } + + /// Returns the [Parsing Canonical Form] of `self` that is self contained (not dependent on + /// any definitions in `schemata`) + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + pub fn independent_canonical_form(&self, schemata: &[Schema]) -> Result { + let mut this = self.clone(); + this.denormalize(schemata)?; + Ok(this.canonical_form()) + } + + /// Generate [fingerprint] of Schema's [Parsing Canonical Form]. + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + /// [fingerprint]: + /// https://avro.apache.org/docs/current/specification/#schema-fingerprints + pub fn fingerprint(&self) -> SchemaFingerprint { + let mut d = D::new(); + d.update(self.canonical_form()); + SchemaFingerprint { + bytes: d.finalize().to_vec(), + } + } + + /// Create a `Schema` from a string representing a JSON Avro schema. + pub fn parse_str(input: &str) -> Result { + let mut parser = Parser::default(); + parser.parse_str(input) + } + + /// Create an array of `Schema`'s from a list of named JSON Avro schemas (Record, Enum, and + /// Fixed). + /// + /// It is allowed that the schemas have cross-dependencies; these will be resolved + /// during parsing. + /// + /// If two of the input schemas have the same fullname, an Error will be returned. + pub fn parse_list( + input: impl IntoIterator>, + ) -> AvroResult> { + let input = input.into_iter(); + let input_len = input.size_hint().0; + let mut input_schemas: HashMap = HashMap::with_capacity(input_len); + let mut input_order: Vec = Vec::with_capacity(input_len); + for json in input { + let json = json.as_ref(); + let schema: Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; + if let Value::Object(inner) = &schema { + let name = Name::parse(inner, &None)?; + let previous_value = input_schemas.insert(name.clone(), schema); + if previous_value.is_some() { + return Err(Details::NameCollision(name.fullname(None)).into()); + } + input_order.push(name); + } else { + return Err(Details::GetNameField.into()); } - input_order.push(name); - } else { - return Err(Details::GetNameField.into()); } - } - let mut parser = Parser { - input_schemas, - resolving_schemas: HashMap::default(), - input_order, - parsed_schemas: HashMap::with_capacity(input_len), - }; - parser.parse_list() - } - - /// Create a `Schema` from a string representing a JSON Avro schema, - /// along with an array of `Schema`'s from a list of named JSON Avro schemas (Record, Enum, and - /// Fixed). - /// - /// It is allowed that the schemas have cross-dependencies; these will be resolved - /// during parsing. - /// - /// If two of the named input schemas have the same fullname, an Error will be returned. - /// - /// # Arguments - /// * `schema` - the JSON string of the schema to parse - /// * `schemata` - a slice of additional schemas that is used to resolve cross-references - pub fn parse_str_with_list( - schema: &str, - schemata: impl IntoIterator>, - ) -> AvroResult<(Schema, Vec)> { - let schemata = schemata.into_iter(); - let schemata_len = schemata.size_hint().0; - let mut input_schemas: HashMap = HashMap::with_capacity(schemata_len); - let mut input_order: Vec = Vec::with_capacity(schemata_len); - for json in schemata { - let json = json.as_ref(); - let schema: Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; - if let Value::Object(inner) = &schema { - let name = Name::parse(inner, &None)?; - if let Some(_previous) = input_schemas.insert(name.clone(), schema) { - return Err(Details::NameCollision(name.fullname(None)).into()); + let mut parser = Parser { + input_schemas, + resolving_schemas: HashMap::default(), + input_order, + parsed_schemas: HashMap::with_capacity(input_len), + }; + parser.parse_list() + } + + /// Create a `Schema` from a string representing a JSON Avro schema, + /// along with an array of `Schema`'s from a list of named JSON Avro schemas (Record, Enum, and + /// Fixed). + /// + /// It is allowed that the schemas have cross-dependencies; these will be resolved + /// during parsing. + /// + /// If two of the named input schemas have the same fullname, an Error will be returned. + /// + /// # Arguments + /// * `schema` - the JSON string of the schema to parse + /// * `schemata` - a slice of additional schemas that is used to resolve cross-references + pub fn parse_str_with_list( + schema: &str, + schemata: impl IntoIterator>, + ) -> AvroResult<(Schema, Vec)> { + let schemata = schemata.into_iter(); + let schemata_len = schemata.size_hint().0; + let mut input_schemas: HashMap = HashMap::with_capacity(schemata_len); + let mut input_order: Vec = Vec::with_capacity(schemata_len); + for json in schemata { + let json = json.as_ref(); + let schema: Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; + if let Value::Object(inner) = &schema { + let name = Name::parse(inner, &None)?; + if let Some(_previous) = input_schemas.insert(name.clone(), schema) { + return Err(Details::NameCollision(name.fullname(None)).into()); + } + input_order.push(name); + } else { + return Err(Details::GetNameField.into()); } - input_order.push(name); - } else { - return Err(Details::GetNameField.into()); } - } - let mut parser = Parser { - input_schemas, - resolving_schemas: HashMap::default(), - input_order, - parsed_schemas: HashMap::with_capacity(schemata_len), - }; - parser.parse_input_schemas()?; + let mut parser = Parser { + input_schemas, + resolving_schemas: HashMap::default(), + input_order, + parsed_schemas: HashMap::with_capacity(schemata_len), + }; + parser.parse_input_schemas()?; - let value = serde_json::from_str(schema).map_err(Details::ParseSchemaJson)?; - let schema = parser.parse(&value, &None)?; - let schemata = parser.parse_list()?; - Ok((schema, schemata)) - } + let value = serde_json::from_str(schema).map_err(Details::ParseSchemaJson)?; + let schema = parser.parse(&value, &None)?; + let schemata = parser.parse_list()?; + Ok((schema, schemata)) + } - /// Create a `Schema` from a reader which implements [`Read`]. - pub fn parse_reader(reader: &mut (impl Read + ?Sized)) -> AvroResult { - let mut buf = String::new(); - match reader.read_to_string(&mut buf) { - Ok(_) => Self::parse_str(&buf), - Err(e) => Err(Details::ReadSchemaFromReader(e).into()), + /// Create a `Schema` from a reader which implements [`Read`]. + pub fn parse_reader(reader: &mut (impl Read + ?Sized)) -> AvroResult { + let mut buf = String::new(); + match reader.read_to_string(&mut buf) { + Ok(_) => Self::parse_str(&buf), + Err(e) => Err(Details::ReadSchemaFromReader(e).into()), + } } - } - /// Parses an Avro schema from JSON. - pub fn parse(value: &Value) -> AvroResult { - let mut parser = Parser::default(); - parser.parse(value, &None) - } + /// Parses an Avro schema from JSON. + pub fn parse(value: &Value) -> AvroResult { + let mut parser = Parser::default(); + parser.parse(value, &None) + } - /// Parses an Avro schema from JSON. - /// Any `Schema::Ref`s must be known in the `names` map. - pub(crate) fn parse_with_names(value: &Value, names: Names) -> AvroResult { - let mut parser = Parser { - input_schemas: HashMap::with_capacity(1), - resolving_schemas: Names::default(), - input_order: Vec::with_capacity(1), - parsed_schemas: names, - }; - parser.parse(value, &None) - } + /// Parses an Avro schema from JSON. + /// Any `Schema::Ref`s must be known in the `names` map. + pub(crate) fn parse_with_names(value: &Value, names: Names) -> AvroResult { + let mut parser = Parser { + input_schemas: HashMap::with_capacity(1), + resolving_schemas: Names::default(), + input_order: Vec::with_capacity(1), + parsed_schemas: names, + }; + parser.parse(value, &None) + } - /// Returns the custom attributes (metadata) if the schema supports them. - pub fn custom_attributes(&self) -> Option<&BTreeMap> { - match self { - Schema::Record(RecordSchema { attributes, .. }) - | Schema::Enum(EnumSchema { attributes, .. }) - | Schema::Fixed(FixedSchema { attributes, .. }) - | Schema::Array(ArraySchema { attributes, .. }) - | Schema::Map(MapSchema { attributes, .. }) => Some(attributes), - _ => None, + /// Returns the custom attributes (metadata) if the schema supports them. + pub fn custom_attributes(&self) -> Option<&BTreeMap> { + match self { + Schema::Record(RecordSchema { attributes, .. }) + | Schema::Enum(EnumSchema { attributes, .. }) + | Schema::Fixed(FixedSchema { attributes, .. }) + | Schema::Array(ArraySchema { attributes, .. }) + | Schema::Map(MapSchema { attributes, .. }) => Some(attributes), + _ => None, + } } - } - /// Returns the name of the schema if it has one. - pub fn name(&self) -> Option<&Name> { - match self { - Schema::Ref { name, .. } - | Schema::Record(RecordSchema { name, .. }) - | Schema::Enum(EnumSchema { name, .. }) - | Schema::Fixed(FixedSchema { name, .. }) => Some(name), - _ => None, + /// Returns the name of the schema if it has one. + pub fn name(&self) -> Option<&Name> { + match self { + Schema::Ref { name, .. } + | Schema::Record(RecordSchema { name, .. }) + | Schema::Enum(EnumSchema { name, .. }) + | Schema::Fixed(FixedSchema { name, .. }) => Some(name), + _ => None, + } } - } - /// Returns the namespace of the schema if it has one. - pub fn namespace(&self) -> Namespace { - self.name().and_then(|n| n.namespace.clone()) - } + /// Returns the namespace of the schema if it has one. + pub fn namespace(&self) -> Namespace { + self.name().and_then(|n| n.namespace.clone()) + } - /// Returns the aliases of the schema if it has ones. - pub fn aliases(&self) -> Option<&Vec> { - match self { - Schema::Record(RecordSchema { aliases, .. }) - | Schema::Enum(EnumSchema { aliases, .. }) - | Schema::Fixed(FixedSchema { aliases, .. }) => aliases.as_ref(), - _ => None, + /// Returns the aliases of the schema if it has ones. + pub fn aliases(&self) -> Option<&Vec> { + match self { + Schema::Record(RecordSchema { aliases, .. }) + | Schema::Enum(EnumSchema { aliases, .. }) + | Schema::Fixed(FixedSchema { aliases, .. }) => aliases.as_ref(), + _ => None, + } } - } - /// Returns the doc of the schema if it has one. - pub fn doc(&self) -> Option<&String> { - match self { - Schema::Record(RecordSchema { doc, .. }) - | Schema::Enum(EnumSchema { doc, .. }) - | Schema::Fixed(FixedSchema { doc, .. }) => doc.as_ref(), - _ => None, + /// Returns the doc of the schema if it has one. + pub fn doc(&self) -> Option<&String> { + match self { + Schema::Record(RecordSchema { doc, .. }) + | Schema::Enum(EnumSchema { doc, .. }) + | Schema::Fixed(FixedSchema { doc, .. }) => doc.as_ref(), + _ => None, + } } - } - /// Returns a Schema::Map with the given types. - pub fn map(types: Schema) -> Self { - Schema::Map(MapSchema { - types: Box::new(types), - attributes: Default::default(), - }) - } + /// Returns a Schema::Map with the given types. + pub fn map(types: Schema) -> Self { + Schema::Map(MapSchema { + types: Box::new(types), + attributes: Default::default(), + }) + } - /// Returns a Schema::Map with the given types and custom attributes. - pub fn map_with_attributes(types: Schema, attributes: BTreeMap) -> Self { - Schema::Map(MapSchema { - types: Box::new(types), - attributes, - }) - } + /// Returns a Schema::Map with the given types and custom attributes. + pub fn map_with_attributes(types: Schema, attributes: BTreeMap) -> Self { + Schema::Map(MapSchema { + types: Box::new(types), + attributes, + }) + } - /// Returns a Schema::Array with the given items. - pub fn array(items: Schema) -> Self { - Schema::Array(ArraySchema { - items: Box::new(items), - attributes: Default::default(), - }) - } + /// Returns a Schema::Array with the given items. + pub fn array(items: Schema) -> Self { + Schema::Array(ArraySchema { + items: Box::new(items), + attributes: Default::default(), + }) + } - /// Returns a Schema::Array with the given items and custom attributes. - pub fn array_with_attributes(items: Schema, attributes: BTreeMap) -> Self { - Schema::Array(ArraySchema { - items: Box::new(items), - attributes, - }) - } + /// Returns a Schema::Array with the given items and custom attributes. + pub fn array_with_attributes(items: Schema, attributes: BTreeMap) -> Self { + Schema::Array(ArraySchema { + items: Box::new(items), + attributes, + }) + } - fn denormalize(&mut self, schemata: &[Schema]) -> AvroResult<()> { - match self { - Schema::Ref { name } => { - let replacement_schema = schemata - .iter() - .find(|s| s.name().map(|n| *n == *name).unwrap_or(false)); - if let Some(schema) = replacement_schema { - let mut denorm = schema.clone(); - denorm.denormalize(schemata)?; - *self = denorm; - } else { - return Err(Details::SchemaResolutionError(name.clone()).into()); + fn denormalize(&mut self, schemata: &[Schema]) -> AvroResult<()> { + match self { + Schema::Ref { name } => { + let replacement_schema = schemata + .iter() + .find(|s| s.name().map(|n| *n == *name).unwrap_or(false)); + if let Some(schema) = replacement_schema { + let mut denorm = schema.clone(); + denorm.denormalize(schemata)?; + *self = denorm; + } else { + return Err(Details::SchemaResolutionError(name.clone()).into()); + } } - } - Schema::Record(record_schema) => { - for field in &mut record_schema.fields { - field.schema.denormalize(schemata)?; + Schema::Record(record_schema) => { + for field in &mut record_schema.fields { + field.schema.denormalize(schemata)?; + } } - } - Schema::Array(array_schema) => { - array_schema.items.denormalize(schemata)?; - } - Schema::Map(map_schema) => { - map_schema.types.denormalize(schemata)?; - } - Schema::Union(union_schema) => { - for schema in &mut union_schema.schemas { - schema.denormalize(schemata)?; + Schema::Array(array_schema) => { + array_schema.items.denormalize(schemata)?; } + Schema::Map(map_schema) => { + map_schema.types.denormalize(schemata)?; + } + Schema::Union(union_schema) => { + for schema in &mut union_schema.schemas { + schema.denormalize(schemata)?; + } + } + _ => (), } - _ => (), + Ok(()) } - Ok(()) - } -} - -impl Parser { - /// Create a `Schema` from a string representing a JSON Avro schema. - fn parse_str(&mut self, input: &str) -> Result { - let value = serde_json::from_str(input).map_err(Details::ParseSchemaJson)?; - self.parse(&value, &None) } - /// Create an array of `Schema`'s from an iterator of JSON Avro schemas. It is allowed that - /// the schemas have cross-dependencies; these will be resolved during parsing. - fn parse_list(&mut self) -> Result, Error> { - self.parse_input_schemas()?; - - let mut parsed_schemas = Vec::with_capacity(self.parsed_schemas.len()); - for name in self.input_order.drain(0..) { - let parsed = self - .parsed_schemas - .remove(&name) - .expect("One of the input schemas was unexpectedly not parsed"); - parsed_schemas.push(parsed); + impl Parser { + /// Create a `Schema` from a string representing a JSON Avro schema. + fn parse_str(&mut self, input: &str) -> Result { + let value = serde_json::from_str(input).map_err(Details::ParseSchemaJson)?; + self.parse(&value, &None) } - Ok(parsed_schemas) - } - /// Convert the input schemas to parsed_schemas - fn parse_input_schemas(&mut self) -> Result<(), Error> { - while !self.input_schemas.is_empty() { - let next_name = self - .input_schemas - .keys() - .next() - .expect("Input schemas unexpectedly empty") - .to_owned(); - let (name, value) = self - .input_schemas - .remove_entry(&next_name) - .expect("Key unexpectedly missing"); - let parsed = self.parse(&value, &None)?; - self.parsed_schemas - .insert(get_schema_type_name(name, value), parsed); - } - Ok(()) - } + /// Create an array of `Schema`'s from an iterator of JSON Avro schemas. It is allowed that + /// the schemas have cross-dependencies; these will be resolved during parsing. + fn parse_list(&mut self) -> Result, Error> { + self.parse_input_schemas()?; - /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro - /// schema. - fn parse(&mut self, value: &Value, enclosing_namespace: &Namespace) -> AvroResult { - match *value { - Value::String(ref t) => self.parse_known_schema(t.as_str(), enclosing_namespace), - Value::Object(ref data) => { - self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) + let mut parsed_schemas = Vec::with_capacity(self.parsed_schemas.len()); + for name in self.input_order.drain(0..) { + let parsed = self + .parsed_schemas + .remove(&name) + .expect("One of the input schemas was unexpectedly not parsed"); + parsed_schemas.push(parsed); } - Value::Array(ref data) => self.parse_union(data, enclosing_namespace), - _ => Err(Details::ParseSchemaFromValidJson.into()), + Ok(parsed_schemas) } - } - /// Parse a `serde_json::Value` representing an Avro type whose Schema is known into a - /// `Schema`. A Schema for a `serde_json::Value` is known if it is primitive or has - /// been parsed previously by the parsed and stored in its map of parsed_schemas. - fn parse_known_schema( - &mut self, - name: &str, - enclosing_namespace: &Namespace, - ) -> AvroResult { - match name { - "null" => Ok(Schema::Null), - "boolean" => Ok(Schema::Boolean), - "int" => Ok(Schema::Int), - "long" => Ok(Schema::Long), - "double" => Ok(Schema::Double), - "float" => Ok(Schema::Float), - "bytes" => Ok(Schema::Bytes), - "string" => Ok(Schema::String), - _ => self.fetch_schema_ref(name, enclosing_namespace), + /// Convert the input schemas to parsed_schemas + fn parse_input_schemas(&mut self) -> Result<(), Error> { + while !self.input_schemas.is_empty() { + let next_name = self + .input_schemas + .keys() + .next() + .expect("Input schemas unexpectedly empty") + .to_owned(); + let (name, value) = self + .input_schemas + .remove_entry(&next_name) + .expect("Key unexpectedly missing"); + let parsed = self.parse(&value, &None)?; + self.parsed_schemas + .insert(get_schema_type_name(name, value), parsed); + } + Ok(()) } - } - /// Given a name, tries to retrieve the parsed schema from `parsed_schemas`. - /// If a parsed schema is not found, it checks if a currently resolving - /// schema with that name exists. - /// If a resolving schema is not found, it checks if a json with that name exists - /// in `input_schemas` and then parses it (removing it from `input_schemas`) - /// and adds the parsed schema to `parsed_schemas`. - /// - /// This method allows schemas definitions that depend on other types to - /// parse their dependencies (or look them up if already parsed). - fn fetch_schema_ref( - &mut self, - name: &str, - enclosing_namespace: &Namespace, - ) -> AvroResult { - fn get_schema_ref(parsed: &Schema) -> Schema { - match parsed { - &Schema::Record(RecordSchema { ref name, .. }) - | &Schema::Enum(EnumSchema { ref name, .. }) - | &Schema::Fixed(FixedSchema { ref name, .. }) => { - Schema::Ref { name: name.clone() } + /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro + /// schema. + fn parse(&mut self, value: &Value, enclosing_namespace: &Namespace) -> AvroResult { + match *value { + Value::String(ref t) => self.parse_known_schema(t.as_str(), enclosing_namespace), + Value::Object(ref data) => { + self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) } - _ => parsed.clone(), + Value::Array(ref data) => self.parse_union(data, enclosing_namespace), + _ => Err(Details::ParseSchemaFromValidJson.into()), } } - let name = Name::new(name)?; - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + /// Parse a `serde_json::Value` representing an Avro type whose Schema is known into a + /// `Schema`. A Schema for a `serde_json::Value` is known if it is primitive or has + /// been parsed previously by the parsed and stored in its map of parsed_schemas. + fn parse_known_schema( + &mut self, + name: &str, + enclosing_namespace: &Namespace, + ) -> AvroResult { + match name { + "null" => Ok(Schema::Null), + "boolean" => Ok(Schema::Boolean), + "int" => Ok(Schema::Int), + "long" => Ok(Schema::Long), + "double" => Ok(Schema::Double), + "float" => Ok(Schema::Float), + "bytes" => Ok(Schema::Bytes), + "string" => Ok(Schema::String), + _ => self.fetch_schema_ref(name, enclosing_namespace), + } + } + + /// Given a name, tries to retrieve the parsed schema from `parsed_schemas`. + /// If a parsed schema is not found, it checks if a currently resolving + /// schema with that name exists. + /// If a resolving schema is not found, it checks if a json with that name exists + /// in `input_schemas` and then parses it (removing it from `input_schemas`) + /// and adds the parsed schema to `parsed_schemas`. + /// + /// This method allows schemas definitions that depend on other types to + /// parse their dependencies (or look them up if already parsed). + fn fetch_schema_ref( + &mut self, + name: &str, + enclosing_namespace: &Namespace, + ) -> AvroResult { + fn get_schema_ref(parsed: &Schema) -> Schema { + match parsed { + &Schema::Record(RecordSchema { ref name, .. }) + | &Schema::Enum(EnumSchema { ref name, .. }) + | &Schema::Fixed(FixedSchema { ref name, .. }) => { + Schema::Ref { name: name.clone() } + } + _ => parsed.clone(), + } + } - if self.parsed_schemas.contains_key(&fully_qualified_name) { - return Ok(Schema::Ref { - name: fully_qualified_name, - }); - } - if let Some(resolving_schema) = self.resolving_schemas.get(&fully_qualified_name) { - return Ok(resolving_schema.clone()); - } + let name = Name::new(name)?; + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - // For good error reporting we add this check - match name.name.as_str() { - "record" | "enum" | "fixed" => { - return Err(Details::InvalidSchemaRecord(name.to_string()).into()); + if self.parsed_schemas.contains_key(&fully_qualified_name) { + return Ok(Schema::Ref { + name: fully_qualified_name, + }); + } + if let Some(resolving_schema) = self.resolving_schemas.get(&fully_qualified_name) { + return Ok(resolving_schema.clone()); } - _ => (), - } - let value = self - .input_schemas - .remove(&fully_qualified_name) - // TODO make a better descriptive error message here that conveys that a named schema cannot be found - .ok_or_else(|| Details::ParsePrimitive(fully_qualified_name.fullname(None)))?; + // For good error reporting we add this check + match name.name.as_str() { + "record" | "enum" | "fixed" => { + return Err(Details::InvalidSchemaRecord(name.to_string()).into()); + } + _ => (), + } + + let value = self + .input_schemas + .remove(&fully_qualified_name) + // TODO make a better descriptive error message here that conveys that a named schema cannot be found + .ok_or_else(|| Details::ParsePrimitive(fully_qualified_name.fullname(None)))?; - // parsing a full schema from inside another schema. Other full schema will not inherit namespace - let parsed = self.parse(&value, &None)?; - self.parsed_schemas - .insert(get_schema_type_name(name, value), parsed.clone()); + // parsing a full schema from inside another schema. Other full schema will not inherit namespace + let parsed = self.parse(&value, &None)?; + self.parsed_schemas + .insert(get_schema_type_name(name, value), parsed.clone()); - Ok(get_schema_ref(&parsed)) - } + Ok(get_schema_ref(&parsed)) + } - fn parse_precision_and_scale( - complex: &Map, - ) -> Result<(Precision, Scale), Error> { - fn get_decimal_integer( + fn parse_precision_and_scale( complex: &Map, - key: &'static str, - ) -> Result { - match complex.get(key) { - Some(Value::Number(value)) => parse_json_integer_for_decimal(value), - None => { - if key == "scale" { - Ok(0) - } else { - Err(Details::GetDecimalMetadataFromJson(key).into()) + ) -> Result<(Precision, Scale), Error> { + fn get_decimal_integer( + complex: &Map, + key: &'static str, + ) -> Result { + match complex.get(key) { + Some(Value::Number(value)) => parse_json_integer_for_decimal(value), + None => { + if key == "scale" { + Ok(0) + } else { + Err(Details::GetDecimalMetadataFromJson(key).into()) + } } + Some(value) => Err(Details::GetDecimalMetadataValueFromJson { + key: key.into(), + value: value.clone(), + } + .into()), } - Some(value) => Err(Details::GetDecimalMetadataValueFromJson { - key: key.into(), - value: value.clone(), - } - .into()), } - } - let precision = get_decimal_integer(complex, "precision")?; - let scale = get_decimal_integer(complex, "scale")?; + let precision = get_decimal_integer(complex, "precision")?; + let scale = get_decimal_integer(complex, "scale")?; - if precision < 1 { - return Err(Details::DecimalPrecisionMuBePositive { precision }.into()); - } + if precision < 1 { + return Err(Details::DecimalPrecisionMuBePositive { precision }.into()); + } - if precision < scale { - Err(Details::DecimalPrecisionLessThanScale { precision, scale }.into()) - } else { - Ok((precision, scale)) + if precision < scale { + Err(Details::DecimalPrecisionLessThanScale { precision, scale }.into()) + } else { + Ok((precision, scale)) + } } - } - /// Parse a `serde_json::Value` representing a complex Avro type into a - /// `Schema`. - /// - /// Avro supports "recursive" definition of types. - /// e.g: {"type": {"type": "string"}} - fn parse_complex( - &mut self, - complex: &Map, - enclosing_namespace: &Namespace, - parse_location: RecordSchemaParseLocation, - ) -> AvroResult { - // Try to parse this as a native complex type. - fn parse_as_native_complex( + /// Parse a `serde_json::Value` representing a complex Avro type into a + /// `Schema`. + /// + /// Avro supports "recursive" definition of types. + /// e.g: {"type": {"type": "string"}} + fn parse_complex( + &mut self, complex: &Map, - parser: &mut Parser, enclosing_namespace: &Namespace, + parse_location: RecordSchemaParseLocation, ) -> AvroResult { - match complex.get("type") { - Some(value) => match value { - Value::String(s) if s == "fixed" => { - parser.parse_fixed(complex, enclosing_namespace) - } - _ => parser.parse(value, enclosing_namespace), - }, - None => Err(Details::GetLogicalTypeField.into()), - } - } - - // This crate support some logical types natively, and this function tries to convert - // a native complex type with a logical type attribute to these logical types. - // This function: - // 1. Checks whether the native complex type is in the supported kinds. - // 2. If it is, using the convert function to convert the native complex type to - // a logical type. - fn try_convert_to_logical_type( - logical_type: &str, - schema: Schema, - supported_schema_kinds: &[SchemaKind], - convert: F, - ) -> AvroResult - where - F: Fn(Schema) -> AvroResult, - { - let kind = SchemaKind::from(schema.clone()); - if supported_schema_kinds.contains(&kind) { - convert(schema) - } else { - warn!( - "Ignoring unknown logical type '{logical_type}' for schema of type: {schema:?}!" - ); - Ok(schema) - } - } - - match complex.get("logicalType") { - Some(Value::String(t)) => match t.as_str() { - "decimal" => { - return try_convert_to_logical_type( - "decimal", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Fixed, SchemaKind::Bytes], - |inner| -> AvroResult { - match Self::parse_precision_and_scale(complex) { - Ok((precision, scale)) => Ok(Schema::Decimal(DecimalSchema { - precision, - scale, - inner: Box::new(inner), - })), - Err(err) => { - warn!("Ignoring invalid decimal logical type: {err}"); - Ok(inner) - } - } - }, - ); - } - "big-decimal" => { - return try_convert_to_logical_type( - "big-decimal", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Bytes], - |_| -> AvroResult { Ok(Schema::BigDecimal) }, - ); - } - "uuid" => { - return try_convert_to_logical_type( - "uuid", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::String, SchemaKind::Fixed], - |schema| match schema { - Schema::String => Ok(Schema::Uuid), - Schema::Fixed(FixedSchema { size: 16, .. }) => Ok(Schema::Uuid), - Schema::Fixed(FixedSchema { size, .. }) => { - warn!( - "Ignoring uuid logical type for a Fixed schema because its size ({size:?}) is not 16! Schema: {schema:?}" - ); - Ok(schema) - } - _ => { - warn!("Ignoring invalid uuid logical type for schema: {schema:?}"); - Ok(schema) - } - }, - ); - } - "date" => { - return try_convert_to_logical_type( - "date", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Int], - |_| -> AvroResult { Ok(Schema::Date) }, - ); - } - "time-millis" => { - return try_convert_to_logical_type( - "date", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Int], - |_| -> AvroResult { Ok(Schema::TimeMillis) }, - ); - } - "time-micros" => { - return try_convert_to_logical_type( - "time-micros", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::TimeMicros) }, - ); - } - "timestamp-millis" => { - return try_convert_to_logical_type( - "timestamp-millis", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::TimestampMillis) }, - ); - } - "timestamp-micros" => { - return try_convert_to_logical_type( - "timestamp-micros", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::TimestampMicros) }, - ); - } - "timestamp-nanos" => { - return try_convert_to_logical_type( - "timestamp-nanos", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::TimestampNanos) }, - ); - } - "local-timestamp-millis" => { - return try_convert_to_logical_type( - "local-timestamp-millis", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::LocalTimestampMillis) }, - ); - } - "local-timestamp-micros" => { - return try_convert_to_logical_type( - "local-timestamp-micros", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::LocalTimestampMicros) }, - ); - } - "local-timestamp-nanos" => { - return try_convert_to_logical_type( - "local-timestamp-nanos", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Long], - |_| -> AvroResult { Ok(Schema::LocalTimestampNanos) }, - ); + // Try to parse this as a native complex type. + fn parse_as_native_complex( + complex: &Map, + parser: &mut Parser, + enclosing_namespace: &Namespace, + ) -> AvroResult { + match complex.get("type") { + Some(value) => match value { + Value::String(s) if s == "fixed" => { + parser.parse_fixed(complex, enclosing_namespace) + } + _ => parser.parse(value, enclosing_namespace), + }, + None => Err(Details::GetLogicalTypeField.into()), } - "duration" => { - return try_convert_to_logical_type( - "duration", - parse_as_native_complex(complex, self, enclosing_namespace)?, - &[SchemaKind::Fixed], - |_| -> AvroResult { Ok(Schema::Duration) }, + } + + // This crate support some logical types natively, and this function tries to convert + // a native complex type with a logical type attribute to these logical types. + // This function: + // 1. Checks whether the native complex type is in the supported kinds. + // 2. If it is, using the convert function to convert the native complex type to + // a logical type. + fn try_convert_to_logical_type( + logical_type: &str, + schema: Schema, + supported_schema_kinds: &[SchemaKind], + convert: F, + ) -> AvroResult + where + F: Fn(Schema) -> AvroResult, + { + let kind = SchemaKind::from(schema.clone()); + if supported_schema_kinds.contains(&kind) { + convert(schema) + } else { + warn!( + "Ignoring unknown logical type '{logical_type}' for schema of type: {schema:?}!" ); + Ok(schema) } - // In this case, of an unknown logical type, we just pass through the underlying - // type. - _ => {} - }, - // The spec says to ignore invalid logical types and just pass through the - // underlying type. It is unclear whether that applies to this case or not, where the - // `logicalType` is not a string. - Some(value) => return Err(Details::GetLogicalTypeFieldType(value.clone()).into()), - _ => {} - } - match complex.get("type") { - Some(Value::String(t)) => match t.as_str() { - "record" => match parse_location { - RecordSchemaParseLocation::Root => { - self.parse_record(complex, enclosing_namespace) + } + + match complex.get("logicalType") { + Some(Value::String(t)) => match t.as_str() { + "decimal" => { + return try_convert_to_logical_type( + "decimal", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Fixed, SchemaKind::Bytes], + |inner| -> AvroResult { + match Self::parse_precision_and_scale(complex) { + Ok((precision, scale)) => Ok(Schema::Decimal(DecimalSchema { + precision, + scale, + inner: Box::new(inner), + })), + Err(err) => { + warn!("Ignoring invalid decimal logical type: {err}"); + Ok(inner) + } + } + }, + ); + } + "big-decimal" => { + return try_convert_to_logical_type( + "big-decimal", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Bytes], + |_| -> AvroResult { Ok(Schema::BigDecimal) }, + ); + } + "uuid" => { + return try_convert_to_logical_type( + "uuid", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::String, SchemaKind::Fixed], + |schema| match schema { + Schema::String => Ok(Schema::Uuid), + Schema::Fixed(FixedSchema { size: 16, .. }) => Ok(Schema::Uuid), + Schema::Fixed(FixedSchema { size, .. }) => { + warn!( + "Ignoring uuid logical type for a Fixed schema because its size ({size:?}) is not 16! Schema: {schema:?}" + ); + Ok(schema) + } + _ => { + warn!( + "Ignoring invalid uuid logical type for schema: {schema:?}" + ); + Ok(schema) + } + }, + ); + } + "date" => { + return try_convert_to_logical_type( + "date", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Int], + |_| -> AvroResult { Ok(Schema::Date) }, + ); } - RecordSchemaParseLocation::FromField => { - self.fetch_schema_ref(t, enclosing_namespace) + "time-millis" => { + return try_convert_to_logical_type( + "date", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Int], + |_| -> AvroResult { Ok(Schema::TimeMillis) }, + ); } + "time-micros" => { + return try_convert_to_logical_type( + "time-micros", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::TimeMicros) }, + ); + } + "timestamp-millis" => { + return try_convert_to_logical_type( + "timestamp-millis", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::TimestampMillis) }, + ); + } + "timestamp-micros" => { + return try_convert_to_logical_type( + "timestamp-micros", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::TimestampMicros) }, + ); + } + "timestamp-nanos" => { + return try_convert_to_logical_type( + "timestamp-nanos", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::TimestampNanos) }, + ); + } + "local-timestamp-millis" => { + return try_convert_to_logical_type( + "local-timestamp-millis", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::LocalTimestampMillis) }, + ); + } + "local-timestamp-micros" => { + return try_convert_to_logical_type( + "local-timestamp-micros", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::LocalTimestampMicros) }, + ); + } + "local-timestamp-nanos" => { + return try_convert_to_logical_type( + "local-timestamp-nanos", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Long], + |_| -> AvroResult { Ok(Schema::LocalTimestampNanos) }, + ); + } + "duration" => { + return try_convert_to_logical_type( + "duration", + parse_as_native_complex(complex, self, enclosing_namespace)?, + &[SchemaKind::Fixed], + |_| -> AvroResult { Ok(Schema::Duration) }, + ); + } + // In this case, of an unknown logical type, we just pass through the underlying + // type. + _ => {} }, - "enum" => self.parse_enum(complex, enclosing_namespace), - "array" => self.parse_array(complex, enclosing_namespace), - "map" => self.parse_map(complex, enclosing_namespace), - "fixed" => self.parse_fixed(complex, enclosing_namespace), - other => self.parse_known_schema(other, enclosing_namespace), - }, - Some(Value::Object(data)) => { - self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) + // The spec says to ignore invalid logical types and just pass through the + // underlying type. It is unclear whether that applies to this case or not, where the + // `logicalType` is not a string. + Some(value) => return Err(Details::GetLogicalTypeFieldType(value.clone()).into()), + _ => {} + } + match complex.get("type") { + Some(Value::String(t)) => match t.as_str() { + "record" => match parse_location { + RecordSchemaParseLocation::Root => { + self.parse_record(complex, enclosing_namespace) + } + RecordSchemaParseLocation::FromField => { + self.fetch_schema_ref(t, enclosing_namespace) + } + }, + "enum" => self.parse_enum(complex, enclosing_namespace), + "array" => self.parse_array(complex, enclosing_namespace), + "map" => self.parse_map(complex, enclosing_namespace), + "fixed" => self.parse_fixed(complex, enclosing_namespace), + other => self.parse_known_schema(other, enclosing_namespace), + }, + Some(Value::Object(data)) => { + self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) + } + Some(Value::Array(variants)) => self.parse_union(variants, enclosing_namespace), + Some(unknown) => Err(Details::GetComplexType(unknown.clone()).into()), + None => Err(Details::GetComplexTypeField.into()), } - Some(Value::Array(variants)) => self.parse_union(variants, enclosing_namespace), - Some(unknown) => Err(Details::GetComplexType(unknown.clone()).into()), - None => Err(Details::GetComplexTypeField.into()), } - } - fn register_resolving_schema(&mut self, name: &Name, aliases: &Aliases) { - let resolving_schema = Schema::Ref { name: name.clone() }; - self.resolving_schemas - .insert(name.clone(), resolving_schema.clone()); + fn register_resolving_schema(&mut self, name: &Name, aliases: &Aliases) { + let resolving_schema = Schema::Ref { name: name.clone() }; + self.resolving_schemas + .insert(name.clone(), resolving_schema.clone()); - let namespace = &name.namespace; + let namespace = &name.namespace; - if let Some(aliases) = aliases { - aliases.iter().for_each(|alias| { - let alias_fullname = alias.fully_qualified_name(namespace); - self.resolving_schemas - .insert(alias_fullname, resolving_schema.clone()); - }); + if let Some(aliases) = aliases { + aliases.iter().for_each(|alias| { + let alias_fullname = alias.fully_qualified_name(namespace); + self.resolving_schemas + .insert(alias_fullname, resolving_schema.clone()); + }); + } } - } - fn register_parsed_schema( - &mut self, - fully_qualified_name: &Name, - schema: &Schema, - aliases: &Aliases, - ) { - // FIXME, this should be globally aware, so if there is something overwriting something - // else then there is an ambiguous schema definition. An appropriate error should be thrown - self.parsed_schemas - .insert(fully_qualified_name.clone(), schema.clone()); - self.resolving_schemas.remove(fully_qualified_name); - - let namespace = &fully_qualified_name.namespace; - - if let Some(aliases) = aliases { - aliases.iter().for_each(|alias| { - let alias_fullname = alias.fully_qualified_name(namespace); - self.resolving_schemas.remove(&alias_fullname); - self.parsed_schemas.insert(alias_fullname, schema.clone()); - }); - } - } + fn register_parsed_schema( + &mut self, + fully_qualified_name: &Name, + schema: &Schema, + aliases: &Aliases, + ) { + // FIXME, this should be globally aware, so if there is something overwriting something + // else then there is an ambiguous schema definition. An appropriate error should be thrown + self.parsed_schemas + .insert(fully_qualified_name.clone(), schema.clone()); + self.resolving_schemas.remove(fully_qualified_name); - /// Returns already parsed schema or a schema that is currently being resolved. - fn get_already_seen_schema( - &self, - complex: &Map, - enclosing_namespace: &Namespace, - ) -> Option<&Schema> { - match complex.get("type") { - Some(Value::String(typ)) => { - let name = Name::new(typ.as_str()) - .unwrap() - .fully_qualified_name(enclosing_namespace); - self.resolving_schemas - .get(&name) - .or_else(|| self.parsed_schemas.get(&name)) - } - _ => None, - } - } + let namespace = &fully_qualified_name.namespace; - /// Parse a `serde_json::Value` representing a Avro record type into a - /// `Schema`. - fn parse_record( - &mut self, - complex: &Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - let fields_opt = complex.get("fields"); + if let Some(aliases) = aliases { + aliases.iter().for_each(|alias| { + let alias_fullname = alias.fully_qualified_name(namespace); + self.resolving_schemas.remove(&alias_fullname); + self.parsed_schemas.insert(alias_fullname, schema.clone()); + }); + } + } - if fields_opt.is_none() { - if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) { - return Ok(seen.clone()); + /// Returns already parsed schema or a schema that is currently being resolved. + fn get_already_seen_schema( + &self, + complex: &Map, + enclosing_namespace: &Namespace, + ) -> Option<&Schema> { + match complex.get("type") { + Some(Value::String(typ)) => { + let name = Name::new(typ.as_str()) + .unwrap() + .fully_qualified_name(enclosing_namespace); + self.resolving_schemas + .get(&name) + .or_else(|| self.parsed_schemas.get(&name)) + } + _ => None, } } - let fully_qualified_name = Name::parse(complex, enclosing_namespace)?; - let aliases = fix_aliases_namespace(complex.aliases(), &fully_qualified_name.namespace); + /// Parse a `serde_json::Value` representing a Avro record type into a + /// `Schema`. + fn parse_record( + &mut self, + complex: &Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + let fields_opt = complex.get("fields"); - let mut lookup = BTreeMap::new(); + if fields_opt.is_none() { + if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) { + return Ok(seen.clone()); + } + } - self.register_resolving_schema(&fully_qualified_name, &aliases); + let fully_qualified_name = Name::parse(complex, enclosing_namespace)?; + let aliases = fix_aliases_namespace(complex.aliases(), &fully_qualified_name.namespace); - debug!("Going to parse record schema: {:?}", &fully_qualified_name); + let mut lookup = BTreeMap::new(); - let fields: Vec = fields_opt - .and_then(|fields| fields.as_array()) - .ok_or_else(|| Error::new(Details::GetRecordFieldsJson)) - .and_then(|fields| { - fields - .iter() - .filter_map(|field| field.as_object()) - .enumerate() - .map(|(position, field)| { - RecordField::parse(field, position, self, &fully_qualified_name) - }) - .collect::>() - })?; + self.register_resolving_schema(&fully_qualified_name, &aliases); - for field in &fields { - if let Some(_old) = lookup.insert(field.name.clone(), field.position) { - return Err(Details::FieldNameDuplicate(field.name.clone()).into()); - } + debug!("Going to parse record schema: {:?}", &fully_qualified_name); - if let Some(ref field_aliases) = field.aliases { - for alias in field_aliases { - lookup.insert(alias.clone(), field.position); + let fields: Vec = fields_opt + .and_then(|fields| fields.as_array()) + .ok_or_else(|| Error::new(Details::GetRecordFieldsJson)) + .and_then(|fields| { + fields + .iter() + .filter_map(|field| field.as_object()) + .enumerate() + .map(|(position, field)| { + RecordField::parse(field, position, self, &fully_qualified_name) + }) + .collect::>() + })?; + + for field in &fields { + if let Some(_old) = lookup.insert(field.name.clone(), field.position) { + return Err(Details::FieldNameDuplicate(field.name.clone()).into()); } - } - } - let schema = Schema::Record(RecordSchema { - name: fully_qualified_name.clone(), - aliases: aliases.clone(), - doc: complex.doc(), - fields, - lookup, - attributes: self.get_custom_attributes(complex, vec!["fields"]), - }); + if let Some(ref field_aliases) = field.aliases { + for alias in field_aliases { + lookup.insert(alias.clone(), field.position); + } + } + } - self.register_parsed_schema(&fully_qualified_name, &schema, &aliases); - Ok(schema) - } + let schema = Schema::Record(RecordSchema { + name: fully_qualified_name.clone(), + aliases: aliases.clone(), + doc: complex.doc(), + fields, + lookup, + attributes: self.get_custom_attributes(complex, vec!["fields"]), + }); - fn get_custom_attributes( - &self, - complex: &Map, - excluded: Vec<&'static str>, - ) -> BTreeMap { - let mut custom_attributes: BTreeMap = BTreeMap::new(); - for (key, value) in complex { - match key.as_str() { - "type" | "name" | "namespace" | "doc" | "aliases" => continue, - candidate if excluded.contains(&candidate) => continue, - _ => custom_attributes.insert(key.clone(), value.clone()), - }; + self.register_parsed_schema(&fully_qualified_name, &schema, &aliases); + Ok(schema) } - custom_attributes - } - - /// Parse a `serde_json::Value` representing a Avro enum type into a - /// `Schema`. - fn parse_enum( - &mut self, - complex: &Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - let symbols_opt = complex.get("symbols"); - if symbols_opt.is_none() { - if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) { - return Ok(seen.clone()); + fn get_custom_attributes( + &self, + complex: &Map, + excluded: Vec<&'static str>, + ) -> BTreeMap { + let mut custom_attributes: BTreeMap = BTreeMap::new(); + for (key, value) in complex { + match key.as_str() { + "type" | "name" | "namespace" | "doc" | "aliases" => continue, + candidate if excluded.contains(&candidate) => continue, + _ => custom_attributes.insert(key.clone(), value.clone()), + }; } + custom_attributes } - let fully_qualified_name = Name::parse(complex, enclosing_namespace)?; - let aliases = fix_aliases_namespace(complex.aliases(), &fully_qualified_name.namespace); + /// Parse a `serde_json::Value` representing a Avro enum type into a + /// `Schema`. + fn parse_enum( + &mut self, + complex: &Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + let symbols_opt = complex.get("symbols"); + + if symbols_opt.is_none() { + if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) { + return Ok(seen.clone()); + } + } - let symbols: Vec = symbols_opt - .and_then(|v| v.as_array()) - .ok_or(Details::GetEnumSymbolsField) - .and_then(|symbols| { - symbols - .iter() - .map(|symbol| symbol.as_str().map(|s| s.to_string())) - .collect::>() - .ok_or(Details::GetEnumSymbols) - })?; + let fully_qualified_name = Name::parse(complex, enclosing_namespace)?; + let aliases = fix_aliases_namespace(complex.aliases(), &fully_qualified_name.namespace); - let mut existing_symbols: HashSet<&String> = HashSet::with_capacity(symbols.len()); - for symbol in symbols.iter() { - validate_enum_symbol_name(symbol)?; + let symbols: Vec = symbols_opt + .and_then(|v| v.as_array()) + .ok_or(Details::GetEnumSymbolsField) + .and_then(|symbols| { + symbols + .iter() + .map(|symbol| symbol.as_str().map(|s| s.to_string())) + .collect::>() + .ok_or(Details::GetEnumSymbols) + })?; + + let mut existing_symbols: HashSet<&String> = HashSet::with_capacity(symbols.len()); + for symbol in symbols.iter() { + validate_enum_symbol_name(symbol)?; + + // Ensure there are no duplicate symbols + if existing_symbols.contains(&symbol) { + return Err(Details::EnumSymbolDuplicate(symbol.to_string()).into()); + } - // Ensure there are no duplicate symbols - if existing_symbols.contains(&symbol) { - return Err(Details::EnumSymbolDuplicate(symbol.to_string()).into()); + existing_symbols.insert(symbol); } - existing_symbols.insert(symbol); - } - - let mut default: Option = None; - if let Some(value) = complex.get("default") { - if let Value::String(ref s) = *value { - default = Some(s.clone()); - } else { - return Err(Details::EnumDefaultWrongType(value.clone()).into()); + let mut default: Option = None; + if let Some(value) = complex.get("default") { + if let Value::String(ref s) = *value { + default = Some(s.clone()); + } else { + return Err(Details::EnumDefaultWrongType(value.clone()).into()); + } } - } - if let Some(ref value) = default { - let resolved = types::Value::from(value.clone()) - .resolve_enum(&symbols, &Some(value.to_string()), &None) - .is_ok(); - if !resolved { - return Err(Details::GetEnumDefault { - symbol: value.to_string(), - symbols, + if let Some(ref value) = default { + let resolved = types::Value::from(value.clone()) + .resolve_enum(&symbols, &Some(value.to_string()), &None) + .is_ok(); + if !resolved { + return Err(Details::GetEnumDefault { + symbol: value.to_string(), + symbols, + } + .into()); } - .into()); } - } - let schema = Schema::Enum(EnumSchema { - name: fully_qualified_name.clone(), - aliases: aliases.clone(), - doc: complex.doc(), - symbols, - default, - attributes: self.get_custom_attributes(complex, vec!["symbols"]), - }); + let schema = Schema::Enum(EnumSchema { + name: fully_qualified_name.clone(), + aliases: aliases.clone(), + doc: complex.doc(), + symbols, + default, + attributes: self.get_custom_attributes(complex, vec!["symbols"]), + }); - self.register_parsed_schema(&fully_qualified_name, &schema, &aliases); + self.register_parsed_schema(&fully_qualified_name, &schema, &aliases); - Ok(schema) - } + Ok(schema) + } - /// Parse a `serde_json::Value` representing a Avro array type into a - /// `Schema`. - fn parse_array( - &mut self, - complex: &Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - complex - .get("items") - .ok_or_else(|| Details::GetArrayItemsField.into()) - .and_then(|items| self.parse(items, enclosing_namespace)) - .map(|items| { - Schema::array_with_attributes( - items, - self.get_custom_attributes(complex, vec!["items"]), - ) - }) - } + /// Parse a `serde_json::Value` representing a Avro array type into a + /// `Schema`. + fn parse_array( + &mut self, + complex: &Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + complex + .get("items") + .ok_or_else(|| Details::GetArrayItemsField.into()) + .and_then(|items| self.parse(items, enclosing_namespace)) + .map(|items| { + Schema::array_with_attributes( + items, + self.get_custom_attributes(complex, vec!["items"]), + ) + }) + } - /// Parse a `serde_json::Value` representing a Avro map type into a - /// `Schema`. - fn parse_map( - &mut self, - complex: &Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - complex - .get("values") - .ok_or_else(|| Details::GetMapValuesField.into()) - .and_then(|items| self.parse(items, enclosing_namespace)) - .map(|items| { - Schema::map_with_attributes( - items, - self.get_custom_attributes(complex, vec!["values"]), - ) - }) - } + /// Parse a `serde_json::Value` representing a Avro map type into a + /// `Schema`. + fn parse_map( + &mut self, + complex: &Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + complex + .get("values") + .ok_or_else(|| Details::GetMapValuesField.into()) + .and_then(|items| self.parse(items, enclosing_namespace)) + .map(|items| { + Schema::map_with_attributes( + items, + self.get_custom_attributes(complex, vec!["values"]), + ) + }) + } - /// Parse a `serde_json::Value` representing a Avro union type into a - /// `Schema`. - fn parse_union( - &mut self, - items: &[Value], - enclosing_namespace: &Namespace, - ) -> AvroResult { - items - .iter() - .map(|v| self.parse(v, enclosing_namespace)) - .collect::, _>>() - .and_then(|schemas| { - if schemas.is_empty() { - error!( - "Union schemas should have at least two members! \ + /// Parse a `serde_json::Value` representing a Avro union type into a + /// `Schema`. + fn parse_union( + &mut self, + items: &[Value], + enclosing_namespace: &Namespace, + ) -> AvroResult { + items + .iter() + .map(|v| self.parse(v, enclosing_namespace)) + .collect::, _>>() + .and_then(|schemas| { + if schemas.is_empty() { + error!( + "Union schemas should have at least two members! \ Please enable debug logging to find out which Record schema \ declares the union with 'RUST_LOG=apache_avro::schema=debug'." - ); - } else if schemas.len() == 1 { - warn!( - "Union schema with just one member! Consider dropping the union! \ + ); + } else if schemas.len() == 1 { + warn!( + "Union schema with just one member! Consider dropping the union! \ Please enable debug logging to find out which Record schema \ declares the union with 'RUST_LOG=apache_avro::schema=debug'." - ); - } - Ok(Schema::Union(UnionSchema::new(schemas)?)) - }) - } + ); + } + Ok(Schema::Union(UnionSchema::new(schemas)?)) + }) + } - /// Parse a `serde_json::Value` representing a Avro fixed type into a - /// `Schema`. - fn parse_fixed( - &mut self, - complex: &Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - let size_opt = complex.get("size"); - if size_opt.is_none() { - if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) { - return Ok(seen.clone()); + /// Parse a `serde_json::Value` representing a Avro fixed type into a + /// `Schema`. + fn parse_fixed( + &mut self, + complex: &Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + let size_opt = complex.get("size"); + if size_opt.is_none() { + if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) { + return Ok(seen.clone()); + } } - } - let doc = complex.get("doc").and_then(|v| match &v { - &Value::String(docstr) => Some(docstr.clone()), - _ => None, - }); + let doc = complex.get("doc").and_then(|v| match &v { + &Value::String(docstr) => Some(docstr.clone()), + _ => None, + }); - let size = match size_opt { - Some(size) => size - .as_u64() - .ok_or_else(|| Details::GetFixedSizeFieldPositive(size.clone())), - None => Err(Details::GetFixedSizeField), - }?; + let size = match size_opt { + Some(size) => size + .as_u64() + .ok_or_else(|| Details::GetFixedSizeFieldPositive(size.clone())), + None => Err(Details::GetFixedSizeField), + }?; - let default = complex.get("default").and_then(|v| match &v { - &Value::String(default) => Some(default.clone()), - _ => None, - }); + let default = complex.get("default").and_then(|v| match &v { + &Value::String(default) => Some(default.clone()), + _ => None, + }); - if default.is_some() { - let len = default.clone().unwrap().len(); - if len != size as usize { - return Err(Details::FixedDefaultLenSizeMismatch(len, size).into()); + if default.is_some() { + let len = default.clone().unwrap().len(); + if len != size as usize { + return Err(Details::FixedDefaultLenSizeMismatch(len, size).into()); + } } - } - let fully_qualified_name = Name::parse(complex, enclosing_namespace)?; - let aliases = fix_aliases_namespace(complex.aliases(), &fully_qualified_name.namespace); + let fully_qualified_name = Name::parse(complex, enclosing_namespace)?; + let aliases = fix_aliases_namespace(complex.aliases(), &fully_qualified_name.namespace); - let schema = Schema::Fixed(FixedSchema { - name: fully_qualified_name.clone(), - aliases: aliases.clone(), - doc, - size: size as usize, - default, - attributes: self.get_custom_attributes(complex, vec!["size"]), - }); + let schema = Schema::Fixed(FixedSchema { + name: fully_qualified_name.clone(), + aliases: aliases.clone(), + doc, + size: size as usize, + default, + attributes: self.get_custom_attributes(complex, vec!["size"]), + }); + + self.register_parsed_schema(&fully_qualified_name, &schema, &aliases); + + Ok(schema) + } + } - self.register_parsed_schema(&fully_qualified_name, &schema, &aliases); + // A type alias may be specified either as a fully namespace-qualified, or relative + // to the namespace of the name it is an alias for. For example, if a type named "a.b" + // has aliases of "c" and "x.y", then the fully qualified names of its aliases are "a.c" + // and "x.y". + // https://avro.apache.org/docs/current/specification/#aliases + fn fix_aliases_namespace(aliases: Option>, namespace: &Namespace) -> Aliases { + aliases.map(|aliases| { + aliases + .iter() + .map(|alias| { + if alias.find('.').is_none() { + match namespace { + Some(ns) => format!("{ns}.{alias}"), + None => alias.clone(), + } + } else { + alias.clone() + } + }) + .map(|alias| Alias::new(alias.as_str()).unwrap()) + .collect() + }) + } - Ok(schema) + fn get_schema_type_name(name: Name, value: Value) -> Name { + match value.get("type") { + Some(Value::Object(complex_type)) => match complex_type.name() { + Some(name) => Name::new(name.as_str()).unwrap(), + _ => name, + }, + _ => name, + } } -} -// A type alias may be specified either as a fully namespace-qualified, or relative -// to the namespace of the name it is an alias for. For example, if a type named "a.b" -// has aliases of "c" and "x.y", then the fully qualified names of its aliases are "a.c" -// and "x.y". -// https://avro.apache.org/docs/current/specification/#aliases -fn fix_aliases_namespace(aliases: Option>, namespace: &Namespace) -> Aliases { - aliases.map(|aliases| { - aliases - .iter() - .map(|alias| { - if alias.find('.').is_none() { - match namespace { - Some(ns) => format!("{ns}.{alias}"), - None => alias.clone(), + impl Serialize for Schema { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match *self { + Schema::Ref { ref name } => serializer.serialize_str(&name.fullname(None)), + Schema::Null => serializer.serialize_str("null"), + Schema::Boolean => serializer.serialize_str("boolean"), + Schema::Int => serializer.serialize_str("int"), + Schema::Long => serializer.serialize_str("long"), + Schema::Float => serializer.serialize_str("float"), + Schema::Double => serializer.serialize_str("double"), + Schema::Bytes => serializer.serialize_str("bytes"), + Schema::String => serializer.serialize_str("string"), + Schema::Array(ref inner) => { + let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; + map.serialize_entry("type", "array")?; + map.serialize_entry("items", &*inner.items.clone())?; + for attr in &inner.attributes { + map.serialize_entry(attr.0, attr.1)?; } - } else { - alias.clone() + map.end() } - }) - .map(|alias| Alias::new(alias.as_str()).unwrap()) - .collect() - }) -} + Schema::Map(ref inner) => { + let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; + map.serialize_entry("type", "map")?; + map.serialize_entry("values", &*inner.types.clone())?; + for attr in &inner.attributes { + map.serialize_entry(attr.0, attr.1)?; + } + map.end() + } + Schema::Union(ref inner) => { + let variants = inner.variants(); + let mut seq = serializer.serialize_seq(Some(variants.len()))?; + for v in variants { + seq.serialize_element(v)?; + } + seq.end() + } + Schema::Record(RecordSchema { + ref name, + ref aliases, + ref doc, + ref fields, + ref attributes, + .. + }) => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "record")?; + if let Some(ref n) = name.namespace { + map.serialize_entry("namespace", n)?; + } + map.serialize_entry("name", &name.name)?; + if let Some(docstr) = doc { + map.serialize_entry("doc", docstr)?; + } + if let Some(aliases) = aliases { + map.serialize_entry("aliases", aliases)?; + } + map.serialize_entry("fields", fields)?; + for attr in attributes { + map.serialize_entry(attr.0, attr.1)?; + } + map.end() + } + Schema::Enum(EnumSchema { + ref name, + ref symbols, + ref aliases, + ref attributes, + .. + }) => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "enum")?; + if let Some(ref n) = name.namespace { + map.serialize_entry("namespace", n)?; + } + map.serialize_entry("name", &name.name)?; + map.serialize_entry("symbols", symbols)?; -fn get_schema_type_name(name: Name, value: Value) -> Name { - match value.get("type") { - Some(Value::Object(complex_type)) => match complex_type.name() { - Some(name) => Name::new(name.as_str()).unwrap(), - _ => name, - }, - _ => name, - } -} + if let Some(aliases) = aliases { + map.serialize_entry("aliases", aliases)?; + } + for attr in attributes { + map.serialize_entry(attr.0, attr.1)?; + } + map.end() + } + Schema::Fixed(ref fixed_schema) => { + let mut map = serializer.serialize_map(None)?; + map = fixed_schema.serialize_to_map::(map)?; + map.end() + } + Schema::Decimal(DecimalSchema { + ref scale, + ref precision, + ref inner, + }) => { + let mut map = serializer.serialize_map(None)?; + match inner.as_ref() { + Schema::Fixed(fixed_schema) => { + map = fixed_schema.serialize_to_map::(map)?; + } + Schema::Bytes => { + map.serialize_entry("type", "bytes")?; + } + others => { + return Err(serde::ser::Error::custom(format!( + "DecimalSchema inner type must be Fixed or Bytes, got {:?}", + SchemaKind::from(others) + ))); + } + } + map.serialize_entry("logicalType", "decimal")?; + map.serialize_entry("scale", scale)?; + map.serialize_entry("precision", precision)?; + map.end() + } -impl Serialize for Schema { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match *self { - Schema::Ref { ref name } => serializer.serialize_str(&name.fullname(None)), - Schema::Null => serializer.serialize_str("null"), - Schema::Boolean => serializer.serialize_str("boolean"), - Schema::Int => serializer.serialize_str("int"), - Schema::Long => serializer.serialize_str("long"), - Schema::Float => serializer.serialize_str("float"), - Schema::Double => serializer.serialize_str("double"), - Schema::Bytes => serializer.serialize_str("bytes"), - Schema::String => serializer.serialize_str("string"), - Schema::Array(ref inner) => { - let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; - map.serialize_entry("type", "array")?; - map.serialize_entry("items", &*inner.items.clone())?; - for attr in &inner.attributes { - map.serialize_entry(attr.0, attr.1)?; + Schema::BigDecimal => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "bytes")?; + map.serialize_entry("logicalType", "big-decimal")?; + map.end() } - map.end() - } - Schema::Map(ref inner) => { - let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; - map.serialize_entry("type", "map")?; - map.serialize_entry("values", &*inner.types.clone())?; - for attr in &inner.attributes { - map.serialize_entry(attr.0, attr.1)?; + Schema::Uuid => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "string")?; + map.serialize_entry("logicalType", "uuid")?; + map.end() } - map.end() - } - Schema::Union(ref inner) => { - let variants = inner.variants(); - let mut seq = serializer.serialize_seq(Some(variants.len()))?; - for v in variants { - seq.serialize_element(v)?; + Schema::Date => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "int")?; + map.serialize_entry("logicalType", "date")?; + map.end() } - seq.end() - } - Schema::Record(RecordSchema { - ref name, - ref aliases, - ref doc, - ref fields, - ref attributes, - .. - }) => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "record")?; - if let Some(ref n) = name.namespace { - map.serialize_entry("namespace", n)?; + Schema::TimeMillis => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "int")?; + map.serialize_entry("logicalType", "time-millis")?; + map.end() } - map.serialize_entry("name", &name.name)?; - if let Some(docstr) = doc { - map.serialize_entry("doc", docstr)?; + Schema::TimeMicros => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "time-micros")?; + map.end() } - if let Some(aliases) = aliases { - map.serialize_entry("aliases", aliases)?; + Schema::TimestampMillis => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "timestamp-millis")?; + map.end() } - map.serialize_entry("fields", fields)?; - for attr in attributes { - map.serialize_entry(attr.0, attr.1)?; + Schema::TimestampMicros => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "timestamp-micros")?; + map.end() } - map.end() - } - Schema::Enum(EnumSchema { - ref name, - ref symbols, - ref aliases, - ref attributes, - .. - }) => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "enum")?; - if let Some(ref n) = name.namespace { - map.serialize_entry("namespace", n)?; + Schema::TimestampNanos => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "timestamp-nanos")?; + map.end() } - map.serialize_entry("name", &name.name)?; - map.serialize_entry("symbols", symbols)?; - - if let Some(aliases) = aliases { - map.serialize_entry("aliases", aliases)?; + Schema::LocalTimestampMillis => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "local-timestamp-millis")?; + map.end() } - for attr in attributes { - map.serialize_entry(attr.0, attr.1)?; + Schema::LocalTimestampMicros => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "local-timestamp-micros")?; + map.end() } - map.end() - } - Schema::Fixed(ref fixed_schema) => { - let mut map = serializer.serialize_map(None)?; - map = fixed_schema.serialize_to_map::(map)?; - map.end() - } - Schema::Decimal(DecimalSchema { - ref scale, - ref precision, - ref inner, - }) => { - let mut map = serializer.serialize_map(None)?; - match inner.as_ref() { - Schema::Fixed(fixed_schema) => { - map = fixed_schema.serialize_to_map::(map)?; - } - Schema::Bytes => { - map.serialize_entry("type", "bytes")?; - } - others => { - return Err(serde::ser::Error::custom(format!( - "DecimalSchema inner type must be Fixed or Bytes, got {:?}", - SchemaKind::from(others) - ))); - } + Schema::LocalTimestampNanos => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "local-timestamp-nanos")?; + map.end() + } + Schema::Duration => { + let mut map = serializer.serialize_map(None)?; + + // the Avro doesn't indicate what the name of the underlying fixed type of a + // duration should be or typically is. + let inner = Schema::Fixed(FixedSchema { + name: Name::new("duration").unwrap(), + aliases: None, + doc: None, + size: 12, + default: None, + attributes: Default::default(), + }); + map.serialize_entry("type", &inner)?; + map.serialize_entry("logicalType", "duration")?; + map.end() } - map.serialize_entry("logicalType", "decimal")?; - map.serialize_entry("scale", scale)?; - map.serialize_entry("precision", precision)?; - map.end() - } - - Schema::BigDecimal => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "bytes")?; - map.serialize_entry("logicalType", "big-decimal")?; - map.end() - } - Schema::Uuid => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "string")?; - map.serialize_entry("logicalType", "uuid")?; - map.end() - } - Schema::Date => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "int")?; - map.serialize_entry("logicalType", "date")?; - map.end() - } - Schema::TimeMillis => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "int")?; - map.serialize_entry("logicalType", "time-millis")?; - map.end() - } - Schema::TimeMicros => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "time-micros")?; - map.end() - } - Schema::TimestampMillis => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "timestamp-millis")?; - map.end() - } - Schema::TimestampMicros => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "timestamp-micros")?; - map.end() - } - Schema::TimestampNanos => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "timestamp-nanos")?; - map.end() - } - Schema::LocalTimestampMillis => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "local-timestamp-millis")?; - map.end() - } - Schema::LocalTimestampMicros => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "local-timestamp-micros")?; - map.end() - } - Schema::LocalTimestampNanos => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "local-timestamp-nanos")?; - map.end() - } - Schema::Duration => { - let mut map = serializer.serialize_map(None)?; - - // the Avro doesn't indicate what the name of the underlying fixed type of a - // duration should be or typically is. - let inner = Schema::Fixed(FixedSchema { - name: Name::new("duration").unwrap(), - aliases: None, - doc: None, - size: 12, - default: None, - attributes: Default::default(), - }); - map.serialize_entry("type", &inner)?; - map.serialize_entry("logicalType", "duration")?; - map.end() } } } -} -impl Serialize for RecordField { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("name", &self.name)?; - map.serialize_entry("type", &self.schema)?; - - if let Some(ref default) = self.default { - map.serialize_entry("default", default)?; - } + impl Serialize for RecordField { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("name", &self.name)?; + map.serialize_entry("type", &self.schema)?; - if let Some(ref aliases) = self.aliases { - map.serialize_entry("aliases", aliases)?; - } + if let Some(ref default) = self.default { + map.serialize_entry("default", default)?; + } - for attr in &self.custom_attributes { - map.serialize_entry(attr.0, attr.1)?; - } + if let Some(ref aliases) = self.aliases { + map.serialize_entry("aliases", aliases)?; + } - map.end() - } -} + for attr in &self.custom_attributes { + map.serialize_entry(attr.0, attr.1)?; + } -/// Parses a **valid** avro schema into the Parsing Canonical Form. -/// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas -fn parsing_canonical_form(schema: &Value, defined_names: &mut HashSet) -> String { - match schema { - Value::Object(map) => pcf_map(map, defined_names), - Value::String(s) => pcf_string(s), - Value::Array(v) => pcf_array(v, defined_names), - json => panic!("got invalid JSON value for canonical form of schema: {json}"), + map.end() + } } -} - -fn pcf_map(schema: &Map, defined_names: &mut HashSet) -> String { - // Look for the namespace variant up front. - let ns = schema.get("namespace").and_then(|v| v.as_str()); - let typ = schema.get("type").and_then(|v| v.as_str()); - let raw_name = schema.get("name").and_then(|v| v.as_str()); - let name = if is_named_type(typ) { - Some(format!( - "{}{}", - ns.map_or("".to_string(), |n| { format!("{n}.") }), - raw_name.unwrap_or_default() - )) - } else { - None - }; - //if this is already a defined type, early return - if let Some(ref n) = name { - if defined_names.contains(n) { - return pcf_string(n); + /// Parses a **valid** avro schema into the Parsing Canonical Form. + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + fn parsing_canonical_form(schema: &Value, defined_names: &mut HashSet) -> String { + match schema { + Value::Object(map) => pcf_map(map, defined_names), + Value::String(s) => pcf_string(s), + Value::Array(v) => pcf_array(v, defined_names), + json => panic!("got invalid JSON value for canonical form of schema: {json}"), + } + } + + fn pcf_map(schema: &Map, defined_names: &mut HashSet) -> String { + // Look for the namespace variant up front. + let ns = schema.get("namespace").and_then(|v| v.as_str()); + let typ = schema.get("type").and_then(|v| v.as_str()); + let raw_name = schema.get("name").and_then(|v| v.as_str()); + let name = if is_named_type(typ) { + Some(format!( + "{}{}", + ns.map_or("".to_string(), |n| { format!("{n}.") }), + raw_name.unwrap_or_default() + )) } else { - defined_names.insert(n.clone()); - } - } + None + }; - let mut fields = Vec::new(); - for (k, v) in schema { - // Reduce primitive types to their simple form. ([PRIMITIVE] rule) - if schema.len() == 1 && k == "type" { - // Invariant: function is only callable from a valid schema, so this is acceptable. - if let Value::String(s) = v { - return pcf_string(s); + //if this is already a defined type, early return + if let Some(ref n) = name { + if defined_names.contains(n) { + return pcf_string(n); + } else { + defined_names.insert(n.clone()); } } - // Strip out unused fields ([STRIP] rule) - if field_ordering_position(k).is_none() - || k == "default" - || k == "doc" - || k == "aliases" - || k == "logicalType" - { - continue; - } + let mut fields = Vec::new(); + for (k, v) in schema { + // Reduce primitive types to their simple form. ([PRIMITIVE] rule) + if schema.len() == 1 && k == "type" { + // Invariant: function is only callable from a valid schema, so this is acceptable. + if let Value::String(s) = v { + return pcf_string(s); + } + } - // Fully qualify the name, if it isn't already ([FULLNAMES] rule). - if k == "name" { - if let Some(ref n) = name { - fields.push(("name", format!("{}:{}", pcf_string(k), pcf_string(n)))); + // Strip out unused fields ([STRIP] rule) + if field_ordering_position(k).is_none() + || k == "default" + || k == "doc" + || k == "aliases" + || k == "logicalType" + { continue; } - } - - // Strip off quotes surrounding "size" type, if they exist ([INTEGERS] rule). - if k == "size" || k == "precision" || k == "scale" { - let i = match v.as_str() { - Some(s) => s.parse::().expect("Only valid schemas are accepted!"), - None => v.as_i64().unwrap(), - }; - fields.push((k, format!("{}:{}", pcf_string(k), i))); - continue; - } - - // For anything else, recursively process the result. - fields.push(( - k, - format!( - "{}:{}", - pcf_string(k), - parsing_canonical_form(v, defined_names) - ), - )); - } - - // Sort the fields by their canonical ordering ([ORDER] rule). - fields.sort_unstable_by_key(|(k, _)| field_ordering_position(k).unwrap()); - let inter = fields - .into_iter() - .map(|(_, v)| v) - .collect::>() - .join(","); - format!("{{{inter}}}") -} - -fn is_named_type(typ: Option<&str>) -> bool { - matches!( - typ, - Some("record") | Some("enum") | Some("fixed") | Some("ref") - ) -} - -fn pcf_array(arr: &[Value], defined_names: &mut HashSet) -> String { - let inter = arr - .iter() - .map(|a| parsing_canonical_form(a, defined_names)) - .collect::>() - .join(","); - format!("[{inter}]") -} -fn pcf_string(s: &str) -> String { - format!("\"{s}\"") -} + // Fully qualify the name, if it isn't already ([FULLNAMES] rule). + if k == "name" { + if let Some(ref n) = name { + fields.push(("name", format!("{}:{}", pcf_string(k), pcf_string(n)))); + continue; + } + } -const RESERVED_FIELDS: &[&str] = &[ - "name", - "type", - "fields", - "symbols", - "items", - "values", - "size", - "logicalType", - "order", - "doc", - "aliases", - "default", - "precision", - "scale", -]; - -// Used to define the ordering and inclusion of fields. -fn field_ordering_position(field: &str) -> Option { - RESERVED_FIELDS - .iter() - .position(|&f| f == field) - .map(|pos| pos + 1) -} + // Strip off quotes surrounding "size" type, if they exist ([INTEGERS] rule). + if k == "size" || k == "precision" || k == "scale" { + let i = match v.as_str() { + Some(s) => s.parse::().expect("Only valid schemas are accepted!"), + None => v.as_i64().unwrap(), + }; + fields.push((k, format!("{}:{}", pcf_string(k), i))); + continue; + } -/// Trait for types that serve as an Avro data model. Derive implementation available -/// through `derive` feature. Do not implement directly! -/// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait -/// through a blanket implementation. -pub trait AvroSchema { - fn get_schema() -> Schema; -} + // For anything else, recursively process the result. + fields.push(( + k, + format!( + "{}:{}", + pcf_string(k), + parsing_canonical_form(v, defined_names) + ), + )); + } -#[cfg(feature = "derive")] -pub mod derive { - use super::*; - use std::borrow::Cow; + // Sort the fields by their canonical ordering ([ORDER] rule). + fields.sort_unstable_by_key(|(k, _)| field_ordering_position(k).unwrap()); + let inter = fields + .into_iter() + .map(|(_, v)| v) + .collect::>() + .join(","); + format!("{{{inter}}}") + } - /// Trait for types that serve as fully defined components inside an Avro data model. Derive - /// implementation available through `derive` feature. This is what is implemented by - /// the `derive(AvroSchema)` macro. - /// - /// # Implementation guide - /// - ///### Simple implementation - /// To construct a non named simple schema, it is possible to ignore the input argument making the - /// general form implementation look like - /// ```ignore - /// impl AvroSchemaComponent for AType { - /// fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { - /// Schema::? - /// } - ///} - /// ``` - /// ### Passthrough implementation - /// To construct a schema for a Type that acts as in "inner" type, such as for smart pointers, simply - /// pass through the arguments to the inner type - /// ```ignore - /// impl AvroSchemaComponent for PassthroughType { - /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { - /// InnerType::get_schema_in_ctxt(names, enclosing_namespace) - /// } - ///} - /// ``` - ///### Complex implementation - /// To implement this for Named schema there is a general form needed to avoid creating invalid - /// schemas or infinite loops. - /// ```ignore - /// impl AvroSchemaComponent for ComplexType { - /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { - /// // Create the fully qualified name for your type given the enclosing namespace - /// let name = apache_avro::schema::Name::new("MyName") - /// .expect("Unable to parse schema name") - /// .fully_qualified_name(enclosing_namespace); - /// let enclosing_namespace = &name.namespace; - /// // Check, if your name is already defined, and if so, return a ref to that name - /// if named_schemas.contains_key(&name) { - /// apache_avro::schema::Schema::Ref{name: name.clone()} - /// } else { - /// named_schemas.insert(name.clone(), apache_avro::schema::Schema::Ref{name: name.clone()}); - /// // YOUR SCHEMA DEFINITION HERE with the name equivalent to "MyName". - /// // For non-simple sub types delegate to their implementation of AvroSchemaComponent - /// } - /// } - ///} - /// ``` - pub trait AvroSchemaComponent { - fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) - -> Schema; + fn is_named_type(typ: Option<&str>) -> bool { + matches!( + typ, + Some("record") | Some("enum") | Some("fixed") | Some("ref") + ) } - impl AvroSchema for T - where - T: AvroSchemaComponent, - { - fn get_schema() -> Schema { - T::get_schema_in_ctxt(&mut HashMap::default(), &None) + fn pcf_array(arr: &[Value], defined_names: &mut HashSet) -> String { + let inter = arr + .iter() + .map(|a| parsing_canonical_form(a, defined_names)) + .collect::>() + .join(","); + format!("[{inter}]") + } + + fn pcf_string(s: &str) -> String { + format!("\"{s}\"") + } + + const RESERVED_FIELDS: &[&str] = &[ + "name", + "type", + "fields", + "symbols", + "items", + "values", + "size", + "logicalType", + "order", + "doc", + "aliases", + "default", + "precision", + "scale", + ]; + + // Used to define the ordering and inclusion of fields. + fn field_ordering_position(field: &str) -> Option { + RESERVED_FIELDS + .iter() + .position(|&f| f == field) + .map(|pos| pos + 1) + } + + /// Trait for types that serve as an Avro data model. Derive implementation available + /// through `derive` feature. Do not implement directly! + /// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait + /// through a blanket implementation. + pub trait AvroSchema { + fn get_schema() -> Schema; + } + + #[cfg(feature = "derive")] + pub mod derive { + use super::*; + use std::borrow::Cow; + + /// Trait for types that serve as fully defined components inside an Avro data model. Derive + /// implementation available through `derive` feature. This is what is implemented by + /// the `derive(AvroSchema)` macro. + /// + /// # Implementation guide + /// + ///### Simple implementation + /// To construct a non named simple schema, it is possible to ignore the input argument making the + /// general form implementation look like + /// ```ignore + /// impl AvroSchemaComponent for AType { + /// fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { + /// Schema::? + /// } + ///} + /// ``` + /// ### Passthrough implementation + /// To construct a schema for a Type that acts as in "inner" type, such as for smart pointers, simply + /// pass through the arguments to the inner type + /// ```ignore + /// impl AvroSchemaComponent for PassthroughType { + /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { + /// InnerType::get_schema_in_ctxt(names, enclosing_namespace) + /// } + ///} + /// ``` + ///### Complex implementation + /// To implement this for Named schema there is a general form needed to avoid creating invalid + /// schemas or infinite loops. + /// ```ignore + /// impl AvroSchemaComponent for ComplexType { + /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { + /// // Create the fully qualified name for your type given the enclosing namespace + /// let name = apache_avro::schema::Name::new("MyName") + /// .expect("Unable to parse schema name") + /// .fully_qualified_name(enclosing_namespace); + /// let enclosing_namespace = &name.namespace; + /// // Check, if your name is already defined, and if so, return a ref to that name + /// if named_schemas.contains_key(&name) { + /// apache_avro::schema::Schema::Ref{name: name.clone()} + /// } else { + /// named_schemas.insert(name.clone(), apache_avro::schema::Schema::Ref{name: name.clone()}); + /// // YOUR SCHEMA DEFINITION HERE with the name equivalent to "MyName". + /// // For non-simple sub types delegate to their implementation of AvroSchemaComponent + /// } + /// } + ///} + /// ``` + pub trait AvroSchemaComponent { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema; + } + + impl AvroSchema for T + where + T: AvroSchemaComponent, + { + fn get_schema() -> Schema { + T::get_schema_in_ctxt(&mut HashMap::default(), &None) + } } - } - macro_rules! impl_schema ( + macro_rules! impl_schema ( ($type:ty, $variant_constructor:expr) => ( impl AvroSchemaComponent for $type { fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { @@ -2530,237 +2588,237 @@ pub mod derive { ); ); - impl_schema!(bool, Schema::Boolean); - impl_schema!(i8, Schema::Int); - impl_schema!(i16, Schema::Int); - impl_schema!(i32, Schema::Int); - impl_schema!(i64, Schema::Long); - impl_schema!(u8, Schema::Int); - impl_schema!(u16, Schema::Int); - impl_schema!(u32, Schema::Long); - impl_schema!(f32, Schema::Float); - impl_schema!(f64, Schema::Double); - impl_schema!(String, Schema::String); - impl_schema!(uuid::Uuid, Schema::Uuid); - impl_schema!(core::time::Duration, Schema::Duration); - - impl AvroSchemaComponent for Vec - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - Schema::array(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + impl_schema!(bool, Schema::Boolean); + impl_schema!(i8, Schema::Int); + impl_schema!(i16, Schema::Int); + impl_schema!(i32, Schema::Int); + impl_schema!(i64, Schema::Long); + impl_schema!(u8, Schema::Int); + impl_schema!(u16, Schema::Int); + impl_schema!(u32, Schema::Long); + impl_schema!(f32, Schema::Float); + impl_schema!(f64, Schema::Double); + impl_schema!(String, Schema::String); + impl_schema!(uuid::Uuid, Schema::Uuid); + impl_schema!(core::time::Duration, Schema::Duration); + + impl AvroSchemaComponent for Vec + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + Schema::array(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + } } - } - impl AvroSchemaComponent for Option - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - let inner_schema = T::get_schema_in_ctxt(named_schemas, enclosing_namespace); - Schema::Union(UnionSchema { - schemas: vec![Schema::Null, inner_schema.clone()], - variant_index: vec![Schema::Null, inner_schema] - .iter() - .enumerate() - .map(|(idx, s)| (SchemaKind::from(s), idx)) - .collect(), - }) + impl AvroSchemaComponent for Option + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + let inner_schema = T::get_schema_in_ctxt(named_schemas, enclosing_namespace); + Schema::Union(UnionSchema { + schemas: vec![Schema::Null, inner_schema.clone()], + variant_index: vec![Schema::Null, inner_schema] + .iter() + .enumerate() + .map(|(idx, s)| (SchemaKind::from(s), idx)) + .collect(), + }) + } } - } - impl AvroSchemaComponent for Map - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + impl AvroSchemaComponent for Map + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + } } - } - impl AvroSchemaComponent for HashMap - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + impl AvroSchemaComponent for HashMap + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + } } - } - impl AvroSchemaComponent for Box - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + impl AvroSchemaComponent for Box + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + } } - } - impl AvroSchemaComponent for std::sync::Mutex - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + impl AvroSchemaComponent for std::sync::Mutex + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + } } - } - impl AvroSchemaComponent for Cow<'_, T> - where - T: AvroSchemaComponent + Clone, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + impl AvroSchemaComponent for Cow<'_, T> + where + T: AvroSchemaComponent + Clone, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + } } } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::{SpecificSingleObjectWriter, error::Details, rabin::Rabin}; - use apache_avro_test_helper::{ - TestResult, - logger::{assert_logged, assert_not_logged}, - }; - use serde_json::json; - use serial_test::serial; - use std::sync::atomic::Ordering; + #[cfg(test)] + mod tests { + use super::*; + use crate::{SpecificSingleObjectWriter, error::Details, rabin::Rabin}; + use apache_avro_test_helper::{ + TestResult, + logger::{assert_logged, assert_not_logged}, + }; + use serde_json::json; + use serial_test::serial; + use std::sync::atomic::Ordering; - #[test] - fn test_invalid_schema() { - assert!(Schema::parse_str("invalid").is_err()); - } + #[test] + fn test_invalid_schema() { + assert!(Schema::parse_str("invalid").is_err()); + } - #[test] - fn test_primitive_schema() -> TestResult { - assert_eq!(Schema::Null, Schema::parse_str("\"null\"")?); - assert_eq!(Schema::Int, Schema::parse_str("\"int\"")?); - assert_eq!(Schema::Double, Schema::parse_str("\"double\"")?); - Ok(()) - } + #[test] + fn test_primitive_schema() -> TestResult { + assert_eq!(Schema::Null, Schema::parse_str("\"null\"")?); + assert_eq!(Schema::Int, Schema::parse_str("\"int\"")?); + assert_eq!(Schema::Double, Schema::parse_str("\"double\"")?); + Ok(()) + } - #[test] - fn test_array_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "array", "items": "string"}"#)?; - assert_eq!(Schema::array(Schema::String), schema); - Ok(()) - } + #[test] + fn test_array_schema() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "array", "items": "string"}"#)?; + assert_eq!(Schema::array(Schema::String), schema); + Ok(()) + } - #[test] - fn test_map_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "map", "values": "double"}"#)?; - assert_eq!(Schema::map(Schema::Double), schema); - Ok(()) - } + #[test] + fn test_map_schema() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "map", "values": "double"}"#)?; + assert_eq!(Schema::map(Schema::Double), schema); + Ok(()) + } - #[test] - fn test_union_schema() -> TestResult { - let schema = Schema::parse_str(r#"["null", "int"]"#)?; - assert_eq!( - Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), - schema - ); - Ok(()) - } + #[test] + fn test_union_schema() -> TestResult { + let schema = Schema::parse_str(r#"["null", "int"]"#)?; + assert_eq!( + Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), + schema + ); + Ok(()) + } - #[test] - fn test_union_unsupported_schema() { - let schema = Schema::parse_str(r#"["null", ["null", "int"], "string"]"#); - assert!(schema.is_err()); - } + #[test] + fn test_union_unsupported_schema() { + let schema = Schema::parse_str(r#"["null", ["null", "int"], "string"]"#); + assert!(schema.is_err()); + } - #[test] - fn test_multi_union_schema() -> TestResult { - let schema = Schema::parse_str(r#"["null", "int", "float", "string", "bytes"]"#); - assert!(schema.is_ok()); - let schema = schema?; - assert_eq!(SchemaKind::from(&schema), SchemaKind::Union); - let union_schema = match schema { - Schema::Union(u) => u, - _ => unreachable!(), - }; - assert_eq!(union_schema.variants().len(), 5); - let mut variants = union_schema.variants().iter(); - assert_eq!(SchemaKind::from(variants.next().unwrap()), SchemaKind::Null); - assert_eq!(SchemaKind::from(variants.next().unwrap()), SchemaKind::Int); - assert_eq!( - SchemaKind::from(variants.next().unwrap()), - SchemaKind::Float - ); - assert_eq!( - SchemaKind::from(variants.next().unwrap()), - SchemaKind::String - ); - assert_eq!( - SchemaKind::from(variants.next().unwrap()), - SchemaKind::Bytes - ); - assert_eq!(variants.next(), None); + #[test] + fn test_multi_union_schema() -> TestResult { + let schema = Schema::parse_str(r#"["null", "int", "float", "string", "bytes"]"#); + assert!(schema.is_ok()); + let schema = schema?; + assert_eq!(SchemaKind::from(&schema), SchemaKind::Union); + let union_schema = match schema { + Schema::Union(u) => u, + _ => unreachable!(), + }; + assert_eq!(union_schema.variants().len(), 5); + let mut variants = union_schema.variants().iter(); + assert_eq!(SchemaKind::from(variants.next().unwrap()), SchemaKind::Null); + assert_eq!(SchemaKind::from(variants.next().unwrap()), SchemaKind::Int); + assert_eq!( + SchemaKind::from(variants.next().unwrap()), + SchemaKind::Float + ); + assert_eq!( + SchemaKind::from(variants.next().unwrap()), + SchemaKind::String + ); + assert_eq!( + SchemaKind::from(variants.next().unwrap()), + SchemaKind::Bytes + ); + assert_eq!(variants.next(), None); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3621_nullable_record_field() -> TestResult { - let nullable_record_field = RecordField::builder() - .name("next".to_string()) - .schema(Schema::Union(UnionSchema::new(vec![ - Schema::Null, - Schema::Ref { - name: Name { - name: "LongList".to_owned(), - namespace: None, + #[test] + fn test_avro_3621_nullable_record_field() -> TestResult { + let nullable_record_field = RecordField::builder() + .name("next".to_string()) + .schema(Schema::Union(UnionSchema::new(vec![ + Schema::Null, + Schema::Ref { + name: Name { + name: "LongList".to_owned(), + namespace: None, + }, }, - }, - ])?)) - .order(RecordFieldOrder::Ascending) - .position(1) - .build(); - - assert!(nullable_record_field.is_nullable()); - - let non_nullable_record_field = RecordField::builder() - .name("next".to_string()) - .default(json!(2)) - .schema(Schema::Long) - .order(RecordFieldOrder::Ascending) - .position(1) - .build(); - - assert!(!non_nullable_record_field.is_nullable()); - Ok(()) - } + ])?)) + .order(RecordFieldOrder::Ascending) + .position(1) + .build(); + + assert!(nullable_record_field.is_nullable()); + + let non_nullable_record_field = RecordField::builder() + .name("next".to_string()) + .default(json!(2)) + .schema(Schema::Long) + .order(RecordFieldOrder::Ascending) + .position(1) + .build(); + + assert!(!non_nullable_record_field.is_nullable()); + Ok(()) + } - // AVRO-3248 - #[test] - fn test_union_of_records() -> TestResult { - use std::iter::FromIterator; + // AVRO-3248 + #[test] + fn test_union_of_records() -> TestResult { + use std::iter::FromIterator; - // A and B are the same except the name. - let schema_str_a = r#"{ + // A and B are the same except the name. + let schema_str_a = r#"{ "name": "A", "type": "record", "fields": [ @@ -2768,7 +2826,7 @@ mod tests { ] }"#; - let schema_str_b = r#"{ + let schema_str_b = r#"{ "name": "B", "type": "record", "fields": [ @@ -2776,8 +2834,8 @@ mod tests { ] }"#; - // we get Details::GetNameField if we put ["A", "B"] directly here. - let schema_str_c = r#"{ + // we get Details::GetNameField if we put ["A", "B"] directly here. + let schema_str_c = r#"{ "name": "C", "type": "record", "fields": [ @@ -2785,39 +2843,39 @@ mod tests { ] }"#; - let schema_c = Schema::parse_list([schema_str_a, schema_str_b, schema_str_c])? - .last() - .unwrap() - .clone(); - - let schema_c_expected = Schema::Record( - RecordSchema::builder() - .name(Name::new("C")?) - .fields(vec![ - RecordField::builder() - .name("field_one".to_string()) - .schema(Schema::Union(UnionSchema::new(vec![ - Schema::Ref { - name: Name::new("A")?, - }, - Schema::Ref { - name: Name::new("B")?, - }, - ])?)) - .build(), - ]) - .lookup(BTreeMap::from_iter(vec![("field_one".to_string(), 0)])) - .build(), - ); + let schema_c = Schema::parse_list([schema_str_a, schema_str_b, schema_str_c])? + .last() + .unwrap() + .clone(); + + let schema_c_expected = Schema::Record( + RecordSchema::builder() + .name(Name::new("C")?) + .fields(vec![ + RecordField::builder() + .name("field_one".to_string()) + .schema(Schema::Union(UnionSchema::new(vec![ + Schema::Ref { + name: Name::new("A")?, + }, + Schema::Ref { + name: Name::new("B")?, + }, + ])?)) + .build(), + ]) + .lookup(BTreeMap::from_iter(vec![("field_one".to_string(), 0)])) + .build(), + ); - assert_eq!(schema_c, schema_c_expected); - Ok(()) - } + assert_eq!(schema_c, schema_c_expected); + Ok(()) + } - #[test] - fn avro_rs_104_test_root_union_of_records() -> TestResult { - // A and B are the same except the name. - let schema_str_a = r#"{ + #[test] + fn avro_rs_104_test_root_union_of_records() -> TestResult { + // A and B are the same except the name. + let schema_str_a = r#"{ "name": "A", "type": "record", "fields": [ @@ -2825,7 +2883,7 @@ mod tests { ] }"#; - let schema_str_b = r#"{ + let schema_str_b = r#"{ "name": "B", "type": "record", "fields": [ @@ -2833,67 +2891,67 @@ mod tests { ] }"#; - let schema_str_c = r#"["A", "B"]"#; + let schema_str_c = r#"["A", "B"]"#; - let (schema_c, schemata) = - Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b])?; + let (schema_c, schemata) = + Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b])?; - let schema_a_expected = Schema::Record(RecordSchema { - name: Name::new("A")?, - aliases: None, - doc: None, - fields: vec![RecordField { - name: "field_one".to_string(), - doc: None, - default: None, + let schema_a_expected = Schema::Record(RecordSchema { + name: Name::new("A")?, aliases: None, - schema: Schema::Float, - order: RecordFieldOrder::Ignore, - position: 0, - custom_attributes: Default::default(), - }], - lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), - attributes: Default::default(), - }); - - let schema_b_expected = Schema::Record(RecordSchema { - name: Name::new("B")?, - aliases: None, - doc: None, - fields: vec![RecordField { - name: "field_one".to_string(), doc: None, - default: None, - aliases: None, - schema: Schema::Float, - order: RecordFieldOrder::Ignore, - position: 0, - custom_attributes: Default::default(), - }], - lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), - attributes: Default::default(), - }); - - let schema_c_expected = Schema::Union(UnionSchema::new(vec![ - Schema::Ref { - name: Name::new("A")?, - }, - Schema::Ref { + fields: vec![RecordField { + name: "field_one".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Float, + order: RecordFieldOrder::Ignore, + position: 0, + custom_attributes: Default::default(), + }], + lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), + attributes: Default::default(), + }); + + let schema_b_expected = Schema::Record(RecordSchema { name: Name::new("B")?, - }, - ])?); + aliases: None, + doc: None, + fields: vec![RecordField { + name: "field_one".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Float, + order: RecordFieldOrder::Ignore, + position: 0, + custom_attributes: Default::default(), + }], + lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), + attributes: Default::default(), + }); + + let schema_c_expected = Schema::Union(UnionSchema::new(vec![ + Schema::Ref { + name: Name::new("A")?, + }, + Schema::Ref { + name: Name::new("B")?, + }, + ])?); - assert_eq!(schema_c, schema_c_expected); - assert_eq!(schemata[0], schema_a_expected); - assert_eq!(schemata[1], schema_b_expected); + assert_eq!(schema_c, schema_c_expected); + assert_eq!(schemata[0], schema_a_expected); + assert_eq!(schemata[1], schema_b_expected); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_rs_104_test_root_union_of_records_name_collision() -> TestResult { - // A and B are exactly the same. - let schema_str_a1 = r#"{ + #[test] + fn avro_rs_104_test_root_union_of_records_name_collision() -> TestResult { + // A and B are exactly the same. + let schema_str_a1 = r#"{ "name": "A", "type": "record", "fields": [ @@ -2901,7 +2959,7 @@ mod tests { ] }"#; - let schema_str_a2 = r#"{ + let schema_str_a2 = r#"{ "name": "A", "type": "record", "fields": [ @@ -2909,22 +2967,22 @@ mod tests { ] }"#; - let schema_str_c = r#"["A", "A"]"#; + let schema_str_c = r#"["A", "A"]"#; - match Schema::parse_str_with_list(schema_str_c, [schema_str_a1, schema_str_a2]) { - Ok(_) => unreachable!("Expected an error that the name is already defined"), - Err(e) => assert_eq!( - e.to_string(), - "Two schemas with the same fullname were given: \"A\"" - ), - } + match Schema::parse_str_with_list(schema_str_c, [schema_str_a1, schema_str_a2]) { + Ok(_) => unreachable!("Expected an error that the name is already defined"), + Err(e) => assert_eq!( + e.to_string(), + "Two schemas with the same fullname were given: \"A\"" + ), + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_rs_104_test_root_union_of_records_no_name() -> TestResult { - let schema_str_a = r#"{ + #[test] + fn avro_rs_104_test_root_union_of_records_no_name() -> TestResult { + let schema_str_a = r#"{ "name": "A", "type": "record", "fields": [ @@ -2932,63 +2990,65 @@ mod tests { ] }"#; - // B has no name field. - let schema_str_b = r#"{ + // B has no name field. + let schema_str_b = r#"{ "type": "record", "fields": [ {"name": "field_one", "type": "float"} ] }"#; - let schema_str_c = r#"["A", "A"]"#; + let schema_str_c = r#"["A", "A"]"#; - match Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]) { - Ok(_) => unreachable!("Expected an error that schema_str_b is missing a name field"), - Err(e) => assert_eq!(e.to_string(), "No `name` field"), - } + match Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]) { + Ok(_) => { + unreachable!("Expected an error that schema_str_b is missing a name field") + } + Err(e) => assert_eq!(e.to_string(), "No `name` field"), + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3584_test_recursion_records() -> TestResult { - // A and B are the same except the name. - let schema_str_a = r#"{ + #[test] + fn avro_3584_test_recursion_records() -> TestResult { + // A and B are the same except the name. + let schema_str_a = r#"{ "name": "A", "type": "record", "fields": [ {"name": "field_one", "type": "B"} ] }"#; - let schema_str_b = r#"{ + let schema_str_b = r#"{ "name": "B", "type": "record", "fields": [ {"name": "field_one", "type": "A"} ] }"#; - let list = Schema::parse_list([schema_str_a, schema_str_b])?; + let list = Schema::parse_list([schema_str_a, schema_str_b])?; - let schema_a = list.first().unwrap().clone(); + let schema_a = list.first().unwrap().clone(); - match schema_a { - Schema::Record(RecordSchema { fields, .. }) => { - let f1 = fields.first(); + match schema_a { + Schema::Record(RecordSchema { fields, .. }) => { + let f1 = fields.first(); - let ref_schema = Schema::Ref { - name: Name::new("B")?, - }; - assert_eq!(ref_schema, f1.unwrap().schema); + let ref_schema = Schema::Ref { + name: Name::new("B")?, + }; + assert_eq!(ref_schema, f1.unwrap().schema); + } + _ => panic!("Expected a record schema!"), } - _ => panic!("Expected a record schema!"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3248_nullable_record() -> TestResult { - use std::iter::FromIterator; + #[test] + fn test_avro_3248_nullable_record() -> TestResult { + use std::iter::FromIterator; - let schema_str_a = r#"{ + let schema_str_a = r#"{ "name": "A", "type": "record", "fields": [ @@ -2996,8 +3056,8 @@ mod tests { ] }"#; - // we get Details::GetNameField if we put ["null", "B"] directly here. - let schema_str_option_a = r#"{ + // we get Details::GetNameField if we put ["null", "B"] directly here. + let schema_str_option_a = r#"{ "name": "OptionA", "type": "record", "fields": [ @@ -3005,43 +3065,43 @@ mod tests { ] }"#; - let schema_option_a = Schema::parse_list([schema_str_a, schema_str_option_a])? - .last() - .unwrap() - .clone(); - - let schema_option_a_expected = Schema::Record(RecordSchema { - name: Name::new("OptionA")?, - aliases: None, - doc: None, - fields: vec![RecordField { - name: "field_one".to_string(), - doc: None, - default: Some(Value::Null), + let schema_option_a = Schema::parse_list([schema_str_a, schema_str_option_a])? + .last() + .unwrap() + .clone(); + + let schema_option_a_expected = Schema::Record(RecordSchema { + name: Name::new("OptionA")?, aliases: None, - schema: Schema::Union(UnionSchema::new(vec![ - Schema::Null, - Schema::Ref { - name: Name::new("A")?, - }, - ])?), - order: RecordFieldOrder::Ignore, - position: 0, - custom_attributes: Default::default(), - }], - lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), - attributes: Default::default(), - }); + doc: None, + fields: vec![RecordField { + name: "field_one".to_string(), + doc: None, + default: Some(Value::Null), + aliases: None, + schema: Schema::Union(UnionSchema::new(vec![ + Schema::Null, + Schema::Ref { + name: Name::new("A")?, + }, + ])?), + order: RecordFieldOrder::Ignore, + position: 0, + custom_attributes: Default::default(), + }], + lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), + attributes: Default::default(), + }); - assert_eq!(schema_option_a, schema_option_a_expected); + assert_eq!(schema_option_a, schema_option_a_expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_record_schema() -> TestResult { - let parsed = Schema::parse_str( - r#" + #[test] + fn test_record_schema() -> TestResult { + let parsed = Schema::parse_str( + r#" { "type": "record", "name": "test", @@ -3051,51 +3111,51 @@ mod tests { ] } "#, - )?; - - let mut lookup = BTreeMap::new(); - lookup.insert("a".to_owned(), 0); - lookup.insert("b".to_owned(), 1); - - let expected = Schema::Record(RecordSchema { - name: Name::new("test")?, - aliases: None, - doc: None, - fields: vec![ - RecordField { - name: "a".to_string(), - doc: None, - default: Some(Value::Number(42i64.into())), - aliases: None, - schema: Schema::Long, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), - }, - RecordField { - name: "b".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::String, - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - ], - lookup, - attributes: Default::default(), - }); + )?; - assert_eq!(parsed, expected); + let mut lookup = BTreeMap::new(); + lookup.insert("a".to_owned(), 0); + lookup.insert("b".to_owned(), 1); - Ok(()) - } + let expected = Schema::Record(RecordSchema { + name: Name::new("test")?, + aliases: None, + doc: None, + fields: vec![ + RecordField { + name: "a".to_string(), + doc: None, + default: Some(Value::Number(42i64.into())), + aliases: None, + schema: Schema::Long, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "b".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::String, + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), + }, + ], + lookup, + attributes: Default::default(), + }); + + assert_eq!(parsed, expected); + + Ok(()) + } - #[test] - fn test_avro_3302_record_schema_with_currently_parsing_schema() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3302_record_schema_with_currently_parsing_schema() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "record", "name": "test", @@ -3112,75 +3172,75 @@ mod tests { }] } "#, - )?; + )?; - let mut lookup = BTreeMap::new(); - lookup.insert("recordField".to_owned(), 0); + let mut lookup = BTreeMap::new(); + lookup.insert("recordField".to_owned(), 0); - let mut node_lookup = BTreeMap::new(); - node_lookup.insert("children".to_owned(), 1); - node_lookup.insert("label".to_owned(), 0); + let mut node_lookup = BTreeMap::new(); + node_lookup.insert("children".to_owned(), 1); + node_lookup.insert("label".to_owned(), 0); - let expected = Schema::Record(RecordSchema { - name: Name::new("test")?, - aliases: None, - doc: None, - fields: vec![RecordField { - name: "recordField".to_string(), - doc: None, - default: None, + let expected = Schema::Record(RecordSchema { + name: Name::new("test")?, aliases: None, - schema: Schema::Record(RecordSchema { - name: Name::new("Node")?, - aliases: None, + doc: None, + fields: vec![RecordField { + name: "recordField".to_string(), doc: None, - fields: vec![ - RecordField { - name: "label".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::String, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), - }, - RecordField { - name: "children".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::array(Schema::Ref { - name: Name::new("Node")?, - }), - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - ], - lookup: node_lookup, - attributes: Default::default(), - }), - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), - }], - lookup, - attributes: Default::default(), - }); - assert_eq!(schema, expected); - - let canonical_form = &schema.canonical_form(); - let expected = r#"{"name":"test","type":"record","fields":[{"name":"recordField","type":{"name":"Node","type":"record","fields":[{"name":"label","type":"string"},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}"#; - assert_eq!(canonical_form, &expected); + default: None, + aliases: None, + schema: Schema::Record(RecordSchema { + name: Name::new("Node")?, + aliases: None, + doc: None, + fields: vec![ + RecordField { + name: "label".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::String, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "children".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::array(Schema::Ref { + name: Name::new("Node")?, + }), + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), + }, + ], + lookup: node_lookup, + attributes: Default::default(), + }), + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }], + lookup, + attributes: Default::default(), + }); + assert_eq!(schema, expected); - Ok(()) - } + let canonical_form = &schema.canonical_form(); + let expected = r#"{"name":"test","type":"record","fields":[{"name":"recordField","type":{"name":"Node","type":"record","fields":[{"name":"label","type":"string"},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}"#; + assert_eq!(canonical_form, &expected); + + Ok(()) + } - // https://github.com/flavray/avro-rs/pull/99#issuecomment-1016948451 - #[test] - fn test_parsing_of_recursive_type_enum() -> TestResult { - let schema = r#" + // https://github.com/flavray/avro-rs/pull/99#issuecomment-1016948451 + #[test] + fn test_parsing_of_recursive_type_enum() -> TestResult { + let schema = r#" { "type": "record", "name": "User", @@ -3223,17 +3283,17 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; - let schema_str = schema.canonical_form(); - let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"gender","type":{"name":"office.Gender","type":"enum","symbols":["male","female"]}}]},{"name":"office.Manager","type":"record","fields":[{"name":"gender","type":"office.Gender"}]}]}]}"#; - assert_eq!(schema_str, expected); + let schema = Schema::parse_str(schema)?; + let schema_str = schema.canonical_form(); + let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"gender","type":{"name":"office.Gender","type":"enum","symbols":["male","female"]}}]},{"name":"office.Manager","type":"record","fields":[{"name":"gender","type":"office.Gender"}]}]}]}"#; + assert_eq!(schema_str, expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_parsing_of_recursive_type_fixed() -> TestResult { - let schema = r#" + #[test] + fn test_parsing_of_recursive_type_fixed() -> TestResult { + let schema = r#" { "type": "record", "name": "User", @@ -3273,18 +3333,18 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; - let schema_str = schema.canonical_form(); - let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"id","type":{"name":"office.EmployeeId","type":"fixed","size":16}}]},{"name":"office.Manager","type":"record","fields":[{"name":"id","type":"office.EmployeeId"}]}]}]}"#; - assert_eq!(schema_str, expected); + let schema = Schema::parse_str(schema)?; + let schema_str = schema.canonical_form(); + let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"id","type":{"name":"office.EmployeeId","type":"fixed","size":16}}]},{"name":"office.Manager","type":"record","fields":[{"name":"id","type":"office.EmployeeId"}]}]}]}"#; + assert_eq!(schema_str, expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3302_record_schema_with_currently_parsing_schema_aliases() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3302_record_schema_with_currently_parsing_schema_aliases() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "record", "name": "LongList", @@ -3295,65 +3355,65 @@ mod tests { ] } "#, - )?; + )?; - let mut lookup = BTreeMap::new(); - lookup.insert("value".to_owned(), 0); - lookup.insert("next".to_owned(), 1); + let mut lookup = BTreeMap::new(); + lookup.insert("value".to_owned(), 0); + lookup.insert("next".to_owned(), 1); - let expected = Schema::Record(RecordSchema { - name: Name { - name: "LongList".to_owned(), - namespace: None, - }, - aliases: Some(vec![Alias::new("LinkedLongs").unwrap()]), - doc: None, - fields: vec![ - RecordField { - name: "value".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Long, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), + let expected = Schema::Record(RecordSchema { + name: Name { + name: "LongList".to_owned(), + namespace: None, }, - RecordField { - name: "next".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Union(UnionSchema::new(vec![ - Schema::Null, - Schema::Ref { - name: Name { - name: "LongList".to_owned(), - namespace: None, + aliases: Some(vec![Alias::new("LinkedLongs").unwrap()]), + doc: None, + fields: vec![ + RecordField { + name: "value".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Long, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "next".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Union(UnionSchema::new(vec![ + Schema::Null, + Schema::Ref { + name: Name { + name: "LongList".to_owned(), + namespace: None, + }, }, - }, - ])?), - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - ], - lookup, - attributes: Default::default(), - }); - assert_eq!(schema, expected); + ])?), + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), + }, + ], + lookup, + attributes: Default::default(), + }); + assert_eq!(schema, expected); - let canonical_form = &schema.canonical_form(); - let expected = r#"{"name":"LongList","type":"record","fields":[{"name":"value","type":"long"},{"name":"next","type":["null","LongList"]}]}"#; - assert_eq!(canonical_form, &expected); + let canonical_form = &schema.canonical_form(); + let expected = r#"{"name":"LongList","type":"record","fields":[{"name":"value","type":"long"},{"name":"next","type":["null","LongList"]}]}"#; + assert_eq!(canonical_form, &expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3370_record_schema_with_currently_parsing_schema_named_record() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3370_record_schema_with_currently_parsing_schema_named_record() -> TestResult { + let schema = Schema::parse_str( + r#" { "type" : "record", "name" : "record", @@ -3363,62 +3423,62 @@ mod tests { ] } "#, - )?; + )?; - let mut lookup = BTreeMap::new(); - lookup.insert("value".to_owned(), 0); - lookup.insert("next".to_owned(), 1); + let mut lookup = BTreeMap::new(); + lookup.insert("value".to_owned(), 0); + lookup.insert("next".to_owned(), 1); - let expected = Schema::Record(RecordSchema { - name: Name { - name: "record".to_owned(), - namespace: None, - }, - aliases: None, - doc: None, - fields: vec![ - RecordField { - name: "value".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Long, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), + let expected = Schema::Record(RecordSchema { + name: Name { + name: "record".to_owned(), + namespace: None, }, - RecordField { - name: "next".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Ref { - name: Name { - name: "record".to_owned(), - namespace: None, + aliases: None, + doc: None, + fields: vec![ + RecordField { + name: "value".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Long, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "next".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Ref { + name: Name { + name: "record".to_owned(), + namespace: None, + }, }, + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), }, - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - ], - lookup, - attributes: Default::default(), - }); - assert_eq!(schema, expected); + ], + lookup, + attributes: Default::default(), + }); + assert_eq!(schema, expected); - let canonical_form = &schema.canonical_form(); - let expected = r#"{"name":"record","type":"record","fields":[{"name":"value","type":"long"},{"name":"next","type":"record"}]}"#; - assert_eq!(canonical_form, &expected); + let canonical_form = &schema.canonical_form(); + let expected = r#"{"name":"record","type":"record","fields":[{"name":"value","type":"long"},{"name":"next","type":"record"}]}"#; + assert_eq!(canonical_form, &expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3370_record_schema_with_currently_parsing_schema_named_enum() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3370_record_schema_with_currently_parsing_schema_named_enum() -> TestResult { + let schema = Schema::parse_str( + r#" { "type" : "record", "name" : "record", @@ -3432,76 +3492,80 @@ mod tests { ] } "#, - )?; + )?; - let mut lookup = BTreeMap::new(); - lookup.insert("enum".to_owned(), 0); - lookup.insert("next".to_owned(), 1); + let mut lookup = BTreeMap::new(); + lookup.insert("enum".to_owned(), 0); + lookup.insert("next".to_owned(), 1); - let expected = Schema::Record(RecordSchema { - name: Name { - name: "record".to_owned(), - namespace: None, - }, - aliases: None, - doc: None, - fields: vec![ - RecordField { - name: "enum".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Enum( - EnumSchema::builder() - .name(Name::new("enum")?) - .symbols(vec![ - "one".to_string(), - "two".to_string(), - "three".to_string(), - ]) - .build(), - ), - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), + let expected = Schema::Record(RecordSchema { + name: Name { + name: "record".to_owned(), + namespace: None, }, - RecordField { - name: "next".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Enum(EnumSchema { - name: Name { - name: "enum".to_owned(), - namespace: None, - }, - aliases: None, + aliases: None, + doc: None, + fields: vec![ + RecordField { + name: "enum".to_string(), doc: None, - symbols: vec!["one".to_string(), "two".to_string(), "three".to_string()], default: None, - attributes: Default::default(), - }), - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - ], - lookup, - attributes: Default::default(), - }); - assert_eq!(schema, expected); + aliases: None, + schema: Schema::Enum( + EnumSchema::builder() + .name(Name::new("enum")?) + .symbols(vec![ + "one".to_string(), + "two".to_string(), + "three".to_string(), + ]) + .build(), + ), + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "next".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Enum(EnumSchema { + name: Name { + name: "enum".to_owned(), + namespace: None, + }, + aliases: None, + doc: None, + symbols: vec![ + "one".to_string(), + "two".to_string(), + "three".to_string(), + ], + default: None, + attributes: Default::default(), + }), + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), + }, + ], + lookup, + attributes: Default::default(), + }); + assert_eq!(schema, expected); - let canonical_form = &schema.canonical_form(); - let expected = r#"{"name":"record","type":"record","fields":[{"name":"enum","type":{"name":"enum","type":"enum","symbols":["one","two","three"]}},{"name":"next","type":"enum"}]}"#; - assert_eq!(canonical_form, &expected); + let canonical_form = &schema.canonical_form(); + let expected = r#"{"name":"record","type":"record","fields":[{"name":"enum","type":{"name":"enum","type":"enum","symbols":["one","two","three"]}},{"name":"next","type":"enum"}]}"#; + assert_eq!(canonical_form, &expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3370_record_schema_with_currently_parsing_schema_named_fixed() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3370_record_schema_with_currently_parsing_schema_named_fixed() -> TestResult { + let schema = Schema::parse_str( + r#" { "type" : "record", "name" : "record", @@ -3515,216 +3579,216 @@ mod tests { ] } "#, - )?; + )?; - let mut lookup = BTreeMap::new(); - lookup.insert("fixed".to_owned(), 0); - lookup.insert("next".to_owned(), 1); + let mut lookup = BTreeMap::new(); + lookup.insert("fixed".to_owned(), 0); + lookup.insert("next".to_owned(), 1); - let expected = Schema::Record(RecordSchema { - name: Name { - name: "record".to_owned(), - namespace: None, - }, - aliases: None, - doc: None, - fields: vec![ - RecordField { - name: "fixed".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Fixed(FixedSchema { - name: Name { - name: "fixed".to_owned(), - namespace: None, - }, - aliases: None, + let expected = Schema::Record(RecordSchema { + name: Name { + name: "record".to_owned(), + namespace: None, + }, + aliases: None, + doc: None, + fields: vec![ + RecordField { + name: "fixed".to_string(), doc: None, - size: 456, default: None, - attributes: Default::default(), - }), - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), - }, - RecordField { - name: "next".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Fixed(FixedSchema { - name: Name { - name: "fixed".to_owned(), - namespace: None, - }, aliases: None, + schema: Schema::Fixed(FixedSchema { + name: Name { + name: "fixed".to_owned(), + namespace: None, + }, + aliases: None, + doc: None, + size: 456, + default: None, + attributes: Default::default(), + }), + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "next".to_string(), doc: None, - size: 456, default: None, - attributes: Default::default(), - }), - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - ], - lookup, - attributes: Default::default(), - }); - assert_eq!(schema, expected); + aliases: None, + schema: Schema::Fixed(FixedSchema { + name: Name { + name: "fixed".to_owned(), + namespace: None, + }, + aliases: None, + doc: None, + size: 456, + default: None, + attributes: Default::default(), + }), + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), + }, + ], + lookup, + attributes: Default::default(), + }); + assert_eq!(schema, expected); - let canonical_form = &schema.canonical_form(); - let expected = r#"{"name":"record","type":"record","fields":[{"name":"fixed","type":{"name":"fixed","type":"fixed","size":456}},{"name":"next","type":"fixed"}]}"#; - assert_eq!(canonical_form, &expected); + let canonical_form = &schema.canonical_form(); + let expected = r#"{"name":"record","type":"record","fields":[{"name":"fixed","type":{"name":"fixed","type":"fixed","size":456}},{"name":"next","type":"fixed"}]}"#; + assert_eq!(canonical_form, &expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_enum_schema() -> TestResult { - let schema = Schema::parse_str( - r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "hearts"]}"#, - )?; - - let expected = Schema::Enum(EnumSchema { - name: Name::new("Suit")?, - aliases: None, - doc: None, - symbols: vec![ - "diamonds".to_owned(), - "spades".to_owned(), - "clubs".to_owned(), - "hearts".to_owned(), - ], - default: None, - attributes: Default::default(), - }); - - assert_eq!(expected, schema); + #[test] + fn test_enum_schema() -> TestResult { + let schema = Schema::parse_str( + r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "hearts"]}"#, + )?; - Ok(()) - } + let expected = Schema::Enum(EnumSchema { + name: Name::new("Suit")?, + aliases: None, + doc: None, + symbols: vec![ + "diamonds".to_owned(), + "spades".to_owned(), + "clubs".to_owned(), + "hearts".to_owned(), + ], + default: None, + attributes: Default::default(), + }); - #[test] - fn test_enum_schema_duplicate() -> TestResult { - // Duplicate "diamonds" - let schema = Schema::parse_str( - r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "diamonds"]}"#, - ); - assert!(schema.is_err()); + assert_eq!(expected, schema); - Ok(()) - } + Ok(()) + } - #[test] - fn test_enum_schema_name() -> TestResult { - // Invalid name "0000" does not match [A-Za-z_][A-Za-z0-9_]* - let schema = Schema::parse_str( - r#"{"type": "enum", "name": "Enum", "symbols": ["0000", "variant"]}"#, - ); - assert!(schema.is_err()); + #[test] + fn test_enum_schema_duplicate() -> TestResult { + // Duplicate "diamonds" + let schema = Schema::parse_str( + r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "diamonds"]}"#, + ); + assert!(schema.is_err()); - Ok(()) - } + Ok(()) + } - #[test] - fn test_fixed_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "fixed", "name": "test", "size": 16}"#)?; + #[test] + fn test_enum_schema_name() -> TestResult { + // Invalid name "0000" does not match [A-Za-z_][A-Za-z0-9_]* + let schema = Schema::parse_str( + r#"{"type": "enum", "name": "Enum", "symbols": ["0000", "variant"]}"#, + ); + assert!(schema.is_err()); - let expected = Schema::Fixed(FixedSchema { - name: Name::new("test")?, - aliases: None, - doc: None, - size: 16_usize, - default: None, - attributes: Default::default(), - }); + Ok(()) + } - assert_eq!(expected, schema); + #[test] + fn test_fixed_schema() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "fixed", "name": "test", "size": 16}"#)?; - Ok(()) - } + let expected = Schema::Fixed(FixedSchema { + name: Name::new("test")?, + aliases: None, + doc: None, + size: 16_usize, + default: None, + attributes: Default::default(), + }); - #[test] - fn test_fixed_schema_with_documentation() -> TestResult { - let schema = Schema::parse_str( - r#"{"type": "fixed", "name": "test", "size": 16, "doc": "FixedSchema documentation"}"#, - )?; + assert_eq!(expected, schema); - let expected = Schema::Fixed(FixedSchema { - name: Name::new("test")?, - aliases: None, - doc: Some(String::from("FixedSchema documentation")), - size: 16_usize, - default: None, - attributes: Default::default(), - }); + Ok(()) + } - assert_eq!(expected, schema); + #[test] + fn test_fixed_schema_with_documentation() -> TestResult { + let schema = Schema::parse_str( + r#"{"type": "fixed", "name": "test", "size": 16, "doc": "FixedSchema documentation"}"#, + )?; - Ok(()) - } + let expected = Schema::Fixed(FixedSchema { + name: Name::new("test")?, + aliases: None, + doc: Some(String::from("FixedSchema documentation")), + size: 16_usize, + default: None, + attributes: Default::default(), + }); - #[test] - fn test_no_documentation() -> TestResult { - let schema = Schema::parse_str( - r#"{"type": "enum", "name": "Coin", "symbols": ["heads", "tails"]}"#, - )?; + assert_eq!(expected, schema); - let doc = match schema { - Schema::Enum(EnumSchema { doc, .. }) => doc, - _ => unreachable!(), - }; + Ok(()) + } - assert!(doc.is_none()); + #[test] + fn test_no_documentation() -> TestResult { + let schema = Schema::parse_str( + r#"{"type": "enum", "name": "Coin", "symbols": ["heads", "tails"]}"#, + )?; - Ok(()) - } + let doc = match schema { + Schema::Enum(EnumSchema { doc, .. }) => doc, + _ => unreachable!(), + }; - #[test] - fn test_documentation() -> TestResult { - let schema = Schema::parse_str( - r#"{"type": "enum", "name": "Coin", "doc": "Some documentation", "symbols": ["heads", "tails"]}"#, - )?; + assert!(doc.is_none()); - let doc = match schema { - Schema::Enum(EnumSchema { doc, .. }) => doc, - _ => None, - }; + Ok(()) + } - assert_eq!("Some documentation".to_owned(), doc.unwrap()); + #[test] + fn test_documentation() -> TestResult { + let schema = Schema::parse_str( + r#"{"type": "enum", "name": "Coin", "doc": "Some documentation", "symbols": ["heads", "tails"]}"#, + )?; - Ok(()) - } + let doc = match schema { + Schema::Enum(EnumSchema { doc, .. }) => doc, + _ => None, + }; - // Tests to ensure Schema is Send + Sync. These tests don't need to _do_ anything, if they can - // compile, they pass. - #[test] - fn test_schema_is_send() { - fn send(_s: S) {} + assert_eq!("Some documentation".to_owned(), doc.unwrap()); - let schema = Schema::Null; - send(schema); - } + Ok(()) + } - #[test] - fn test_schema_is_sync() { - fn sync(_s: S) {} + // Tests to ensure Schema is Send + Sync. These tests don't need to _do_ anything, if they can + // compile, they pass. + #[test] + fn test_schema_is_send() { + fn send(_s: S) {} - let schema = Schema::Null; - sync(&schema); - sync(schema); - } + let schema = Schema::Null; + send(schema); + } + + #[test] + fn test_schema_is_sync() { + fn sync(_s: S) {} + + let schema = Schema::Null; + sync(&schema); + sync(schema); + } - #[test] - fn test_schema_fingerprint() -> TestResult { - use crate::rabin::Rabin; - use md5::Md5; - use sha2::Sha256; + #[test] + fn test_schema_fingerprint() -> TestResult { + use crate::rabin::Rabin; + use md5::Md5; + use sha2::Sha256; - let raw_schema = r#" + let raw_schema = r#" { "type": "record", "name": "test", @@ -3736,76 +3800,77 @@ mod tests { } "#; - let schema = Schema::parse_str(raw_schema)?; - assert_eq!( - "7eb3b28d73dfc99bdd9af1848298b40804a2f8ad5d2642be2ecc2ad34842b987", - format!("{}", schema.fingerprint::()) - ); + let schema = Schema::parse_str(raw_schema)?; + assert_eq!( + "7eb3b28d73dfc99bdd9af1848298b40804a2f8ad5d2642be2ecc2ad34842b987", + format!("{}", schema.fingerprint::()) + ); - assert_eq!( - "cb11615e412ee5d872620d8df78ff6ae", - format!("{}", schema.fingerprint::()) - ); - assert_eq!( - "92f2ccef718c6754", - format!("{}", schema.fingerprint::()) - ); + assert_eq!( + "cb11615e412ee5d872620d8df78ff6ae", + format!("{}", schema.fingerprint::()) + ); + assert_eq!( + "92f2ccef718c6754", + format!("{}", schema.fingerprint::()) + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_logical_types() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "int", "logicalType": "date"}"#)?; - assert_eq!(schema, Schema::Date); + #[test] + fn test_logical_types() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "int", "logicalType": "date"}"#)?; + assert_eq!(schema, Schema::Date); - let schema = Schema::parse_str(r#"{"type": "long", "logicalType": "timestamp-micros"}"#)?; - assert_eq!(schema, Schema::TimestampMicros); + let schema = + Schema::parse_str(r#"{"type": "long", "logicalType": "timestamp-micros"}"#)?; + assert_eq!(schema, Schema::TimestampMicros); - Ok(()) - } + Ok(()) + } - #[test] - fn test_nullable_logical_type() -> TestResult { - let schema = Schema::parse_str( - r#"{"type": ["null", {"type": "long", "logicalType": "timestamp-micros"}]}"#, - )?; - assert_eq!( - schema, - Schema::Union(UnionSchema::new(vec![ - Schema::Null, - Schema::TimestampMicros, - ])?) - ); + #[test] + fn test_nullable_logical_type() -> TestResult { + let schema = Schema::parse_str( + r#"{"type": ["null", {"type": "long", "logicalType": "timestamp-micros"}]}"#, + )?; + assert_eq!( + schema, + Schema::Union(UnionSchema::new(vec![ + Schema::Null, + Schema::TimestampMicros, + ])?) + ); - Ok(()) - } + Ok(()) + } - #[test] - fn record_field_order_from_str() -> TestResult { - use std::str::FromStr; + #[test] + fn record_field_order_from_str() -> TestResult { + use std::str::FromStr; - assert_eq!( - RecordFieldOrder::from_str("ascending").unwrap(), - RecordFieldOrder::Ascending - ); - assert_eq!( - RecordFieldOrder::from_str("descending").unwrap(), - RecordFieldOrder::Descending - ); - assert_eq!( - RecordFieldOrder::from_str("ignore").unwrap(), - RecordFieldOrder::Ignore - ); - assert!(RecordFieldOrder::from_str("not an ordering").is_err()); + assert_eq!( + RecordFieldOrder::from_str("ascending").unwrap(), + RecordFieldOrder::Ascending + ); + assert_eq!( + RecordFieldOrder::from_str("descending").unwrap(), + RecordFieldOrder::Descending + ); + assert_eq!( + RecordFieldOrder::from_str("ignore").unwrap(), + RecordFieldOrder::Ignore + ); + assert!(RecordFieldOrder::from_str("not an ordering").is_err()); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3374_preserve_namespace_for_primitive() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn test_avro_3374_preserve_namespace_for_primitive() -> TestResult { + let schema = Schema::parse_str( + r#" { "type" : "record", "name" : "ns.int", @@ -3815,20 +3880,20 @@ mod tests { ] } "#, - )?; + )?; - let json = schema.canonical_form(); - assert_eq!( - json, - r#"{"name":"ns.int","type":"record","fields":[{"name":"value","type":"int"},{"name":"next","type":["null","ns.int"]}]}"# - ); + let json = schema.canonical_form(); + assert_eq!( + json, + r#"{"name":"ns.int","type":"record","fields":[{"name":"value","type":"int"},{"name":"next","type":["null","ns.int"]}]}"# + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3433_preserve_schema_refs_in_json() -> TestResult { - let schema = r#" + #[test] + fn test_avro_3433_preserve_schema_refs_in_json() -> TestResult { + let schema = r#" { "name": "test.test", "type": "record", @@ -3842,17 +3907,17 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema)?; - let expected = r#"{"name":"test.test","type":"record","fields":[{"name":"bar","type":{"name":"test.foo","type":"record","fields":[{"name":"id","type":"long"}]}},{"name":"baz","type":"test.foo"}]}"#; - assert_eq!(schema.canonical_form(), expected); + let expected = r#"{"name":"test.test","type":"record","fields":[{"name":"bar","type":{"name":"test.foo","type":"record","fields":[{"name":"id","type":"long"}]}},{"name":"baz","type":"test.foo"}]}"#; + assert_eq!(schema.canonical_form(), expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_read_namespace_from_name() -> TestResult { - let schema = r#" + #[test] + fn test_read_namespace_from_name() -> TestResult { + let schema = r#" { "name": "space.name", "type": "record", @@ -3865,20 +3930,20 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; - if let Schema::Record(RecordSchema { name, .. }) = schema { - assert_eq!(name.name, "name"); - assert_eq!(name.namespace, Some("space".to_string())); - } else { - panic!("Expected a record schema!"); - } + let schema = Schema::parse_str(schema)?; + if let Schema::Record(RecordSchema { name, .. }) = schema { + assert_eq!(name.name, "name"); + assert_eq!(name.namespace, Some("space".to_string())); + } else { + panic!("Expected a record schema!"); + } - Ok(()) - } + Ok(()) + } - #[test] - fn test_namespace_from_name_has_priority_over_from_field() -> TestResult { - let schema = r#" + #[test] + fn test_namespace_from_name_has_priority_over_from_field() -> TestResult { + let schema = r#" { "name": "space1.name", "namespace": "space2", @@ -3892,19 +3957,19 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; - if let Schema::Record(RecordSchema { name, .. }) = schema { - assert_eq!(name.namespace, Some("space1".to_string())); - } else { - panic!("Expected a record schema!"); - } + let schema = Schema::parse_str(schema)?; + if let Schema::Record(RecordSchema { name, .. }) = schema { + assert_eq!(name.namespace, Some("space1".to_string())); + } else { + panic!("Expected a record schema!"); + } - Ok(()) - } + Ok(()) + } - #[test] - fn test_namespace_from_field() -> TestResult { - let schema = r#" + #[test] + fn test_namespace_from_field() -> TestResult { + let schema = r#" { "name": "name", "namespace": "space2", @@ -3918,47 +3983,47 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; - if let Schema::Record(RecordSchema { name, .. }) = schema { - assert_eq!(name.namespace, Some("space2".to_string())); - } else { - panic!("Expected a record schema!"); - } + let schema = Schema::parse_str(schema)?; + if let Schema::Record(RecordSchema { name, .. }) = schema { + assert_eq!(name.namespace, Some("space2".to_string())); + } else { + panic!("Expected a record schema!"); + } - Ok(()) - } + Ok(()) + } - #[test] - /// Zero-length namespace is considered as no-namespace. - fn test_namespace_from_name_with_empty_value() -> TestResult { - let name = Name::new(".name")?; - assert_eq!(name.name, "name"); - assert_eq!(name.namespace, None); + #[test] + /// Zero-length namespace is considered as no-namespace. + fn test_namespace_from_name_with_empty_value() -> TestResult { + let name = Name::new(".name")?; + assert_eq!(name.name, "name"); + assert_eq!(name.namespace, None); - Ok(()) - } + Ok(()) + } - #[test] - /// Whitespace is not allowed in the name. - fn test_name_with_whitespace_value() { - match Name::new(" ").map_err(Error::into_details) { - Err(Details::InvalidSchemaName(_, _)) => {} - _ => panic!("Expected an Details::InvalidSchemaName!"), + #[test] + /// Whitespace is not allowed in the name. + fn test_name_with_whitespace_value() { + match Name::new(" ").map_err(Error::into_details) { + Err(Details::InvalidSchemaName(_, _)) => {} + _ => panic!("Expected an Details::InvalidSchemaName!"), + } } - } - #[test] - /// The name must be non-empty. - fn test_name_with_no_name_part() { - match Name::new("space.").map_err(Error::into_details) { - Err(Details::InvalidSchemaName(_, _)) => {} - _ => panic!("Expected an Details::InvalidSchemaName!"), + #[test] + /// The name must be non-empty. + fn test_name_with_no_name_part() { + match Name::new("space.").map_err(Error::into_details) { + Err(Details::InvalidSchemaName(_, _)) => {} + _ => panic!("Expected an Details::InvalidSchemaName!"), + } } - } - #[test] - fn avro_3448_test_proper_resolution_inner_record_inherited_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_record_inherited_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -3987,19 +4052,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.inner_record_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.inner_record_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_record_qualified_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_record_qualified_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4028,19 +4093,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.inner_record_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.inner_record_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_enum_inherited_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_enum_inherited_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4064,19 +4129,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.inner_enum_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.inner_enum_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_enum_qualified_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_enum_qualified_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4100,19 +4165,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.inner_enum_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.inner_enum_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_fixed_inherited_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_fixed_inherited_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4136,19 +4201,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.inner_fixed_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.inner_fixed_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_fixed_qualified_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_fixed_qualified_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4172,19 +4237,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.inner_fixed_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.inner_fixed_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_record_inner_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_record_inner_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4214,19 +4279,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "inner_space.inner_record_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "inner_space.inner_record_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_enum_inner_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_enum_inner_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4251,19 +4316,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "inner_space.inner_enum_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "inner_space.inner_enum_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_resolution_inner_fixed_inner_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_resolution_inner_fixed_inner_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4288,19 +4353,20 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "inner_space.inner_fixed_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "inner_space.inner_fixed_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_multi_level_resolution_inner_record_outer_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_multi_level_resolution_inner_record_outer_namespace() -> TestResult + { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4341,23 +4407,24 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 3); - for s in &[ - "space.record_name", - "space.middle_record_name", - "space.inner_record_name", - ] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 3); + for s in &[ + "space.record_name", + "space.middle_record_name", + "space.inner_record_name", + ] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_multi_level_resolution_inner_record_middle_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_multi_level_resolution_inner_record_middle_namespace() -> TestResult + { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4399,23 +4466,24 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 3); - for s in &[ - "space.record_name", - "middle_namespace.middle_record_name", - "middle_namespace.inner_record_name", - ] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 3); + for s in &[ + "space.record_name", + "middle_namespace.middle_record_name", + "middle_namespace.inner_record_name", + ] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_multi_level_resolution_inner_record_inner_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_multi_level_resolution_inner_record_inner_namespace() -> TestResult + { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4458,23 +4526,23 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 3); - for s in &[ - "space.record_name", - "middle_namespace.middle_record_name", - "inner_namespace.inner_record_name", - ] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 3); + for s in &[ + "space.record_name", + "middle_namespace.middle_record_name", + "inner_namespace.inner_record_name", + ] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_in_array_resolution_inherited_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_in_array_resolution_inherited_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4503,19 +4571,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.in_array_record"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.in_array_record"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3448_test_proper_in_map_resolution_inherited_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3448_test_proper_in_map_resolution_inherited_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4544,19 +4612,19 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "space.in_map_record"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "space.in_map_record"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3466_test_to_json_inner_enum_inner_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3466_test_to_json_inner_enum_inner_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4581,26 +4649,26 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - // confirm we have expected 2 full-names - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "inner_space.inner_enum_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + // confirm we have expected 2 full-names + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "inner_space.inner_enum_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - // convert Schema back to JSON string - let schema_str = serde_json::to_string(&schema).expect("test failed"); - let _schema = Schema::parse_str(&schema_str).expect("test failed"); - assert_eq!(schema, _schema); + // convert Schema back to JSON string + let schema_str = serde_json::to_string(&schema).expect("test failed"); + let _schema = Schema::parse_str(&schema_str).expect("test failed"); + assert_eq!(schema, _schema); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3466_test_to_json_inner_fixed_inner_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3466_test_to_json_inner_fixed_inner_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -4625,41 +4693,41 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema)?; - let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); + let schema = Schema::parse_str(schema)?; + let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); - // confirm we have expected 2 full-names - assert_eq!(rs.get_names().len(), 2); - for s in &["space.record_name", "inner_space.inner_fixed_name"] { - assert!(rs.get_names().contains_key(&Name::new(s)?)); - } + // confirm we have expected 2 full-names + assert_eq!(rs.get_names().len(), 2); + for s in &["space.record_name", "inner_space.inner_fixed_name"] { + assert!(rs.get_names().contains_key(&Name::new(s)?)); + } - // convert Schema back to JSON string - let schema_str = serde_json::to_string(&schema).expect("test failed"); - let _schema = Schema::parse_str(&schema_str).expect("test failed"); - assert_eq!(schema, _schema); + // convert Schema back to JSON string + let schema_str = serde_json::to_string(&schema).expect("test failed"); + let _schema = Schema::parse_str(&schema_str).expect("test failed"); + assert_eq!(schema, _schema); - Ok(()) - } + Ok(()) + } - fn assert_avro_3512_aliases(aliases: &Aliases) { - match aliases { - Some(aliases) => { - assert_eq!(aliases.len(), 3); - assert_eq!(aliases[0], Alias::new("space.b").unwrap()); - assert_eq!(aliases[1], Alias::new("x.y").unwrap()); - assert_eq!(aliases[2], Alias::new(".c").unwrap()); - } - None => { - panic!("'aliases' must be Some"); + fn assert_avro_3512_aliases(aliases: &Aliases) { + match aliases { + Some(aliases) => { + assert_eq!(aliases.len(), 3); + assert_eq!(aliases[0], Alias::new("space.b").unwrap()); + assert_eq!(aliases[1], Alias::new("x.y").unwrap()); + assert_eq!(aliases[2], Alias::new(".c").unwrap()); + } + None => { + panic!("'aliases' must be Some"); + } } } - } - #[test] - fn avro_3512_alias_with_null_namespace_record() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn avro_3512_alias_with_null_namespace_record() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "record", "name": "a", @@ -4670,21 +4738,21 @@ mod tests { ] } "#, - )?; + )?; - if let Schema::Record(RecordSchema { ref aliases, .. }) = schema { - assert_avro_3512_aliases(aliases); - } else { - panic!("The Schema should be a record: {schema:?}"); - } + if let Schema::Record(RecordSchema { ref aliases, .. }) = schema { + assert_avro_3512_aliases(aliases); + } else { + panic!("The Schema should be a record: {schema:?}"); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3512_alias_with_null_namespace_enum() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn avro_3512_alias_with_null_namespace_enum() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "enum", "name": "a", @@ -4695,21 +4763,21 @@ mod tests { ] } "#, - )?; + )?; - if let Schema::Enum(EnumSchema { ref aliases, .. }) = schema { - assert_avro_3512_aliases(aliases); - } else { - panic!("The Schema should be an enum: {schema:?}"); - } + if let Schema::Enum(EnumSchema { ref aliases, .. }) = schema { + assert_avro_3512_aliases(aliases); + } else { + panic!("The Schema should be an enum: {schema:?}"); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3512_alias_with_null_namespace_fixed() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn avro_3512_alias_with_null_namespace_fixed() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "fixed", "name": "a", @@ -4718,21 +4786,21 @@ mod tests { "size" : 12 } "#, - )?; + )?; - if let Schema::Fixed(FixedSchema { ref aliases, .. }) = schema { - assert_avro_3512_aliases(aliases); - } else { - panic!("The Schema should be a fixed: {schema:?}"); - } + if let Schema::Fixed(FixedSchema { ref aliases, .. }) = schema { + assert_avro_3512_aliases(aliases); + } else { + panic!("The Schema should be a fixed: {schema:?}"); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3518_serialize_aliases_record() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn avro_3518_serialize_aliases_record() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "record", "name": "a", @@ -4749,23 +4817,23 @@ mod tests { ] } "#, - )?; + )?; - let value = serde_json::to_value(&schema)?; - let serialized = serde_json::to_string(&value)?; - assert_eq!( - r#"{"aliases":["space.b","x.y","c"],"fields":[{"aliases":["time1","ns.time2"],"default":123,"name":"time","type":"long"}],"name":"a","namespace":"space","type":"record"}"#, - &serialized - ); - assert_eq!(schema, Schema::parse_str(&serialized)?); + let value = serde_json::to_value(&schema)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"aliases":["space.b","x.y","c"],"fields":[{"aliases":["time1","ns.time2"],"default":123,"name":"time","type":"long"}],"name":"a","namespace":"space","type":"record"}"#, + &serialized + ); + assert_eq!(schema, Schema::parse_str(&serialized)?); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3518_serialize_aliases_enum() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn avro_3518_serialize_aliases_enum() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "enum", "name": "a", @@ -4776,23 +4844,23 @@ mod tests { ] } "#, - )?; + )?; - let value = serde_json::to_value(&schema)?; - let serialized = serde_json::to_string(&value)?; - assert_eq!( - r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","symbols":["symbol1","symbol2"],"type":"enum"}"#, - &serialized - ); - assert_eq!(schema, Schema::parse_str(&serialized)?); + let value = serde_json::to_value(&schema)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","symbols":["symbol1","symbol2"],"type":"enum"}"#, + &serialized + ); + assert_eq!(schema, Schema::parse_str(&serialized)?); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3518_serialize_aliases_fixed() -> TestResult { - let schema = Schema::parse_str( - r#" + #[test] + fn avro_3518_serialize_aliases_fixed() -> TestResult { + let schema = Schema::parse_str( + r#" { "type": "fixed", "name": "a", @@ -4801,22 +4869,22 @@ mod tests { "size" : 12 } "#, - )?; + )?; - let value = serde_json::to_value(&schema)?; - let serialized = serde_json::to_string(&value)?; - assert_eq!( - r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","size":12,"type":"fixed"}"#, - &serialized - ); - assert_eq!(schema, Schema::parse_str(&serialized)?); + let value = serde_json::to_value(&schema)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","size":12,"type":"fixed"}"#, + &serialized + ); + assert_eq!(schema, Schema::parse_str(&serialized)?); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3130_parse_anonymous_union_type() -> TestResult { - let schema_str = r#" + #[test] + fn avro_3130_parse_anonymous_union_type() -> TestResult { + let schema_str = r#" { "type": "record", "name": "AccountEvent", @@ -4835,40 +4903,40 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str)?; - if let Schema::Record(RecordSchema { name, fields, .. }) = schema { - assert_eq!(name, Name::new("AccountEvent")?); + if let Schema::Record(RecordSchema { name, fields, .. }) = schema { + assert_eq!(name, Name::new("AccountEvent")?); - let field = &fields[0]; - assert_eq!(&field.name, "NullableLongArray"); + let field = &fields[0]; + assert_eq!(&field.name, "NullableLongArray"); - if let Schema::Union(ref union) = field.schema { - assert_eq!(union.schemas[0], Schema::Null); + if let Schema::Union(ref union) = field.schema { + assert_eq!(union.schemas[0], Schema::Null); - if let Schema::Array(ref array_schema) = union.schemas[1] { - if let Schema::Long = *array_schema.items { - // OK + if let Schema::Array(ref array_schema) = union.schemas[1] { + if let Schema::Long = *array_schema.items { + // OK + } else { + panic!("Expected a Schema::Array of type Long"); + } } else { - panic!("Expected a Schema::Array of type Long"); + panic!("Expected Schema::Array"); } } else { - panic!("Expected Schema::Array"); + panic!("Expected Schema::Union"); } } else { - panic!("Expected Schema::Union"); + panic!("Expected Schema::Record"); } - } else { - panic!("Expected Schema::Record"); - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_custom_attributes_schema_without_attributes() -> TestResult { - let schemata_str = [ - r#" + #[test] + fn avro_custom_attributes_schema_without_attributes() -> TestResult { + let schemata_str = [ + r#" { "type": "record", "name": "Rec", @@ -4876,7 +4944,7 @@ mod tests { "fields": [] } "#, - r#" + r#" { "type": "enum", "name": "Enum", @@ -4884,7 +4952,7 @@ mod tests { "symbols": [] } "#, - r#" + r#" { "type": "fixed", "name": "Fixed", @@ -4892,16 +4960,16 @@ mod tests { "size": 0 } "#, - ]; - for schema_str in schemata_str.iter() { - let schema = Schema::parse_str(schema_str)?; - assert_eq!(schema.custom_attributes(), Some(&Default::default())); - } + ]; + for schema_str in schemata_str.iter() { + let schema = Schema::parse_str(schema_str)?; + assert_eq!(schema.custom_attributes(), Some(&Default::default())); + } - Ok(()) - } + Ok(()) + } - const CUSTOM_ATTRS_SUFFIX: &str = r#" + const CUSTOM_ATTRS_SUFFIX: &str = r#" "string_key": "value", "number_key": 1.23, "null_key": null, @@ -4911,10 +4979,10 @@ mod tests { } "#; - #[test] - fn avro_3609_custom_attributes_schema_with_attributes() -> TestResult { - let schemata_str = [ - r#" + #[test] + fn avro_3609_custom_attributes_schema_with_attributes() -> TestResult { + let schemata_str = [ + r#" { "type": "record", "name": "Rec", @@ -4924,7 +4992,7 @@ mod tests { {{{}}} } "#, - r#" + r#" { "type": "enum", "name": "Enum", @@ -4934,7 +5002,7 @@ mod tests { {{{}}} } "#, - r#" + r#" { "type": "fixed", "name": "Fixed", @@ -4944,44 +5012,45 @@ mod tests { {{{}}} } "#, - ]; + ]; - for schema_str in schemata_str.iter() { - let schema = Schema::parse_str( - schema_str - .to_owned() - .replace("{{{}}}", CUSTOM_ATTRS_SUFFIX) - .as_str(), - )?; + for schema_str in schemata_str.iter() { + let schema = Schema::parse_str( + schema_str + .to_owned() + .replace("{{{}}}", CUSTOM_ATTRS_SUFFIX) + .as_str(), + )?; - assert_eq!( - schema.custom_attributes(), - Some(&expected_custom_attributes()) - ); - } + assert_eq!( + schema.custom_attributes(), + Some(&expected_custom_attributes()) + ); + } - Ok(()) - } + Ok(()) + } - fn expected_custom_attributes() -> BTreeMap { - let mut expected_attributes: BTreeMap = Default::default(); - expected_attributes.insert("string_key".to_string(), Value::String("value".to_string())); - expected_attributes.insert("number_key".to_string(), json!(1.23)); - expected_attributes.insert("null_key".to_string(), Value::Null); - expected_attributes.insert( - "array_key".to_string(), - Value::Array(vec![json!(1), json!(2), json!(3)]), - ); - let mut object_value: HashMap = HashMap::new(); - object_value.insert("key".to_string(), Value::String("value".to_string())); - expected_attributes.insert("object_key".to_string(), json!(object_value)); - expected_attributes - } + fn expected_custom_attributes() -> BTreeMap { + let mut expected_attributes: BTreeMap = Default::default(); + expected_attributes + .insert("string_key".to_string(), Value::String("value".to_string())); + expected_attributes.insert("number_key".to_string(), json!(1.23)); + expected_attributes.insert("null_key".to_string(), Value::Null); + expected_attributes.insert( + "array_key".to_string(), + Value::Array(vec![json!(1), json!(2), json!(3)]), + ); + let mut object_value: HashMap = HashMap::new(); + object_value.insert("key".to_string(), Value::String("value".to_string())); + expected_attributes.insert("object_key".to_string(), json!(object_value)); + expected_attributes + } - #[test] - fn avro_3609_custom_attributes_record_field_without_attributes() -> TestResult { - let schema_str = String::from( - r#" + #[test] + fn avro_3609_custom_attributes_record_field_without_attributes() -> TestResult { + let schema_str = String::from( + r#" { "type": "record", "name": "Rec", @@ -4995,28 +5064,29 @@ mod tests { ] } "#, - ); + ); - let schema = Schema::parse_str(schema_str.replace("{{{}}}", CUSTOM_ATTRS_SUFFIX).as_str())?; + let schema = + Schema::parse_str(schema_str.replace("{{{}}}", CUSTOM_ATTRS_SUFFIX).as_str())?; - match schema { - Schema::Record(RecordSchema { name, fields, .. }) => { - assert_eq!(name, Name::new("Rec")?); - assert_eq!(fields.len(), 1); - let field = &fields[0]; - assert_eq!(&field.name, "field_one"); - assert_eq!(field.custom_attributes, expected_custom_attributes()); + match schema { + Schema::Record(RecordSchema { name, fields, .. }) => { + assert_eq!(name, Name::new("Rec")?); + assert_eq!(fields.len(), 1); + let field = &fields[0]; + assert_eq!(&field.name, "field_one"); + assert_eq!(field.custom_attributes, expected_custom_attributes()); + } + _ => panic!("Expected Schema::Record"), } - _ => panic!("Expected Schema::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3625_null_is_first() -> TestResult { - let schema_str = String::from( - r#" + #[test] + fn avro_3625_null_is_first() -> TestResult { + let schema_str = String::from( + r#" { "type": "record", "name": "union_schema_test", @@ -5025,37 +5095,37 @@ mod tests { ] } "#, - ); + ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str)?; - match schema { - Schema::Record(RecordSchema { name, fields, .. }) => { - assert_eq!(name, Name::new("union_schema_test")?); - assert_eq!(fields.len(), 1); - let field = &fields[0]; - assert_eq!(&field.name, "a"); - assert_eq!(&field.default, &Some(Value::Null)); - match &field.schema { - Schema::Union(union) => { - assert_eq!(union.variants().len(), 2); - assert!(union.is_nullable()); - assert_eq!(union.variants()[0], Schema::Null); - assert_eq!(union.variants()[1], Schema::Long); + match schema { + Schema::Record(RecordSchema { name, fields, .. }) => { + assert_eq!(name, Name::new("union_schema_test")?); + assert_eq!(fields.len(), 1); + let field = &fields[0]; + assert_eq!(&field.name, "a"); + assert_eq!(&field.default, &Some(Value::Null)); + match &field.schema { + Schema::Union(union) => { + assert_eq!(union.variants().len(), 2); + assert!(union.is_nullable()); + assert_eq!(union.variants()[0], Schema::Null); + assert_eq!(union.variants()[1], Schema::Long); + } + _ => panic!("Expected Schema::Union"), } - _ => panic!("Expected Schema::Union"), } + _ => panic!("Expected Schema::Record"), } - _ => panic!("Expected Schema::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3625_null_is_last() -> TestResult { - let schema_str = String::from( - r#" + #[test] + fn avro_3625_null_is_last() -> TestResult { + let schema_str = String::from( + r#" { "type": "record", "name": "union_schema_test", @@ -5064,36 +5134,36 @@ mod tests { ] } "#, - ); + ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str)?; - match schema { - Schema::Record(RecordSchema { name, fields, .. }) => { - assert_eq!(name, Name::new("union_schema_test")?); - assert_eq!(fields.len(), 1); - let field = &fields[0]; - assert_eq!(&field.name, "a"); - assert_eq!(&field.default, &Some(json!(123))); - match &field.schema { - Schema::Union(union) => { - assert_eq!(union.variants().len(), 2); - assert_eq!(union.variants()[0], Schema::Long); - assert_eq!(union.variants()[1], Schema::Null); + match schema { + Schema::Record(RecordSchema { name, fields, .. }) => { + assert_eq!(name, Name::new("union_schema_test")?); + assert_eq!(fields.len(), 1); + let field = &fields[0]; + assert_eq!(&field.name, "a"); + assert_eq!(&field.default, &Some(json!(123))); + match &field.schema { + Schema::Union(union) => { + assert_eq!(union.variants().len(), 2); + assert_eq!(union.variants()[0], Schema::Long); + assert_eq!(union.variants()[1], Schema::Null); + } + _ => panic!("Expected Schema::Union"), } - _ => panic!("Expected Schema::Union"), } + _ => panic!("Expected Schema::Record"), } - _ => panic!("Expected Schema::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3625_null_is_the_middle() -> TestResult { - let schema_str = String::from( - r#" + #[test] + fn avro_3625_null_is_the_middle() -> TestResult { + let schema_str = String::from( + r#" { "type": "record", "name": "union_schema_test", @@ -5102,37 +5172,37 @@ mod tests { ] } "#, - ); + ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str)?; - match schema { - Schema::Record(RecordSchema { name, fields, .. }) => { - assert_eq!(name, Name::new("union_schema_test")?); - assert_eq!(fields.len(), 1); - let field = &fields[0]; - assert_eq!(&field.name, "a"); - assert_eq!(&field.default, &Some(json!(123))); - match &field.schema { - Schema::Union(union) => { - assert_eq!(union.variants().len(), 3); - assert_eq!(union.variants()[0], Schema::Long); - assert_eq!(union.variants()[1], Schema::Null); - assert_eq!(union.variants()[2], Schema::Int); + match schema { + Schema::Record(RecordSchema { name, fields, .. }) => { + assert_eq!(name, Name::new("union_schema_test")?); + assert_eq!(fields.len(), 1); + let field = &fields[0]; + assert_eq!(&field.name, "a"); + assert_eq!(&field.default, &Some(json!(123))); + match &field.schema { + Schema::Union(union) => { + assert_eq!(union.variants().len(), 3); + assert_eq!(union.variants()[0], Schema::Long); + assert_eq!(union.variants()[1], Schema::Null); + assert_eq!(union.variants()[2], Schema::Int); + } + _ => panic!("Expected Schema::Union"), } - _ => panic!("Expected Schema::Union"), } + _ => panic!("Expected Schema::Record"), } - _ => panic!("Expected Schema::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3649_default_notintfirst() -> TestResult { - let schema_str = String::from( - r#" + #[test] + fn avro_3649_default_notintfirst() -> TestResult { + let schema_str = String::from( + r#" { "type": "record", "name": "union_schema_test", @@ -5141,35 +5211,35 @@ mod tests { ] } "#, - ); + ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str)?; - match schema { - Schema::Record(RecordSchema { name, fields, .. }) => { - assert_eq!(name, Name::new("union_schema_test")?); - assert_eq!(fields.len(), 1); - let field = &fields[0]; - assert_eq!(&field.name, "a"); - assert_eq!(&field.default, &Some(json!(123))); - match &field.schema { - Schema::Union(union) => { - assert_eq!(union.variants().len(), 2); - assert_eq!(union.variants()[0], Schema::String); - assert_eq!(union.variants()[1], Schema::Int); + match schema { + Schema::Record(RecordSchema { name, fields, .. }) => { + assert_eq!(name, Name::new("union_schema_test")?); + assert_eq!(fields.len(), 1); + let field = &fields[0]; + assert_eq!(&field.name, "a"); + assert_eq!(&field.default, &Some(json!(123))); + match &field.schema { + Schema::Union(union) => { + assert_eq!(union.variants().len(), 2); + assert_eq!(union.variants()[0], Schema::String); + assert_eq!(union.variants()[1], Schema::Int); + } + _ => panic!("Expected Schema::Union"), } - _ => panic!("Expected Schema::Union"), } + _ => panic!("Expected Schema::Record"), } - _ => panic!("Expected Schema::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3709_parsing_of_record_field_aliases() -> TestResult { - let schema = r#" + #[test] + fn avro_3709_parsing_of_record_field_aliases() -> TestResult { + let schema = r#" { "name": "rec", "type": "record", @@ -5183,21 +5253,21 @@ mod tests { } "#; - let schema = Schema::parse_str(schema)?; - if let Schema::Record(RecordSchema { fields, .. }) = schema { - let num_field = &fields[0]; - assert_eq!(num_field.name, "num"); - assert_eq!(num_field.aliases, Some(vec!("num1".into(), "num2".into()))); - } else { - panic!("Expected a record schema!"); - } + let schema = Schema::parse_str(schema)?; + if let Schema::Record(RecordSchema { fields, .. }) = schema { + let num_field = &fields[0]; + assert_eq!(num_field.name, "num"); + assert_eq!(num_field.aliases, Some(vec!("num1".into(), "num2".into()))); + } else { + panic!("Expected a record schema!"); + } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3735_parse_enum_namespace() -> TestResult { - let schema = r#" + #[test] + fn avro_3735_parse_enum_namespace() -> TestResult { + let schema = r#" { "type": "record", "name": "Foo", @@ -5225,65 +5295,81 @@ mod tests { } "#; - #[derive( - Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, serde::Deserialize, serde::Serialize, - )] - pub enum Bar { - #[serde(rename = "bar0")] - Bar0, - #[serde(rename = "bar1")] - Bar1, - } - - #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] - pub struct Foo { - #[serde(rename = "barInit")] - pub bar_init: Bar, - #[serde(rename = "barUse")] - pub bar_use: Bar, - } - - let schema = Schema::parse_str(schema)?; - - let foo = Foo { - bar_init: Bar::Bar0, - bar_use: Bar::Bar1, - }; + #[derive( + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Clone, + serde::Deserialize, + serde::Serialize, + )] + pub enum Bar { + #[serde(rename = "bar0")] + Bar0, + #[serde(rename = "bar1")] + Bar1, + } + + #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] + pub struct Foo { + #[serde(rename = "barInit")] + pub bar_init: Bar, + #[serde(rename = "barUse")] + pub bar_use: Bar, + } + + let schema = Schema::parse_str(schema)?; + + let foo = Foo { + bar_init: Bar::Bar0, + bar_use: Bar::Bar1, + }; - let avro_value = crate::to_value(foo)?; - assert!(avro_value.validate(&schema)); + let avro_value = crate::to_value(foo)?; + assert!(avro_value.validate(&schema)); - let mut writer = crate::Writer::new(&schema, Vec::new()); + let mut writer = crate::Writer::new(&schema, Vec::new()); - // schema validation happens here - writer.append(avro_value)?; + // schema validation happens here + writer.append(avro_value)?; - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3755_deserialize() -> TestResult { - #[derive( - Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, serde::Deserialize, serde::Serialize, - )] - pub enum Bar { - #[serde(rename = "bar0")] - Bar0, - #[serde(rename = "bar1")] - Bar1, - #[serde(rename = "bar2")] - Bar2, - } - - #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] - pub struct Foo { - #[serde(rename = "barInit")] - pub bar_init: Bar, - #[serde(rename = "barUse")] - pub bar_use: Bar, - } - - let writer_schema = r#"{ + #[test] + fn avro_3755_deserialize() -> TestResult { + #[derive( + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Clone, + serde::Deserialize, + serde::Serialize, + )] + pub enum Bar { + #[serde(rename = "bar0")] + Bar0, + #[serde(rename = "bar1")] + Bar1, + #[serde(rename = "bar2")] + Bar2, + } + + #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] + pub struct Foo { + #[serde(rename = "barInit")] + pub bar_init: Bar, + #[serde(rename = "barUse")] + pub bar_use: Bar, + } + + let writer_schema = r#"{ "type": "record", "name": "Foo", "fields": @@ -5308,7 +5394,7 @@ mod tests { ] }"#; - let reader_schema = r#"{ + let reader_schema = r#"{ "type": "record", "name": "Foo", "namespace": "name.space", @@ -5335,64 +5421,64 @@ mod tests { ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; - let foo = Foo { - bar_init: Bar::Bar0, - bar_use: Bar::Bar1, - }; - let avro_value = crate::to_value(foo)?; - assert!( - avro_value.validate(&writer_schema), - "value is valid for schema", - ); - let datum = crate::to_avro_datum(&writer_schema, avro_value)?; - let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; - let deser_value = crate::from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; - match deser_value { - types::Value::Record(fields) => { - assert_eq!(fields.len(), 2); - assert_eq!(fields[0].0, "barInit"); - assert_eq!(fields[0].1, types::Value::Enum(0, "bar0".to_string())); - assert_eq!(fields[1].0, "barUse"); - assert_eq!(fields[1].1, types::Value::Enum(1, "bar1".to_string())); + let writer_schema = Schema::parse_str(writer_schema)?; + let foo = Foo { + bar_init: Bar::Bar0, + bar_use: Bar::Bar1, + }; + let avro_value = crate::to_value(foo)?; + assert!( + avro_value.validate(&writer_schema), + "value is valid for schema", + ); + let datum = crate::to_avro_datum(&writer_schema, avro_value)?; + let mut x = &datum[..]; + let reader_schema = Schema::parse_str(reader_schema)?; + let deser_value = crate::from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; + match deser_value { + types::Value::Record(fields) => { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].0, "barInit"); + assert_eq!(fields[0].1, types::Value::Enum(0, "bar0".to_string())); + assert_eq!(fields[1].0, "barUse"); + assert_eq!(fields[1].1, types::Value::Enum(1, "bar1".to_string())); + } + _ => panic!("Expected Value::Record"), } - _ => panic!("Expected Value::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3780_decimal_schema_type_with_fixed() -> TestResult { - let schema = json!( - { - "type": "record", - "name": "recordWithDecimal", - "fields": [ + #[test] + fn test_avro_3780_decimal_schema_type_with_fixed() -> TestResult { + let schema = json!( { - "name": "decimal", - "type": "fixed", - "name": "nestedFixed", - "size": 8, - "logicalType": "decimal", - "precision": 4 - } - ] - }); + "type": "record", + "name": "recordWithDecimal", + "fields": [ + { + "name": "decimal", + "type": "fixed", + "name": "nestedFixed", + "size": 8, + "logicalType": "decimal", + "precision": 4 + } + ] + }); - let parse_result = Schema::parse(&schema); - assert!( - parse_result.is_ok(), - "parse result must be ok, got: {parse_result:?}" - ); + let parse_result = Schema::parse(&schema); + assert!( + parse_result.is_ok(), + "parse result must be ok, got: {parse_result:?}" + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3772_enum_default_wrong_type() -> TestResult { - let schema = r#" + #[test] + fn test_avro_3772_enum_default_wrong_type() -> TestResult { + let schema = r#" { "type": "record", "name": "test", @@ -5412,21 +5498,21 @@ mod tests { } "#; - match Schema::parse_str(schema) { - Err(err) => { - assert_eq!( - err.to_string(), - "Default value for enum must be a string! Got: 123" - ); + match Schema::parse_str(schema) { + Err(err) => { + assert_eq!( + err.to_string(), + "Default value for enum must be a string! Got: 123" + ); + } + _ => panic!("Expected an error"), } - _ => panic!("Expected an error"), + Ok(()) } - Ok(()) - } - #[test] - fn test_avro_3812_handle_null_namespace_properly() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3812_handle_null_namespace_properly() -> TestResult { + let schema_str = r#" { "namespace": "", "type": "record", @@ -5453,24 +5539,24 @@ mod tests { } "#; - let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"a","type":{"name":"my_enum","type":"enum","symbols":["a","b"]}},{"name":"b","type":{"name":"my_fixed","type":"fixed","size":10}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"a","type":{"name":"my_enum","type":"enum","symbols":["a","b"]}},{"name":"b","type":{"name":"my_fixed","type":"fixed","size":10}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - let name = Name::new("my_name")?; - let fullname = name.fullname(Some("".to_string())); - assert_eq!(fullname, "my_name"); - let qname = name.fully_qualified_name(&Some("".to_string())).to_string(); - assert_eq!(qname, "my_name"); + let name = Name::new("my_name")?; + let fullname = name.fullname(Some("".to_string())); + assert_eq!(fullname, "my_name"); + let qname = name.fully_qualified_name(&Some("".to_string())).to_string(); + assert_eq!(qname, "my_name"); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3818_inherit_enclosing_namespace() -> TestResult { - // Enclosing namespace is specified but inner namespaces are not. - let schema_str = r#" + #[test] + fn test_avro_3818_inherit_enclosing_namespace() -> TestResult { + // Enclosing namespace is specified but inner namespaces are not. + let schema_str = r#" { "namespace": "my_ns", "type": "record", @@ -5495,14 +5581,14 @@ mod tests { } "#; - let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"my_ns.fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"my_ns.fixed1","type":"fixed","size":1}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - // Enclosing namespace and inner namespaces are specified - // but inner namespaces are "" - let schema_str = r#" + // Enclosing namespace and inner namespaces are specified + // but inner namespaces are "" + let schema_str = r#" { "namespace": "my_ns", "type": "record", @@ -5529,13 +5615,13 @@ mod tests { } "#; - let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fixed1","type":"fixed","size":1}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - // Enclosing namespace is "" and inner non-empty namespaces are specified. - let schema_str = r#" + // Enclosing namespace is "" and inner non-empty namespaces are specified. + let schema_str = r#" { "namespace": "", "type": "record", @@ -5561,13 +5647,13 @@ mod tests { } "#; - let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"f1.ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"f2.ns.fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"f1.ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"f2.ns.fixed1","type":"fixed","size":1}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - // Nested complex types with non-empty enclosing namespace. - let schema_str = r#" + // Nested complex types with non-empty enclosing namespace. + let schema_str = r#" { "type": "record", "name": "my_ns.my_schema", @@ -5610,40 +5696,40 @@ mod tests { } "#; - let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.inner_record1","type":"record","fields":[{"name":"f1_1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}}]}},{"name":"f2","type":{"name":"inner_ns.inner_record2","type":"record","fields":[{"name":"f2_1","type":{"name":"inner_ns.enum2","type":"enum","symbols":["a"]}}]}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.inner_record1","type":"record","fields":[{"name":"f1_1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}}]}},{"name":"f2","type":{"name":"inner_ns.inner_record2","type":"record","fields":[{"name":"f2_1","type":{"name":"inner_ns.enum2","type":"enum","symbols":["a"]}}]}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3779_bigdecimal_schema() -> TestResult { - let schema = json!( - { - "name": "decimal", - "type": "bytes", - "logicalType": "big-decimal" + #[test] + fn test_avro_3779_bigdecimal_schema() -> TestResult { + let schema = json!( + { + "name": "decimal", + "type": "bytes", + "logicalType": "big-decimal" + } + ); + + let parse_result = Schema::parse(&schema); + assert!( + parse_result.is_ok(), + "parse result must be ok, got: {parse_result:?}" + ); + match parse_result? { + Schema::BigDecimal => (), + other => panic!("Expected Schema::BigDecimal but got: {other:?}"), } - ); - let parse_result = Schema::parse(&schema); - assert!( - parse_result.is_ok(), - "parse result must be ok, got: {parse_result:?}" - ); - match parse_result? { - Schema::BigDecimal => (), - other => panic!("Expected Schema::BigDecimal but got: {other:?}"), + Ok(()) } - Ok(()) - } - - #[test] - fn test_avro_3820_deny_invalid_field_names() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3820_deny_invalid_field_names() -> TestResult { + let schema_str = r#" { "name": "my_record", "type": "record", @@ -5667,15 +5753,15 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::FieldName(x)) if x == "f1.x" => Ok(()), - other => Err(format!("Expected Details::FieldName, got {other:?}").into()), + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::FieldName(x)) if x == "f1.x" => Ok(()), + other => Err(format!("Expected Details::FieldName, got {other:?}").into()), + } } - } - #[test] - fn test_avro_3827_disallow_duplicate_field_names() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3827_disallow_duplicate_field_names() -> TestResult { + let schema_str = r#" { "name": "my_schema", "type": "record", @@ -5699,14 +5785,16 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::FieldNameDuplicate(_)) => (), - other => { - return Err(format!("Expected Details::FieldNameDuplicate, got {other:?}").into()); - } - }; + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::FieldNameDuplicate(_)) => (), + other => { + return Err( + format!("Expected Details::FieldNameDuplicate, got {other:?}").into(), + ); + } + }; - let schema_str = r#" + let schema_str = r#" { "name": "my_schema", "type": "record", @@ -5732,19 +5820,19 @@ mod tests { } "#; - let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"a","type":"record","fields":[{"name":"f1","type":{"name":"b","type":"record","fields":[]}}]}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"a","type":"record","fields":[{"name":"f1","type":{"name":"b","type":"record","fields":[]}}]}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3830_null_namespace_in_fully_qualified_names() -> TestResult { - // Check whether all the named types don't refer to the namespace field - // if their name starts with a dot. - let schema_str = r#" + #[test] + fn test_avro_3830_null_namespace_in_fully_qualified_names() -> TestResult { + // Check whether all the named types don't refer to the namespace field + // if their name starts with a dot. + let schema_str = r#" { "name": ".record1", "namespace": "ns1", @@ -5771,13 +5859,13 @@ mod tests { } "#; - let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - // Check whether inner types don't inherit ns1. - let schema_str = r#" + // Check whether inner types don't inherit ns1. + let schema_str = r#" { "name": ".record1", "namespace": "ns1", @@ -5802,148 +5890,148 @@ mod tests { } "#; - let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - let name = Name::new(".my_name")?; - let fullname = name.fullname(None); - assert_eq!(fullname, "my_name"); - let qname = name.fully_qualified_name(&None).to_string(); - assert_eq!(qname, "my_name"); + let name = Name::new(".my_name")?; + let fullname = name.fullname(None); + assert_eq!(fullname, "my_name"); + let qname = name.fully_qualified_name(&None).to_string(); + assert_eq!(qname, "my_name"); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3814_schema_resolution_failure() -> TestResult { - // Define a reader schema: a nested record with an optional field. - let reader_schema = json!( - { - "type": "record", - "name": "MyOuterRecord", - "fields": [ - { - "name": "inner_record", - "type": [ - "null", - { - "type": "record", - "name": "MyRecord", - "fields": [ - {"name": "a", "type": "string"} - ] - } - ], - "default": null - } - ] - } - ); + #[test] + fn test_avro_3814_schema_resolution_failure() -> TestResult { + // Define a reader schema: a nested record with an optional field. + let reader_schema = json!( + { + "type": "record", + "name": "MyOuterRecord", + "fields": [ + { + "name": "inner_record", + "type": [ + "null", + { + "type": "record", + "name": "MyRecord", + "fields": [ + {"name": "a", "type": "string"} + ] + } + ], + "default": null + } + ] + } + ); - // Define a writer schema: a nested record with an optional field, which - // may optionally contain an enum. - let writer_schema = json!( - { - "type": "record", - "name": "MyOuterRecord", - "fields": [ - { - "name": "inner_record", - "type": [ - "null", - { - "type": "record", - "name": "MyRecord", - "fields": [ - {"name": "a", "type": "string"}, - { - "name": "b", - "type": [ - "null", - { - "type": "enum", - "name": "MyEnum", - "symbols": ["A", "B", "C"], - "default": "C" - } - ], - "default": null - }, - ] - } - ] - } - ], - "default": null - } - ); + // Define a writer schema: a nested record with an optional field, which + // may optionally contain an enum. + let writer_schema = json!( + { + "type": "record", + "name": "MyOuterRecord", + "fields": [ + { + "name": "inner_record", + "type": [ + "null", + { + "type": "record", + "name": "MyRecord", + "fields": [ + {"name": "a", "type": "string"}, + { + "name": "b", + "type": [ + "null", + { + "type": "enum", + "name": "MyEnum", + "symbols": ["A", "B", "C"], + "default": "C" + } + ], + "default": null + }, + ] + } + ] + } + ], + "default": null + } + ); - // Use different structs to represent the "Reader" and the "Writer" - // to mimic two different versions of a producer & consumer application. - #[derive(Serialize, Deserialize, Debug)] - struct MyInnerRecordReader { - a: String, - } + // Use different structs to represent the "Reader" and the "Writer" + // to mimic two different versions of a producer & consumer application. + #[derive(Serialize, Deserialize, Debug)] + struct MyInnerRecordReader { + a: String, + } - #[derive(Serialize, Deserialize, Debug)] - struct MyRecordReader { - inner_record: Option, - } + #[derive(Serialize, Deserialize, Debug)] + struct MyRecordReader { + inner_record: Option, + } - #[derive(Serialize, Deserialize, Debug)] - enum MyEnum { - A, - B, - C, - } + #[derive(Serialize, Deserialize, Debug)] + enum MyEnum { + A, + B, + C, + } - #[derive(Serialize, Deserialize, Debug)] - struct MyInnerRecordWriter { - a: String, - b: Option, - } + #[derive(Serialize, Deserialize, Debug)] + struct MyInnerRecordWriter { + a: String, + b: Option, + } - #[derive(Serialize, Deserialize, Debug)] - struct MyRecordWriter { - inner_record: Option, - } + #[derive(Serialize, Deserialize, Debug)] + struct MyRecordWriter { + inner_record: Option, + } - let s = MyRecordWriter { - inner_record: Some(MyInnerRecordWriter { - a: "foo".to_string(), - b: None, - }), - }; + let s = MyRecordWriter { + inner_record: Some(MyInnerRecordWriter { + a: "foo".to_string(), + b: None, + }), + }; - // Serialize using the writer schema. - let writer_schema = Schema::parse(&writer_schema)?; - let avro_value = crate::to_value(s)?; - assert!( - avro_value.validate(&writer_schema), - "value is valid for schema", - ); - let datum = crate::to_avro_datum(&writer_schema, avro_value)?; + // Serialize using the writer schema. + let writer_schema = Schema::parse(&writer_schema)?; + let avro_value = crate::to_value(s)?; + assert!( + avro_value.validate(&writer_schema), + "value is valid for schema", + ); + let datum = crate::to_avro_datum(&writer_schema, avro_value)?; - // Now, attempt to deserialize using the reader schema. - let reader_schema = Schema::parse(&reader_schema)?; - let mut x = &datum[..]; + // Now, attempt to deserialize using the reader schema. + let reader_schema = Schema::parse(&reader_schema)?; + let mut x = &datum[..]; - // Deserialization should succeed and we should be able to resolve the schema. - let deser_value = crate::from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; - assert!(deser_value.validate(&reader_schema)); + // Deserialization should succeed and we should be able to resolve the schema. + let deser_value = crate::from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; + assert!(deser_value.validate(&reader_schema)); - // Verify that we can read a field from the record. - let d: MyRecordReader = crate::from_value(&deser_value)?; - assert_eq!(d.inner_record.unwrap().a, "foo".to_string()); - Ok(()) - } + // Verify that we can read a field from the record. + let d: MyRecordReader = crate::from_value(&deser_value)?; + assert_eq!(d.inner_record.unwrap().a, "foo".to_string()); + Ok(()) + } - #[test] - fn test_avro_3837_disallow_invalid_namespace() -> TestResult { - // Valid namespace #1 (Single name portion) - let schema_str = r#" + #[test] + fn test_avro_3837_disallow_invalid_namespace() -> TestResult { + // Valid namespace #1 (Single name portion) + let schema_str = r#" { "name": "record1", "namespace": "ns1", @@ -5952,13 +6040,13 @@ mod tests { } "#; - let expected = r#"{"name":"ns1.record1","type":"record","fields":[]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"ns1.record1","type":"record","fields":[]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - // Valid namespace #2 (multiple name portions). - let schema_str = r#" + // Valid namespace #2 (multiple name portions). + let schema_str = r#" { "name": "enum1", "namespace": "ns1.foo.bar", @@ -5967,13 +6055,13 @@ mod tests { } "#; - let expected = r#"{"name":"ns1.foo.bar.enum1","type":"enum","symbols":["a"]}"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - assert_eq!(canonical_form, expected); + let expected = r#"{"name":"ns1.foo.bar.enum1","type":"enum","symbols":["a"]}"#; + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + assert_eq!(canonical_form, expected); - // Invalid namespace #1 (a name portion starts with dot) - let schema_str = r#" + // Invalid namespace #1 (a name portion starts with dot) + let schema_str = r#" { "name": "fixed1", "namespace": ".ns1.a.b", @@ -5982,15 +6070,15 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::InvalidNamespace(_, _)) => (), - other => { - return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); - } - }; + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::InvalidNamespace(_, _)) => (), + other => { + return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); + } + }; - // Invalid namespace #2 (invalid character in a name portion) - let schema_str = r#" + // Invalid namespace #2 (invalid character in a name portion) + let schema_str = r#" { "name": "record1", "namespace": "ns1.a*b.c", @@ -5999,15 +6087,15 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::InvalidNamespace(_, _)) => (), - other => { - return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); - } - }; + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::InvalidNamespace(_, _)) => (), + other => { + return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); + } + }; - // Invalid namespace #3 (a name portion starts with a digit) - let schema_str = r#" + // Invalid namespace #3 (a name portion starts with a digit) + let schema_str = r#" { "name": "fixed1", "namespace": "ns1.1a.b", @@ -6016,15 +6104,15 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::InvalidNamespace(_, _)) => (), - other => { - return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); - } - }; + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::InvalidNamespace(_, _)) => (), + other => { + return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); + } + }; - // Invalid namespace #4 (a name portion is missing - two dots in a row) - let schema_str = r#" + // Invalid namespace #4 (a name portion is missing - two dots in a row) + let schema_str = r#" { "name": "fixed1", "namespace": "ns1..a", @@ -6033,15 +6121,15 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::InvalidNamespace(_, _)) => (), - other => { - return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); - } - }; + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::InvalidNamespace(_, _)) => (), + other => { + return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); + } + }; - // Invalid namespace #5 (a name portion is missing - ends with a dot) - let schema_str = r#" + // Invalid namespace #5 (a name portion is missing - ends with a dot) + let schema_str = r#" { "name": "fixed1", "namespace": "ns1.a.", @@ -6050,19 +6138,19 @@ mod tests { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { - Err(Details::InvalidNamespace(_, _)) => (), - other => { - return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); - } - }; + match Schema::parse_str(schema_str).map_err(Error::into_details) { + Err(Details::InvalidNamespace(_, _)) => (), + other => { + return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); + } + }; - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_simple_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_simple_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6076,26 +6164,26 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f1".to_string(), - "ns.record1".to_string(), - r#""int""#.to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f1".to_string(), + "ns.record1".to_string(), + r#""int""#.to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_nested_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_nested_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6118,27 +6206,27 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f1".to_string(), - "ns.record1".to_string(), - r#"{"name":"ns.record2","type":"record","fields":[{"name":"f1_1","type":"int"}]}"# - .to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f1".to_string(), + "ns.record1".to_string(), + r#"{"name":"ns.record2","type":"record","fields":[{"name":"f1_1","type":"int"}]}"# + .to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_enum_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_enum_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6156,26 +6244,26 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f1".to_string(), - "ns.record1".to_string(), - r#"{"name":"ns.enum1","type":"enum","symbols":["a","b","c"]}"#.to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f1".to_string(), + "ns.record1".to_string(), + r#"{"name":"ns.enum1","type":"enum","symbols":["a","b","c"]}"#.to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_fixed_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_fixed_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6193,26 +6281,26 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f1".to_string(), - "ns.record1".to_string(), - r#"{"name":"ns.fixed1","type":"fixed","size":3}"#.to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f1".to_string(), + "ns.record1".to_string(), + r#"{"name":"ns.fixed1","type":"fixed","size":3}"#.to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_array_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_array_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6227,26 +6315,26 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f1".to_string(), - "ns.record1".to_string(), - r#"{"type":"array","items":"int"}"#.to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f1".to_string(), + "ns.record1".to_string(), + r#"{"type":"array","items":"int"}"#.to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_map_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_map_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6261,26 +6349,26 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f1".to_string(), - "ns.record1".to_string(), - r#"{"type":"map","values":"string"}"#.to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f1".to_string(), + "ns.record1".to_string(), + r#"{"type":"map","values":"string"}"#.to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_ref_record_field() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_ref_record_field() -> TestResult { + let schema_str = r#" { "name": "record1", "namespace": "ns", @@ -6306,26 +6394,26 @@ mod tests { ] } "#; - let expected = Details::GetDefaultRecordField( - "f2".to_string(), - "ns.record1".to_string(), - r#""ns.record2""#.to_string(), - ) - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetDefaultRecordField( + "f2".to_string(), + "ns.record1".to_string(), + r#""ns.record2""#.to_string(), + ) + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3851_validate_default_value_of_enum() -> TestResult { - let schema_str = r#" + #[test] + fn test_avro_3851_validate_default_value_of_enum() -> TestResult { + let schema_str = r#" { "name": "enum1", "namespace": "ns", @@ -6334,16 +6422,16 @@ mod tests { "default": 100 } "#; - let expected = Details::EnumDefaultWrongType(100.into()).to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); - - let schema_str = r#" + let expected = Details::EnumDefaultWrongType(100.into()).to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); + + let schema_str = r#" { "name": "enum1", "namespace": "ns", @@ -6352,26 +6440,26 @@ mod tests { "default": "d" } "#; - let expected = Details::GetEnumDefault { - symbol: "d".to_string(), - symbols: vec!["a".to_string(), "b".to_string(), "c".to_string()], - } - .to_string(); - let result = Schema::parse_str(schema_str); - assert!(result.is_err()); - let err = result - .map_err(|e| e.to_string()) - .err() - .unwrap_or_else(|| "unexpected".to_string()); - assert_eq!(expected, err); + let expected = Details::GetEnumDefault { + symbol: "d".to_string(), + symbols: vec!["a".to_string(), "b".to_string(), "c".to_string()], + } + .to_string(); + let result = Schema::parse_str(schema_str); + assert!(result.is_err()); + let err = result + .map_err(|e| e.to_string()) + .err() + .unwrap_or_else(|| "unexpected".to_string()); + assert_eq!(expected, err); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3862_get_aliases() -> TestResult { - // Test for Record - let schema_str = r#" + #[test] + fn test_avro_3862_get_aliases() -> TestResult { + // Test for Record + let schema_str = r#" { "name": "record1", "namespace": "ns1", @@ -6383,14 +6471,14 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema_str)?; - let expected = vec![Alias::new("ns1.r1")?, Alias::new("ns2.r2")?]; - match schema.aliases() { - Some(aliases) => assert_eq!(aliases, &expected), - None => panic!("Expected Some({expected:?}), got None"), - } + let schema = Schema::parse_str(schema_str)?; + let expected = vec![Alias::new("ns1.r1")?, Alias::new("ns2.r2")?]; + match schema.aliases() { + Some(aliases) => assert_eq!(aliases, &expected), + None => panic!("Expected Some({expected:?}), got None"), + } - let schema_str = r#" + let schema_str = r#" { "name": "record1", "namespace": "ns1", @@ -6401,14 +6489,14 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema_str)?; - match schema.aliases() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + let schema = Schema::parse_str(schema_str)?; + match schema.aliases() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - // Test for Enum - let schema_str = r#" + // Test for Enum + let schema_str = r#" { "name": "enum1", "namespace": "ns1", @@ -6417,14 +6505,14 @@ mod tests { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; - let expected = vec![Alias::new("ns1.en1")?, Alias::new("ns2.en2")?]; - match schema.aliases() { - Some(aliases) => assert_eq!(aliases, &expected), - None => panic!("Expected Some({expected:?}), got None"), - } + let schema = Schema::parse_str(schema_str)?; + let expected = vec![Alias::new("ns1.en1")?, Alias::new("ns2.en2")?]; + match schema.aliases() { + Some(aliases) => assert_eq!(aliases, &expected), + None => panic!("Expected Some({expected:?}), got None"), + } - let schema_str = r#" + let schema_str = r#" { "name": "enum1", "namespace": "ns1", @@ -6432,14 +6520,14 @@ mod tests { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; - match schema.aliases() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + let schema = Schema::parse_str(schema_str)?; + match schema.aliases() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - // Test for Fixed - let schema_str = r#" + // Test for Fixed + let schema_str = r#" { "name": "fixed1", "namespace": "ns1", @@ -6448,14 +6536,14 @@ mod tests { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; - let expected = vec![Alias::new("ns1.fx1")?, Alias::new("ns2.fx2")?]; - match schema.aliases() { - Some(aliases) => assert_eq!(aliases, &expected), - None => panic!("Expected Some({expected:?}), got None"), - } + let schema = Schema::parse_str(schema_str)?; + let expected = vec![Alias::new("ns1.fx1")?, Alias::new("ns2.fx2")?]; + match schema.aliases() { + Some(aliases) => assert_eq!(aliases, &expected), + None => panic!("Expected Some({expected:?}), got None"), + } - let schema_str = r#" + let schema_str = r#" { "name": "fixed1", "namespace": "ns1", @@ -6463,26 +6551,26 @@ mod tests { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; - match schema.aliases() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + let schema = Schema::parse_str(schema_str)?; + match schema.aliases() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - // Test for non-named type - let schema = Schema::Int; - match schema.aliases() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + // Test for non-named type + let schema = Schema::Int; + match schema.aliases() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3862_get_doc() -> TestResult { - // Test for Record - let schema_str = r#" + #[test] + fn test_avro_3862_get_doc() -> TestResult { + // Test for Record + let schema_str = r#" { "name": "record1", "type": "record", @@ -6493,14 +6581,14 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema_str)?; - let expected = "Record Document"; - match schema.doc() { - Some(doc) => assert_eq!(doc, expected), - None => panic!("Expected Some({expected:?}), got None"), - } + let schema = Schema::parse_str(schema_str)?; + let expected = "Record Document"; + match schema.doc() { + Some(doc) => assert_eq!(doc, expected), + None => panic!("Expected Some({expected:?}), got None"), + } - let schema_str = r#" + let schema_str = r#" { "name": "record1", "type": "record", @@ -6510,14 +6598,14 @@ mod tests { ] } "#; - let schema = Schema::parse_str(schema_str)?; - match schema.doc() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + let schema = Schema::parse_str(schema_str)?; + match schema.doc() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - // Test for Enum - let schema_str = r#" + // Test for Enum + let schema_str = r#" { "name": "enum1", "type": "enum", @@ -6525,28 +6613,28 @@ mod tests { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; - let expected = "Enum Document"; - match schema.doc() { - Some(doc) => assert_eq!(doc, expected), - None => panic!("Expected Some({expected:?}), got None"), - } + let schema = Schema::parse_str(schema_str)?; + let expected = "Enum Document"; + match schema.doc() { + Some(doc) => assert_eq!(doc, expected), + None => panic!("Expected Some({expected:?}), got None"), + } - let schema_str = r#" + let schema_str = r#" { "name": "enum1", "type": "enum", "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; - match schema.doc() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + let schema = Schema::parse_str(schema_str)?; + match schema.doc() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - // Test for Fixed - let schema_str = r#" + // Test for Fixed + let schema_str = r#" { "name": "fixed1", "type": "fixed", @@ -6554,171 +6642,171 @@ mod tests { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; - let expected = "Fixed Document"; - match schema.doc() { - Some(doc) => assert_eq!(doc, expected), - None => panic!("Expected Some({expected:?}), got None"), - } + let schema = Schema::parse_str(schema_str)?; + let expected = "Fixed Document"; + match schema.doc() { + Some(doc) => assert_eq!(doc, expected), + None => panic!("Expected Some({expected:?}), got None"), + } - let schema_str = r#" + let schema_str = r#" { "name": "fixed1", "type": "fixed", "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; - match schema.doc() { - None => (), - some => panic!("Expected None, got {some:?}"), - } - - // Test for non-named type - let schema = Schema::Int; - match schema.doc() { - None => (), - some => panic!("Expected None, got {some:?}"), - } + let schema = Schema::parse_str(schema_str)?; + match schema.doc() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - Ok(()) - } + // Test for non-named type + let schema = Schema::Int; + match schema.doc() { + None => (), + some => panic!("Expected None, got {some:?}"), + } - #[test] - fn avro_3886_serialize_attributes() -> TestResult { - let attributes = BTreeMap::from([ - ("string_key".into(), "value".into()), - ("number_key".into(), 1.23.into()), - ("null_key".into(), Value::Null), - ( - "array_key".into(), - Value::Array(vec![1.into(), 2.into(), 3.into()]), - ), - ("object_key".into(), Value::Object(Map::default())), - ]); - - // Test serialize enum attributes - let schema = Schema::Enum(EnumSchema { - name: Name::new("a")?, - aliases: None, - doc: None, - symbols: vec![], - default: None, - attributes: attributes.clone(), - }); - let serialized = serde_json::to_string(&schema)?; - assert_eq!( - r#"{"type":"enum","name":"a","symbols":[],"array_key":[1,2,3],"null_key":null,"number_key":1.23,"object_key":{},"string_key":"value"}"#, - &serialized - ); + Ok(()) + } - // Test serialize fixed custom_attributes - let schema = Schema::Fixed(FixedSchema { - name: Name::new("a")?, - aliases: None, - doc: None, - size: 1, - default: None, - attributes: attributes.clone(), - }); - let serialized = serde_json::to_string(&schema)?; - assert_eq!( - r#"{"type":"fixed","name":"a","size":1,"array_key":[1,2,3],"null_key":null,"number_key":1.23,"object_key":{},"string_key":"value"}"#, - &serialized - ); + #[test] + fn avro_3886_serialize_attributes() -> TestResult { + let attributes = BTreeMap::from([ + ("string_key".into(), "value".into()), + ("number_key".into(), 1.23.into()), + ("null_key".into(), Value::Null), + ( + "array_key".into(), + Value::Array(vec![1.into(), 2.into(), 3.into()]), + ), + ("object_key".into(), Value::Object(Map::default())), + ]); + + // Test serialize enum attributes + let schema = Schema::Enum(EnumSchema { + name: Name::new("a")?, + aliases: None, + doc: None, + symbols: vec![], + default: None, + attributes: attributes.clone(), + }); + let serialized = serde_json::to_string(&schema)?; + assert_eq!( + r#"{"type":"enum","name":"a","symbols":[],"array_key":[1,2,3],"null_key":null,"number_key":1.23,"object_key":{},"string_key":"value"}"#, + &serialized + ); - // Test serialize record custom_attributes - let schema = Schema::Record(RecordSchema { - name: Name::new("a")?, - aliases: None, - doc: None, - fields: vec![], - lookup: BTreeMap::new(), - attributes, - }); - let serialized = serde_json::to_string(&schema)?; - assert_eq!( - r#"{"type":"record","name":"a","fields":[],"array_key":[1,2,3],"null_key":null,"number_key":1.23,"object_key":{},"string_key":"value"}"#, - &serialized - ); + // Test serialize fixed custom_attributes + let schema = Schema::Fixed(FixedSchema { + name: Name::new("a")?, + aliases: None, + doc: None, + size: 1, + default: None, + attributes: attributes.clone(), + }); + let serialized = serde_json::to_string(&schema)?; + assert_eq!( + r#"{"type":"fixed","name":"a","size":1,"array_key":[1,2,3],"null_key":null,"number_key":1.23,"object_key":{},"string_key":"value"}"#, + &serialized + ); - Ok(()) - } + // Test serialize record custom_attributes + let schema = Schema::Record(RecordSchema { + name: Name::new("a")?, + aliases: None, + doc: None, + fields: vec![], + lookup: BTreeMap::new(), + attributes, + }); + let serialized = serde_json::to_string(&schema)?; + assert_eq!( + r#"{"type":"record","name":"a","fields":[],"array_key":[1,2,3],"null_key":null,"number_key":1.23,"object_key":{},"string_key":"value"}"#, + &serialized + ); - /// A test cases showing that names and namespaces can be constructed - /// entirely by underscores. - #[test] - fn test_avro_3897_funny_valid_names_and_namespaces() -> TestResult { - for funny_name in ["_", "_._", "__._", "_.__", "_._._"] { - let name = Name::new(funny_name); - assert!(name.is_ok()); + Ok(()) } - Ok(()) - } - #[test] - fn test_avro_3896_decimal_schema() -> TestResult { - // bytes decimal, represented as native logical type. - let schema = json!( - { - "type": "bytes", - "name": "BytesDecimal", - "logicalType": "decimal", - "size": 38, - "precision": 9, - "scale": 2 - }); - let parse_result = Schema::parse(&schema)?; - assert!(matches!( - parse_result, - Schema::Decimal(DecimalSchema { - precision: 9, - scale: 2, - .. - }) - )); + /// A test cases showing that names and namespaces can be constructed + /// entirely by underscores. + #[test] + fn test_avro_3897_funny_valid_names_and_namespaces() -> TestResult { + for funny_name in ["_", "_._", "__._", "_.__", "_._._"] { + let name = Name::new(funny_name); + assert!(name.is_ok()); + } + Ok(()) + } - // long decimal, represents as native complex type. - let schema = json!( - { - "type": "long", - "name": "LongDecimal", - "logicalType": "decimal" - }); - let parse_result = Schema::parse(&schema)?; - // assert!(matches!(parse_result, Schema::Long)); - assert_eq!(parse_result, Schema::Long); + #[test] + fn test_avro_3896_decimal_schema() -> TestResult { + // bytes decimal, represented as native logical type. + let schema = json!( + { + "type": "bytes", + "name": "BytesDecimal", + "logicalType": "decimal", + "size": 38, + "precision": 9, + "scale": 2 + }); + let parse_result = Schema::parse(&schema)?; + assert!(matches!( + parse_result, + Schema::Decimal(DecimalSchema { + precision: 9, + scale: 2, + .. + }) + )); - Ok(()) - } + // long decimal, represents as native complex type. + let schema = json!( + { + "type": "long", + "name": "LongDecimal", + "logicalType": "decimal" + }); + let parse_result = Schema::parse(&schema)?; + // assert!(matches!(parse_result, Schema::Long)); + assert_eq!(parse_result, Schema::Long); - #[test] - fn avro_3896_uuid_schema_for_string() -> TestResult { - // string uuid, represents as native logical type. - let schema = json!( - { - "type": "string", - "name": "StringUUID", - "logicalType": "uuid" - }); - let parse_result = Schema::parse(&schema)?; - assert_eq!(parse_result, Schema::Uuid); + Ok(()) + } - Ok(()) - } + #[test] + fn avro_3896_uuid_schema_for_string() -> TestResult { + // string uuid, represents as native logical type. + let schema = json!( + { + "type": "string", + "name": "StringUUID", + "logicalType": "uuid" + }); + let parse_result = Schema::parse(&schema)?; + assert_eq!(parse_result, Schema::Uuid); - #[test] - #[serial(serde_is_human_readable)] - fn avro_rs_53_uuid_with_fixed() -> TestResult { - #[derive(Debug, Serialize, Deserialize)] - struct Comment { - id: crate::Uuid, + Ok(()) } - impl AvroSchema for Comment { - fn get_schema() -> Schema { - Schema::parse_str( - r#"{ + #[test] + #[serial(serde_is_human_readable)] + fn avro_rs_53_uuid_with_fixed() -> TestResult { + #[derive(Debug, Serialize, Deserialize)] + struct Comment { + id: crate::Uuid, + } + + impl AvroSchema for Comment { + fn get_schema() -> Schema { + Schema::parse_str( + r#"{ "type" : "record", "name" : "Comment", "fields" : [ { @@ -6731,123 +6819,123 @@ mod tests { } } ] }"#, - ) - .expect("Invalid Comment Avro schema") + ) + .expect("Invalid Comment Avro schema") + } } - } - let payload = Comment { - id: "de2df598-9948-4988-b00a-a41c0e287398".parse()?, - }; - let mut buffer = Vec::new(); + let payload = Comment { + id: "de2df598-9948-4988-b00a-a41c0e287398".parse()?, + }; + let mut buffer = Vec::new(); - // serialize the Uuid as String - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let bytes = SpecificSingleObjectWriter::::with_capacity(64)? - .write_ref(&payload, &mut buffer)?; - assert_eq!(bytes, 47); + // serialize the Uuid as String + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let bytes = SpecificSingleObjectWriter::::with_capacity(64)? + .write_ref(&payload, &mut buffer)?; + assert_eq!(bytes, 47); - // serialize the Uuid as Bytes - crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); - let bytes = SpecificSingleObjectWriter::::with_capacity(64)? - .write_ref(&payload, &mut buffer)?; - assert_eq!(bytes, 27); + // serialize the Uuid as Bytes + crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); + let bytes = SpecificSingleObjectWriter::::with_capacity(64)? + .write_ref(&payload, &mut buffer)?; + assert_eq!(bytes, 27); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3926_uuid_schema_for_fixed_with_size_16() -> TestResult { - let schema = json!( - { - "type": "fixed", - "name": "FixedUUID", - "size": 16, - "logicalType": "uuid" - }); - let parse_result = Schema::parse(&schema)?; - assert_eq!(parse_result, Schema::Uuid); - assert_not_logged( - r#"Ignoring uuid logical type for a Fixed schema because its size (6) is not 16! Schema: Fixed(FixedSchema { name: Name { name: "FixedUUID", namespace: None }, aliases: None, doc: None, size: 6, attributes: {"logicalType": String("uuid")} })"#, - ); + #[test] + fn avro_3926_uuid_schema_for_fixed_with_size_16() -> TestResult { + let schema = json!( + { + "type": "fixed", + "name": "FixedUUID", + "size": 16, + "logicalType": "uuid" + }); + let parse_result = Schema::parse(&schema)?; + assert_eq!(parse_result, Schema::Uuid); + assert_not_logged( + r#"Ignoring uuid logical type for a Fixed schema because its size (6) is not 16! Schema: Fixed(FixedSchema { name: Name { name: "FixedUUID", namespace: None }, aliases: None, doc: None, size: 6, attributes: {"logicalType": String("uuid")} })"#, + ); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3926_uuid_schema_for_fixed_with_size_different_than_16() -> TestResult { - let schema = json!( - { - "type": "fixed", - "name": "FixedUUID", - "size": 6, - "logicalType": "uuid" - }); - let parse_result = Schema::parse(&schema)?; - - assert_eq!( - parse_result, - Schema::Fixed(FixedSchema { - name: Name::new("FixedUUID")?, - aliases: None, - doc: None, - size: 6, - default: None, - attributes: BTreeMap::from([("logicalType".to_string(), "uuid".into())]), - }) - ); - assert_logged( - r#"Ignoring uuid logical type for a Fixed schema because its size (6) is not 16! Schema: Fixed(FixedSchema { name: Name { name: "FixedUUID", namespace: None }, aliases: None, doc: None, size: 6, default: None, attributes: {"logicalType": String("uuid")} })"#, - ); + #[test] + fn avro_3926_uuid_schema_for_fixed_with_size_different_than_16() -> TestResult { + let schema = json!( + { + "type": "fixed", + "name": "FixedUUID", + "size": 6, + "logicalType": "uuid" + }); + let parse_result = Schema::parse(&schema)?; - Ok(()) - } + assert_eq!( + parse_result, + Schema::Fixed(FixedSchema { + name: Name::new("FixedUUID")?, + aliases: None, + doc: None, + size: 6, + default: None, + attributes: BTreeMap::from([("logicalType".to_string(), "uuid".into())]), + }) + ); + assert_logged( + r#"Ignoring uuid logical type for a Fixed schema because its size (6) is not 16! Schema: Fixed(FixedSchema { name: Name { name: "FixedUUID", namespace: None }, aliases: None, doc: None, size: 6, default: None, attributes: {"logicalType": String("uuid")} })"#, + ); - #[test] - fn test_avro_3896_timestamp_millis_schema() -> TestResult { - // long timestamp-millis, represents as native logical type. - let schema = json!( - { - "type": "long", - "name": "LongTimestampMillis", - "logicalType": "timestamp-millis" - }); - let parse_result = Schema::parse(&schema)?; - assert_eq!(parse_result, Schema::TimestampMillis); - - // int timestamp-millis, represents as native complex type. - let schema = json!( - { - "type": "int", - "name": "IntTimestampMillis", - "logicalType": "timestamp-millis" - }); - let parse_result = Schema::parse(&schema)?; - assert_eq!(parse_result, Schema::Int); + Ok(()) + } - Ok(()) - } + #[test] + fn test_avro_3896_timestamp_millis_schema() -> TestResult { + // long timestamp-millis, represents as native logical type. + let schema = json!( + { + "type": "long", + "name": "LongTimestampMillis", + "logicalType": "timestamp-millis" + }); + let parse_result = Schema::parse(&schema)?; + assert_eq!(parse_result, Schema::TimestampMillis); - #[test] - fn test_avro_3896_custom_bytes_schema() -> TestResult { - // log type, represents as complex type. - let schema = json!( - { - "type": "bytes", - "name": "BytesLog", - "logicalType": "custom" - }); - let parse_result = Schema::parse(&schema)?; - assert_eq!(parse_result, Schema::Bytes); - assert_eq!(parse_result.custom_attributes(), None); + // int timestamp-millis, represents as native complex type. + let schema = json!( + { + "type": "int", + "name": "IntTimestampMillis", + "logicalType": "timestamp-millis" + }); + let parse_result = Schema::parse(&schema)?; + assert_eq!(parse_result, Schema::Int); - Ok(()) - } + Ok(()) + } + + #[test] + fn test_avro_3896_custom_bytes_schema() -> TestResult { + // log type, represents as complex type. + let schema = json!( + { + "type": "bytes", + "name": "BytesLog", + "logicalType": "custom" + }); + let parse_result = Schema::parse(&schema)?; + assert_eq!(parse_result, Schema::Bytes); + assert_eq!(parse_result.custom_attributes(), None); + + Ok(()) + } - #[test] - fn test_avro_3899_parse_decimal_type() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_avro_3899_parse_decimal_type() -> TestResult { + let schema = Schema::parse_str( + r#"{ "name": "InvalidDecimal", "type": "fixed", "size": 16, @@ -6855,100 +6943,100 @@ mod tests { "precision": 2, "scale": 3 }"#, - )?; - match schema { - Schema::Fixed(fixed_schema) => { - let attrs = fixed_schema.attributes; - let precision = attrs - .get("precision") - .expect("The 'precision' attribute is missing"); - let scale = attrs - .get("scale") - .expect("The 'scale' attribute is missing"); - assert_logged(&format!( - "Ignoring invalid decimal logical type: The decimal precision ({precision}) must be bigger or equal to the scale ({scale})" - )); - } - _ => unreachable!("Expected Schema::Fixed, got {:?}", schema), - } - - let schema = Schema::parse_str( - r#"{ + )?; + match schema { + Schema::Fixed(fixed_schema) => { + let attrs = fixed_schema.attributes; + let precision = attrs + .get("precision") + .expect("The 'precision' attribute is missing"); + let scale = attrs + .get("scale") + .expect("The 'scale' attribute is missing"); + assert_logged(&format!( + "Ignoring invalid decimal logical type: The decimal precision ({precision}) must be bigger or equal to the scale ({scale})" + )); + } + _ => unreachable!("Expected Schema::Fixed, got {:?}", schema), + } + + let schema = Schema::parse_str( + r#"{ "name": "ValidDecimal", "type": "bytes", "logicalType": "decimal", "precision": 3, "scale": 2 }"#, - )?; - match schema { - Schema::Decimal(_) => { - assert_not_logged( - "Ignoring invalid decimal logical type: The decimal precision (2) must be bigger or equal to the scale (3)", - ); + )?; + match schema { + Schema::Decimal(_) => { + assert_not_logged( + "Ignoring invalid decimal logical type: The decimal precision (2) must be bigger or equal to the scale (3)", + ); + } + _ => unreachable!("Expected Schema::Decimal, got {:?}", schema), } - _ => unreachable!("Expected Schema::Decimal, got {:?}", schema), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3920_serialize_record_with_custom_attributes() -> TestResult { - let expected = { - let mut lookup = BTreeMap::new(); - lookup.insert("value".to_owned(), 0); - Schema::Record(RecordSchema { - name: Name { - name: "LongList".to_owned(), - namespace: None, - }, - aliases: Some(vec![Alias::new("LinkedLongs").unwrap()]), - doc: None, - fields: vec![RecordField { - name: "value".to_string(), + #[test] + fn avro_3920_serialize_record_with_custom_attributes() -> TestResult { + let expected = { + let mut lookup = BTreeMap::new(); + lookup.insert("value".to_owned(), 0); + Schema::Record(RecordSchema { + name: Name { + name: "LongList".to_owned(), + namespace: None, + }, + aliases: Some(vec![Alias::new("LinkedLongs").unwrap()]), doc: None, - default: None, - aliases: None, - schema: Schema::Long, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: BTreeMap::from([("field-id".to_string(), 1.into())]), - }], - lookup, - attributes: BTreeMap::from([("custom-attribute".to_string(), "value".into())]), - }) - }; + fields: vec![RecordField { + name: "value".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Long, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: BTreeMap::from([("field-id".to_string(), 1.into())]), + }], + lookup, + attributes: BTreeMap::from([("custom-attribute".to_string(), "value".into())]), + }) + }; - let value = serde_json::to_value(&expected)?; - let serialized = serde_json::to_string(&value)?; - assert_eq!( - r#"{"aliases":["LinkedLongs"],"custom-attribute":"value","fields":[{"field-id":1,"name":"value","type":"long"}],"name":"LongList","type":"record"}"#, - &serialized - ); - assert_eq!(expected, Schema::parse_str(&serialized)?); + let value = serde_json::to_value(&expected)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"aliases":["LinkedLongs"],"custom-attribute":"value","fields":[{"field-id":1,"name":"value","type":"long"}],"name":"LongList","type":"record"}"#, + &serialized + ); + assert_eq!(expected, Schema::parse_str(&serialized)?); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3925_serialize_decimal_inner_fixed() -> TestResult { - let schema = Schema::Decimal(DecimalSchema { - precision: 36, - scale: 10, - inner: Box::new(Schema::Fixed(FixedSchema { - name: Name::new("decimal_36_10").unwrap(), - aliases: None, - doc: None, - size: 16, - default: None, - attributes: Default::default(), - })), - }); + #[test] + fn test_avro_3925_serialize_decimal_inner_fixed() -> TestResult { + let schema = Schema::Decimal(DecimalSchema { + precision: 36, + scale: 10, + inner: Box::new(Schema::Fixed(FixedSchema { + name: Name::new("decimal_36_10").unwrap(), + aliases: None, + doc: None, + size: 16, + default: None, + attributes: Default::default(), + })), + }); - let serialized_json = serde_json::to_string_pretty(&schema)?; + let serialized_json = serde_json::to_string_pretty(&schema)?; - let expected_json = r#"{ + let expected_json = r#"{ "type": "fixed", "name": "decimal_36_10", "size": 16, @@ -6957,97 +7045,97 @@ mod tests { "precision": 36 }"#; - assert_eq!(serialized_json, expected_json); + assert_eq!(serialized_json, expected_json); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3925_serialize_decimal_inner_bytes() -> TestResult { - let schema = Schema::Decimal(DecimalSchema { - precision: 36, - scale: 10, - inner: Box::new(Schema::Bytes), - }); + #[test] + fn test_avro_3925_serialize_decimal_inner_bytes() -> TestResult { + let schema = Schema::Decimal(DecimalSchema { + precision: 36, + scale: 10, + inner: Box::new(Schema::Bytes), + }); - let serialized_json = serde_json::to_string_pretty(&schema)?; + let serialized_json = serde_json::to_string_pretty(&schema)?; - let expected_json = r#"{ + let expected_json = r#"{ "type": "bytes", "logicalType": "decimal", "scale": 10, "precision": 36 }"#; - assert_eq!(serialized_json, expected_json); + assert_eq!(serialized_json, expected_json); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3925_serialize_decimal_inner_invalid() -> TestResult { - let schema = Schema::Decimal(DecimalSchema { - precision: 36, - scale: 10, - inner: Box::new(Schema::String), - }); + #[test] + fn test_avro_3925_serialize_decimal_inner_invalid() -> TestResult { + let schema = Schema::Decimal(DecimalSchema { + precision: 36, + scale: 10, + inner: Box::new(Schema::String), + }); - let serialized_json = serde_json::to_string_pretty(&schema); + let serialized_json = serde_json::to_string_pretty(&schema); - assert!(serialized_json.is_err()); + assert!(serialized_json.is_err()); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3927_serialize_array_with_custom_attributes() -> TestResult { - let expected = Schema::array_with_attributes( - Schema::Long, - BTreeMap::from([("field-id".to_string(), "1".into())]), - ); + #[test] + fn test_avro_3927_serialize_array_with_custom_attributes() -> TestResult { + let expected = Schema::array_with_attributes( + Schema::Long, + BTreeMap::from([("field-id".to_string(), "1".into())]), + ); - let value = serde_json::to_value(&expected)?; - let serialized = serde_json::to_string(&value)?; - assert_eq!( - r#"{"field-id":"1","items":"long","type":"array"}"#, - &serialized - ); - let actual_schema = Schema::parse_str(&serialized)?; - assert_eq!(expected, actual_schema); - assert_eq!( - expected.custom_attributes(), - actual_schema.custom_attributes() - ); + let value = serde_json::to_value(&expected)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"field-id":"1","items":"long","type":"array"}"#, + &serialized + ); + let actual_schema = Schema::parse_str(&serialized)?; + assert_eq!(expected, actual_schema); + assert_eq!( + expected.custom_attributes(), + actual_schema.custom_attributes() + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3927_serialize_map_with_custom_attributes() -> TestResult { - let expected = Schema::map_with_attributes( - Schema::Long, - BTreeMap::from([("field-id".to_string(), "1".into())]), - ); + #[test] + fn test_avro_3927_serialize_map_with_custom_attributes() -> TestResult { + let expected = Schema::map_with_attributes( + Schema::Long, + BTreeMap::from([("field-id".to_string(), "1".into())]), + ); - let value = serde_json::to_value(&expected)?; - let serialized = serde_json::to_string(&value)?; - assert_eq!( - r#"{"field-id":"1","type":"map","values":"long"}"#, - &serialized - ); - let actual_schema = Schema::parse_str(&serialized)?; - assert_eq!(expected, actual_schema); - assert_eq!( - expected.custom_attributes(), - actual_schema.custom_attributes() - ); + let value = serde_json::to_value(&expected)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"field-id":"1","type":"map","values":"long"}"#, + &serialized + ); + let actual_schema = Schema::parse_str(&serialized)?; + assert_eq!(expected, actual_schema); + assert_eq!( + expected.custom_attributes(), + actual_schema.custom_attributes() + ); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3928_parse_int_based_schema_with_default() -> TestResult { - let schema = r#" + #[test] + fn avro_3928_parse_int_based_schema_with_default() -> TestResult { + let schema = r#" { "type": "record", "name": "DateLogicalType", @@ -7058,26 +7146,26 @@ mod tests { } ] }"#; - match Schema::parse_str(schema)? { - Schema::Record(record_schema) => { - assert_eq!(record_schema.fields.len(), 1); - let field = record_schema.fields.first().unwrap(); - assert_eq!(field.name, "birthday"); - assert_eq!(field.schema, Schema::Date); - assert_eq!( - types::Value::from(field.default.clone().unwrap()), - types::Value::Int(1681601653) - ); + match Schema::parse_str(schema)? { + Schema::Record(record_schema) => { + assert_eq!(record_schema.fields.len(), 1); + let field = record_schema.fields.first().unwrap(); + assert_eq!(field.name, "birthday"); + assert_eq!(field.schema, Schema::Date); + assert_eq!( + types::Value::from(field.default.clone().unwrap()), + types::Value::Int(1681601653) + ); + } + _ => unreachable!("Expected Schema::Record"), } - _ => unreachable!("Expected Schema::Record"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3946_union_with_single_type() -> TestResult { - let schema = r#" + #[test] + fn avro_3946_union_with_single_type() -> TestResult { + let schema = r#" { "type": "record", "name": "Issue", @@ -7090,20 +7178,20 @@ mod tests { ] }"#; - let _ = Schema::parse_str(schema)?; + let _ = Schema::parse_str(schema)?; - assert_logged( - "Union schema with just one member! Consider dropping the union! \ + assert_logged( + "Union schema with just one member! Consider dropping the union! \ Please enable debug logging to find out which Record schema \ declares the union with 'RUST_LOG=apache_avro::schema=debug'.", - ); + ); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3946_union_without_any_types() -> TestResult { - let schema = r#" + #[test] + fn avro_3946_union_without_any_types() -> TestResult { + let schema = r#" { "type": "record", "name": "Issue", @@ -7116,42 +7204,42 @@ mod tests { ] }"#; - let _ = Schema::parse_str(schema)?; + let _ = Schema::parse_str(schema)?; - assert_logged( - "Union schemas should have at least two members! \ + assert_logged( + "Union schemas should have at least two members! \ Please enable debug logging to find out which Record schema \ declares the union with 'RUST_LOG=apache_avro::schema=debug'.", - ); + ); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3965_fixed_schema_with_default_bigger_than_size() -> TestResult { - match Schema::parse_str( - r#"{ + #[test] + fn avro_3965_fixed_schema_with_default_bigger_than_size() -> TestResult { + match Schema::parse_str( + r#"{ "type": "fixed", "name": "test", "size": 1, "default": "123456789" }"#, - ) { - Ok(_schema) => panic!("Must fail!"), - Err(err) => { - assert_eq!( - err.to_string(), - "Fixed schema's default value length (9) does not match its size (1)" - ); + ) { + Ok(_schema) => panic!("Must fail!"), + Err(err) => { + assert_eq!( + err.to_string(), + "Fixed schema's default value length (9) does not match its size (1)" + ); + } } - } - Ok(()) - } + Ok(()) + } - #[test] - fn avro_4004_canonical_form_strip_logical_types() -> TestResult { - let schema_str = r#" + #[test] + fn avro_4004_canonical_form_strip_logical_types() -> TestResult { + let schema_str = r#" { "type": "record", "name": "test", @@ -7162,21 +7250,21 @@ mod tests { ] }"#; - let schema = Schema::parse_str(schema_str)?; - let canonical_form = schema.canonical_form(); - let fp_rabin = schema.fingerprint::(); - assert_eq!( - r#"{"name":"test","type":"record","fields":[{"name":"a","type":"long"},{"name":"b","type":"string"},{"name":"c","type":{"type":"long"}}]}"#, - canonical_form - ); - assert_eq!("92f2ccef718c6754", fp_rabin.to_string()); - Ok(()) - } + let schema = Schema::parse_str(schema_str)?; + let canonical_form = schema.canonical_form(); + let fp_rabin = schema.fingerprint::(); + assert_eq!( + r#"{"name":"test","type":"record","fields":[{"name":"a","type":"long"},{"name":"b","type":"string"},{"name":"c","type":{"type":"long"}}]}"#, + canonical_form + ); + assert_eq!("92f2ccef718c6754", fp_rabin.to_string()); + Ok(()) + } - #[test] - fn avro_4055_should_fail_to_parse_invalid_schema() -> TestResult { - // This is invalid because the record type should be inside the type field. - let invalid_schema_str = r#" + #[test] + fn avro_4055_should_fail_to_parse_invalid_schema() -> TestResult { + // This is invalid because the record type should be inside the type field. + let invalid_schema_str = r#" { "type": "record", "name": "SampleSchema", @@ -7196,14 +7284,14 @@ mod tests { ] }"#; - let schema = Schema::parse_str(invalid_schema_str); - assert!(schema.is_err()); - assert_eq!( - schema.unwrap_err().to_string(), - "Invalid schema: There is no type called 'record', if you meant to define a non-primitive schema, it should be defined inside `type` attribute. Please review the specification" - ); + let schema = Schema::parse_str(invalid_schema_str); + assert!(schema.is_err()); + assert_eq!( + schema.unwrap_err().to_string(), + "Invalid schema: There is no type called 'record', if you meant to define a non-primitive schema, it should be defined inside `type` attribute. Please review the specification" + ); - let valid_schema = r#" + let valid_schema = r#" { "type": "record", "name": "SampleSchema", @@ -7225,9 +7313,10 @@ mod tests { } ] }"#; - let schema = Schema::parse_str(valid_schema); - assert!(schema.is_ok()); + let schema = Schema::parse_str(valid_schema); + assert!(schema.is_ok()); - Ok(()) + Ok(()) + } } } diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index 3fec12af..abee6156 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -16,831 +16,862 @@ // under the License. //! Logic for checking schema compatibility -use crate::{ - error::CompatibilityError, - schema::{EnumSchema, FixedSchema, RecordSchema, Schema, SchemaKind}, -}; -use std::{ - collections::{HashSet, hash_map::DefaultHasher}, - hash::Hasher, - ptr, -}; - -fn match_ref_schemas( - writers_schema: &Schema, - readers_schema: &Schema, -) -> Result<(), CompatibilityError> { - match (readers_schema, writers_schema) { - (Schema::Ref { name: r_name }, Schema::Ref { name: w_name }) => { - if r_name == w_name { - Ok(()) - } else { - Err(CompatibilityError::NameMismatch { - writer_name: w_name.fullname(None), - reader_name: r_name.fullname(None), - }) + +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + headers::tokio => headers::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + types::tokio => types::sync, + #[tokio::test] => #[test] + ); + } +)] +mod schema_compatibility { + + use crate::{ + error::tokio::CompatibilityError, + schema::tokio::{EnumSchema, FixedSchema, RecordSchema, Schema, SchemaKind}, + }; + use std::{ + collections::{HashSet, hash_map::DefaultHasher}, + hash::Hasher, + ptr, + }; + + fn match_ref_schemas( + writers_schema: &Schema, + readers_schema: &Schema, + ) -> Result<(), CompatibilityError> { + match (readers_schema, writers_schema) { + (Schema::Ref { name: r_name }, Schema::Ref { name: w_name }) => { + if r_name == w_name { + Ok(()) + } else { + Err(CompatibilityError::NameMismatch { + writer_name: w_name.fullname(None), + reader_name: r_name.fullname(None), + }) + } } + _ => Err(CompatibilityError::WrongType { + writer_schema_type: format!("{writers_schema:#?}"), + reader_schema_type: format!("{readers_schema:#?}"), + }), } - _ => Err(CompatibilityError::WrongType { - writer_schema_type: format!("{writers_schema:#?}"), - reader_schema_type: format!("{readers_schema:#?}"), - }), } -} -pub struct SchemaCompatibility; + pub struct SchemaCompatibility; -struct Checker { - recursion: HashSet<(u64, u64)>, -} + struct Checker { + recursion: HashSet<(u64, u64)>, + } -impl Checker { - /// Create a new checker, with recursion set to an empty set. - pub(crate) fn new() -> Self { - Self { - recursion: HashSet::new(), + impl Checker { + /// Create a new checker, with recursion set to an empty set. + pub(crate) fn new() -> Self { + Self { + recursion: HashSet::new(), + } } - } - pub(crate) fn can_read( - &mut self, - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - self.full_match_schemas(writers_schema, readers_schema) - } + pub(crate) fn can_read( + &mut self, + writers_schema: &Schema, + readers_schema: &Schema, + ) -> Result<(), CompatibilityError> { + self.full_match_schemas(writers_schema, readers_schema) + } - pub(crate) fn full_match_schemas( - &mut self, - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - if self.recursion_in_progress(writers_schema, readers_schema) { - return Ok(()); - } - - SchemaCompatibility::match_schemas(writers_schema, readers_schema)?; - - let w_type = SchemaKind::from(writers_schema); - let r_type = SchemaKind::from(readers_schema); - - if w_type != SchemaKind::Union - && (r_type.is_primitive() - || r_type == SchemaKind::Fixed - || r_type == SchemaKind::Uuid - || r_type == SchemaKind::Date - || r_type == SchemaKind::TimeMillis - || r_type == SchemaKind::TimeMicros - || r_type == SchemaKind::TimestampMillis - || r_type == SchemaKind::TimestampMicros - || r_type == SchemaKind::TimestampNanos - || r_type == SchemaKind::LocalTimestampMillis - || r_type == SchemaKind::LocalTimestampMicros - || r_type == SchemaKind::LocalTimestampNanos) - { - return Ok(()); - } - - match r_type { - SchemaKind::Ref => match_ref_schemas(writers_schema, readers_schema), - SchemaKind::Record => self.match_record_schemas(writers_schema, readers_schema), - SchemaKind::Map => { - if let Schema::Map(w_m) = writers_schema { - match readers_schema { - Schema::Map(r_m) => self.full_match_schemas(&w_m.types, &r_m.types), - _ => Err(CompatibilityError::WrongType { - writer_schema_type: format!("{writers_schema:#?}"), - reader_schema_type: format!("{readers_schema:#?}"), - }), + pub(crate) fn full_match_schemas( + &mut self, + writers_schema: &Schema, + readers_schema: &Schema, + ) -> Result<(), CompatibilityError> { + if self.recursion_in_progress(writers_schema, readers_schema) { + return Ok(()); + } + + SchemaCompatibility::match_schemas(writers_schema, readers_schema)?; + + let w_type = SchemaKind::from(writers_schema); + let r_type = SchemaKind::from(readers_schema); + + if w_type != SchemaKind::Union + && (r_type.is_primitive() + || r_type == SchemaKind::Fixed + || r_type == SchemaKind::Uuid + || r_type == SchemaKind::Date + || r_type == SchemaKind::TimeMillis + || r_type == SchemaKind::TimeMicros + || r_type == SchemaKind::TimestampMillis + || r_type == SchemaKind::TimestampMicros + || r_type == SchemaKind::TimestampNanos + || r_type == SchemaKind::LocalTimestampMillis + || r_type == SchemaKind::LocalTimestampMicros + || r_type == SchemaKind::LocalTimestampNanos) + { + return Ok(()); + } + + match r_type { + SchemaKind::Ref => match_ref_schemas(writers_schema, readers_schema), + SchemaKind::Record => self.match_record_schemas(writers_schema, readers_schema), + SchemaKind::Map => { + if let Schema::Map(w_m) = writers_schema { + match readers_schema { + Schema::Map(r_m) => self.full_match_schemas(&w_m.types, &r_m.types), + _ => Err(CompatibilityError::WrongType { + writer_schema_type: format!("{writers_schema:#?}"), + reader_schema_type: format!("{readers_schema:#?}"), + }), + } + } else { + Err(CompatibilityError::TypeExpected { + schema_type: String::from("writers_schema"), + expected_type: vec![SchemaKind::Record], + }) } - } else { - Err(CompatibilityError::TypeExpected { - schema_type: String::from("writers_schema"), - expected_type: vec![SchemaKind::Record], - }) } - } - SchemaKind::Array => { - if let Schema::Array(w_a) = writers_schema { - match readers_schema { - Schema::Array(r_a) => self.full_match_schemas(&w_a.items, &r_a.items), - _ => Err(CompatibilityError::WrongType { - writer_schema_type: format!("{writers_schema:#?}"), - reader_schema_type: format!("{readers_schema:#?}"), - }), + SchemaKind::Array => { + if let Schema::Array(w_a) = writers_schema { + match readers_schema { + Schema::Array(r_a) => self.full_match_schemas(&w_a.items, &r_a.items), + _ => Err(CompatibilityError::WrongType { + writer_schema_type: format!("{writers_schema:#?}"), + reader_schema_type: format!("{readers_schema:#?}"), + }), + } + } else { + Err(CompatibilityError::TypeExpected { + schema_type: String::from("writers_schema"), + expected_type: vec![SchemaKind::Array], + }) } - } else { - Err(CompatibilityError::TypeExpected { - schema_type: String::from("writers_schema"), - expected_type: vec![SchemaKind::Array], - }) } - } - SchemaKind::Union => self.match_union_schemas(writers_schema, readers_schema), - SchemaKind::Enum => { - // reader's symbols must contain all writer's symbols - if let Schema::Enum(EnumSchema { - symbols: w_symbols, .. - }) = writers_schema - { + SchemaKind::Union => self.match_union_schemas(writers_schema, readers_schema), + SchemaKind::Enum => { + // reader's symbols must contain all writer's symbols if let Schema::Enum(EnumSchema { - symbols: r_symbols, .. - }) = readers_schema + symbols: w_symbols, .. + }) = writers_schema { - if w_symbols.iter().all(|e| r_symbols.contains(e)) { - return Ok(()); + if let Schema::Enum(EnumSchema { + symbols: r_symbols, .. + }) = readers_schema + { + if w_symbols.iter().all(|e| r_symbols.contains(e)) { + return Ok(()); + } } } + Err(CompatibilityError::MissingSymbols) } - Err(CompatibilityError::MissingSymbols) - } - _ => { - if w_type == SchemaKind::Union { - if let Schema::Union(r) = writers_schema { - if r.schemas.len() == 1 { - return self.full_match_schemas(&r.schemas[0], readers_schema); + _ => { + if w_type == SchemaKind::Union { + if let Schema::Union(r) = writers_schema { + if r.schemas.len() == 1 { + return self.full_match_schemas(&r.schemas[0], readers_schema); + } } } + Err(CompatibilityError::Inconclusive(String::from( + "writers_schema", + ))) } - Err(CompatibilityError::Inconclusive(String::from( - "writers_schema", - ))) } } - } - fn match_record_schemas( - &mut self, - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - let w_type = SchemaKind::from(writers_schema); + fn match_record_schemas( + &mut self, + writers_schema: &Schema, + readers_schema: &Schema, + ) -> Result<(), CompatibilityError> { + let w_type = SchemaKind::from(writers_schema); - if w_type == SchemaKind::Union { - return Err(CompatibilityError::TypeExpected { - schema_type: String::from("writers_schema"), - expected_type: vec![SchemaKind::Record], - }); - } + if w_type == SchemaKind::Union { + return Err(CompatibilityError::TypeExpected { + schema_type: String::from("writers_schema"), + expected_type: vec![SchemaKind::Record], + }); + } - if let Schema::Record(RecordSchema { - fields: w_fields, - lookup: w_lookup, - .. - }) = writers_schema - { if let Schema::Record(RecordSchema { - fields: r_fields, .. - }) = readers_schema + fields: w_fields, + lookup: w_lookup, + .. + }) = writers_schema { - for field in r_fields.iter() { - // get all field names in a vector (field.name + aliases) - let mut fields_names = vec![&field.name]; - if let Some(ref aliases) = field.aliases { - for alias in aliases { - fields_names.push(alias); - } - } - - // Check whether any of the possible fields names are in the writer schema. - // If the field was found, then it must have the exact same name with the writer field, - // otherwise we would have a false positive with the writers aliases - let position = fields_names.iter().find_map(|field_name| { - if let Some(pos) = w_lookup.get(*field_name) { - if &w_fields[*pos].name == *field_name { - return Some(pos); + if let Schema::Record(RecordSchema { + fields: r_fields, .. + }) = readers_schema + { + for field in r_fields.iter() { + // get all field names in a vector (field.name + aliases) + let mut fields_names = vec![&field.name]; + if let Some(ref aliases) = field.aliases { + for alias in aliases { + fields_names.push(alias); } } - None - }); - match position { - Some(pos) => { - if let Err(err) = - self.full_match_schemas(&w_fields[*pos].schema, &field.schema) - { - return Err(CompatibilityError::FieldTypeMismatch( - field.name.clone(), - Box::new(err), - )); + // Check whether any of the possible fields names are in the writer schema. + // If the field was found, then it must have the exact same name with the writer field, + // otherwise we would have a false positive with the writers aliases + let position = fields_names.iter().find_map(|field_name| { + if let Some(pos) = w_lookup.get(*field_name) { + if &w_fields[*pos].name == *field_name { + return Some(pos); + } } - } - _ => { - if field.default.is_none() { - return Err(CompatibilityError::MissingDefaultValue( - field.name.clone(), - )); + None + }); + + match position { + Some(pos) => { + if let Err(err) = + self.full_match_schemas(&w_fields[*pos].schema, &field.schema) + { + return Err(CompatibilityError::FieldTypeMismatch( + field.name.clone(), + Box::new(err), + )); + } + } + _ => { + if field.default.is_none() { + return Err(CompatibilityError::MissingDefaultValue( + field.name.clone(), + )); + } } } } } } + Ok(()) } - Ok(()) - } - fn match_union_schemas( - &mut self, - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - if let Schema::Union(u) = writers_schema { - if u.schemas - .iter() - .all(|schema| self.full_match_schemas(schema, readers_schema).is_ok()) - { - return Ok(()); - } else { - return Err(CompatibilityError::MissingUnionElements); - } - } else if let Schema::Union(u) = readers_schema { - // This check is needed because the writer_schema can be not union - // but the type can be contain in the union of the reader schema - // e.g. writer_schema is string and reader_schema is [string, int] - if u.schemas - .iter() - .any(|schema| self.full_match_schemas(writers_schema, schema).is_ok()) - { - return Ok(()); + fn match_union_schemas( + &mut self, + writers_schema: &Schema, + readers_schema: &Schema, + ) -> Result<(), CompatibilityError> { + if let Schema::Union(u) = writers_schema { + if u.schemas + .iter() + .all(|schema| self.full_match_schemas(schema, readers_schema).is_ok()) + { + return Ok(()); + } else { + return Err(CompatibilityError::MissingUnionElements); + } + } else if let Schema::Union(u) = readers_schema { + // This check is needed because the writer_schema can be not union + // but the type can be contain in the union of the reader schema + // e.g. writer_schema is string and reader_schema is [string, int] + if u.schemas + .iter() + .any(|schema| self.full_match_schemas(writers_schema, schema).is_ok()) + { + return Ok(()); + } } + Err(CompatibilityError::MissingUnionElements) } - Err(CompatibilityError::MissingUnionElements) - } - - fn recursion_in_progress(&mut self, writers_schema: &Schema, readers_schema: &Schema) -> bool { - let mut hasher = DefaultHasher::new(); - ptr::hash(writers_schema, &mut hasher); - let w_hash = hasher.finish(); - - hasher = DefaultHasher::new(); - ptr::hash(readers_schema, &mut hasher); - let r_hash = hasher.finish(); - - let key = (w_hash, r_hash); - // This is a shortcut to add if not exists *and* return false. It will return true - // if it was able to insert. - !self.recursion.insert(key) - } -} -impl SchemaCompatibility { - /// `can_read` performs a full, recursive check that a datum written using the - /// writers_schema can be read using the readers_schema. - pub fn can_read( - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - let mut c = Checker::new(); - c.can_read(writers_schema, readers_schema) - } - - /// `mutual_read` performs a full, recursive check that a datum written using either - /// the writers_schema or the readers_schema can be read using the other schema. - pub fn mutual_read( - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - SchemaCompatibility::can_read(writers_schema, readers_schema)?; - SchemaCompatibility::can_read(readers_schema, writers_schema) + fn recursion_in_progress( + &mut self, + writers_schema: &Schema, + readers_schema: &Schema, + ) -> bool { + let mut hasher = DefaultHasher::new(); + ptr::hash(writers_schema, &mut hasher); + let w_hash = hasher.finish(); + + hasher = DefaultHasher::new(); + ptr::hash(readers_schema, &mut hasher); + let r_hash = hasher.finish(); + + let key = (w_hash, r_hash); + // This is a shortcut to add if not exists *and* return false. It will return true + // if it was able to insert. + !self.recursion.insert(key) + } } - /// `match_schemas` performs a basic check that a datum written with the - /// writers_schema could be read using the readers_schema. This check only includes - /// matching the types, including schema promotion, and matching the full name for - /// named types. Aliases for named types are not supported here, and the rust - /// implementation of Avro in general does not include support for aliases (I think). - pub(crate) fn match_schemas( - writers_schema: &Schema, - readers_schema: &Schema, - ) -> Result<(), CompatibilityError> { - fn check_reader_type_multi( - reader_type: SchemaKind, - allowed_reader_types: Vec, - writer_type: SchemaKind, + impl SchemaCompatibility { + /// `can_read` performs a full, recursive check that a datum written using the + /// writers_schema can be read using the readers_schema. + pub fn can_read( + writers_schema: &Schema, + readers_schema: &Schema, ) -> Result<(), CompatibilityError> { - if allowed_reader_types.contains(&reader_type) { - Ok(()) - } else { - let mut allowed_types: Vec = vec![writer_type]; - allowed_types.extend_from_slice(allowed_reader_types.as_slice()); - Err(CompatibilityError::TypeExpected { - schema_type: String::from("readers_schema"), - expected_type: allowed_types, - }) - } + let mut c = Checker::new(); + c.can_read(writers_schema, readers_schema) } - fn check_reader_type( - reader_type: SchemaKind, - allowed_reader_type: SchemaKind, - writer_type: SchemaKind, + /// `mutual_read` performs a full, recursive check that a datum written using either + /// the writers_schema or the readers_schema can be read using the other schema. + pub fn mutual_read( + writers_schema: &Schema, + readers_schema: &Schema, ) -> Result<(), CompatibilityError> { - if reader_type == allowed_reader_type { - Ok(()) - } else { - Err(CompatibilityError::TypeExpected { - schema_type: String::from("readers_schema"), - expected_type: vec![writer_type, allowed_reader_type], - }) - } + SchemaCompatibility::can_read(writers_schema, readers_schema)?; + SchemaCompatibility::can_read(readers_schema, writers_schema) } - fn check_writer_type( + /// `match_schemas` performs a basic check that a datum written with the + /// writers_schema could be read using the readers_schema. This check only includes + /// matching the types, including schema promotion, and matching the full name for + /// named types. Aliases for named types are not supported here, and the rust + /// implementation of Avro in general does not include support for aliases (I think). + pub(crate) fn match_schemas( writers_schema: &Schema, - allowed_schema: &Schema, - expected_schema_types: Vec, + readers_schema: &Schema, ) -> Result<(), CompatibilityError> { - if *allowed_schema == *writers_schema { - Ok(()) - } else { - Err(CompatibilityError::TypeExpected { - schema_type: String::from("writers_schema"), - expected_type: expected_schema_types, - }) + fn check_reader_type_multi( + reader_type: SchemaKind, + allowed_reader_types: Vec, + writer_type: SchemaKind, + ) -> Result<(), CompatibilityError> { + if allowed_reader_types.contains(&reader_type) { + Ok(()) + } else { + let mut allowed_types: Vec = vec![writer_type]; + allowed_types.extend_from_slice(allowed_reader_types.as_slice()); + Err(CompatibilityError::TypeExpected { + schema_type: String::from("readers_schema"), + expected_type: allowed_types, + }) + } } - } - let w_type = SchemaKind::from(writers_schema); - let r_type = SchemaKind::from(readers_schema); + fn check_reader_type( + reader_type: SchemaKind, + allowed_reader_type: SchemaKind, + writer_type: SchemaKind, + ) -> Result<(), CompatibilityError> { + if reader_type == allowed_reader_type { + Ok(()) + } else { + Err(CompatibilityError::TypeExpected { + schema_type: String::from("readers_schema"), + expected_type: vec![writer_type, allowed_reader_type], + }) + } + } - if w_type == SchemaKind::Union || r_type == SchemaKind::Union { - return Ok(()); - } + fn check_writer_type( + writers_schema: &Schema, + allowed_schema: &Schema, + expected_schema_types: Vec, + ) -> Result<(), CompatibilityError> { + if *allowed_schema == *writers_schema { + Ok(()) + } else { + Err(CompatibilityError::TypeExpected { + schema_type: String::from("writers_schema"), + expected_type: expected_schema_types, + }) + } + } - if w_type == r_type { - if r_type.is_primitive() { + let w_type = SchemaKind::from(writers_schema); + let r_type = SchemaKind::from(readers_schema); + + if w_type == SchemaKind::Union || r_type == SchemaKind::Union { return Ok(()); } - match r_type { - SchemaKind::Record | SchemaKind::Enum => { - let msg = format!("A {readers_schema} type must always has a name"); - let writers_name = writers_schema.name().expect(&msg); - let readers_name = readers_schema.name().expect(&msg); + if w_type == r_type { + if r_type.is_primitive() { + return Ok(()); + } - if writers_name.name == readers_name.name { - return Ok(()); - } + match r_type { + SchemaKind::Record | SchemaKind::Enum => { + let msg = format!("A {readers_schema} type must always has a name"); + let writers_name = writers_schema.name().expect(&msg); + let readers_name = readers_schema.name().expect(&msg); - return Err(CompatibilityError::NameMismatch { - writer_name: writers_name.name.clone(), - reader_name: readers_name.name.clone(), - }); - } - SchemaKind::Fixed => { - if let Schema::Fixed(FixedSchema { - name: w_name, - aliases: _, - doc: _w_doc, - size: w_size, - default: _w_default, - attributes: _, - }) = writers_schema - { + if writers_name.name == readers_name.name { + return Ok(()); + } + + return Err(CompatibilityError::NameMismatch { + writer_name: writers_name.name.clone(), + reader_name: readers_name.name.clone(), + }); + } + SchemaKind::Fixed => { if let Schema::Fixed(FixedSchema { - name: r_name, + name: w_name, aliases: _, - doc: _r_doc, - size: r_size, - default: _r_default, + doc: _w_doc, + size: w_size, + default: _w_default, attributes: _, - }) = readers_schema + }) = writers_schema { - return (w_name.name == r_name.name && w_size == r_size) - .then_some(()) - .ok_or(CompatibilityError::FixedMismatch); + if let Schema::Fixed(FixedSchema { + name: r_name, + aliases: _, + doc: _r_doc, + size: r_size, + default: _r_default, + attributes: _, + }) = readers_schema + { + return (w_name.name == r_name.name && w_size == r_size) + .then_some(()) + .ok_or(CompatibilityError::FixedMismatch); + } } } - } - SchemaKind::Map => { - if let Schema::Map(w_m) = writers_schema { - if let Schema::Map(r_m) = readers_schema { - return SchemaCompatibility::match_schemas(&w_m.types, &r_m.types); + SchemaKind::Map => { + if let Schema::Map(w_m) = writers_schema { + if let Schema::Map(r_m) = readers_schema { + return SchemaCompatibility::match_schemas(&w_m.types, &r_m.types); + } } } - } - SchemaKind::Array => { - if let Schema::Array(w_a) = writers_schema { - if let Schema::Array(r_a) = readers_schema { - return SchemaCompatibility::match_schemas(&w_a.items, &r_a.items); + SchemaKind::Array => { + if let Schema::Array(w_a) = writers_schema { + if let Schema::Array(r_a) = readers_schema { + return SchemaCompatibility::match_schemas(&w_a.items, &r_a.items); + } } } - } - SchemaKind::Uuid => { - return check_writer_type( - writers_schema, - readers_schema, - vec![r_type, SchemaKind::String], - ); - } + SchemaKind::Uuid => { + return check_writer_type( + writers_schema, + readers_schema, + vec![r_type, SchemaKind::String], + ); + } + SchemaKind::Date | SchemaKind::TimeMillis => { + return check_writer_type( + writers_schema, + readers_schema, + vec![r_type, SchemaKind::Int], + ); + } + SchemaKind::TimeMicros + | SchemaKind::TimestampNanos + | SchemaKind::TimestampMillis + | SchemaKind::TimestampMicros + | SchemaKind::LocalTimestampMillis + | SchemaKind::LocalTimestampMicros + | SchemaKind::LocalTimestampNanos => { + return check_writer_type( + writers_schema, + readers_schema, + vec![r_type, SchemaKind::Long], + ); + } + SchemaKind::Duration => { + return Ok(()); + } + SchemaKind::Ref => return match_ref_schemas(writers_schema, readers_schema), + _ => { + return Err(CompatibilityError::Inconclusive(String::from( + "readers_schema", + ))); + } + }; + } + + // Here are the checks for primitive types + match w_type { + SchemaKind::Int => check_reader_type_multi( + r_type, + vec![ + SchemaKind::Long, + SchemaKind::Float, + SchemaKind::Double, + SchemaKind::Date, + SchemaKind::TimeMillis, + ], + w_type, + ), + SchemaKind::Long => check_reader_type_multi( + r_type, + vec![ + SchemaKind::Float, + SchemaKind::Double, + SchemaKind::TimeMicros, + SchemaKind::TimestampMillis, + SchemaKind::TimestampMicros, + SchemaKind::TimestampNanos, + SchemaKind::LocalTimestampMillis, + SchemaKind::LocalTimestampMicros, + SchemaKind::LocalTimestampNanos, + ], + w_type, + ), + SchemaKind::Float => check_reader_type_multi( + r_type, + vec![SchemaKind::Float, SchemaKind::Double], + w_type, + ), + SchemaKind::String => check_reader_type_multi( + r_type, + vec![SchemaKind::Bytes, SchemaKind::Uuid], + w_type, + ), + SchemaKind::Bytes => check_reader_type(r_type, SchemaKind::String, w_type), + SchemaKind::Uuid => check_reader_type(r_type, SchemaKind::String, w_type), SchemaKind::Date | SchemaKind::TimeMillis => { - return check_writer_type( - writers_schema, - readers_schema, - vec![r_type, SchemaKind::Int], - ); + check_reader_type(r_type, SchemaKind::Int, w_type) } SchemaKind::TimeMicros - | SchemaKind::TimestampNanos - | SchemaKind::TimestampMillis | SchemaKind::TimestampMicros + | SchemaKind::TimestampMillis + | SchemaKind::TimestampNanos | SchemaKind::LocalTimestampMillis | SchemaKind::LocalTimestampMicros | SchemaKind::LocalTimestampNanos => { - return check_writer_type( - writers_schema, - readers_schema, - vec![r_type, SchemaKind::Long], - ); + check_reader_type(r_type, SchemaKind::Long, w_type) } - SchemaKind::Duration => { - return Ok(()); - } - SchemaKind::Ref => return match_ref_schemas(writers_schema, readers_schema), - _ => { - return Err(CompatibilityError::Inconclusive(String::from( - "readers_schema", - ))); - } - }; - } - - // Here are the checks for primitive types - match w_type { - SchemaKind::Int => check_reader_type_multi( - r_type, - vec![ - SchemaKind::Long, - SchemaKind::Float, - SchemaKind::Double, - SchemaKind::Date, - SchemaKind::TimeMillis, - ], - w_type, - ), - SchemaKind::Long => check_reader_type_multi( - r_type, - vec![ - SchemaKind::Float, - SchemaKind::Double, - SchemaKind::TimeMicros, - SchemaKind::TimestampMillis, - SchemaKind::TimestampMicros, - SchemaKind::TimestampNanos, - SchemaKind::LocalTimestampMillis, - SchemaKind::LocalTimestampMicros, - SchemaKind::LocalTimestampNanos, - ], - w_type, - ), - SchemaKind::Float => { - check_reader_type_multi(r_type, vec![SchemaKind::Float, SchemaKind::Double], w_type) - } - SchemaKind::String => { - check_reader_type_multi(r_type, vec![SchemaKind::Bytes, SchemaKind::Uuid], w_type) - } - SchemaKind::Bytes => check_reader_type(r_type, SchemaKind::String, w_type), - SchemaKind::Uuid => check_reader_type(r_type, SchemaKind::String, w_type), - SchemaKind::Date | SchemaKind::TimeMillis => { - check_reader_type(r_type, SchemaKind::Int, w_type) - } - SchemaKind::TimeMicros - | SchemaKind::TimestampMicros - | SchemaKind::TimestampMillis - | SchemaKind::TimestampNanos - | SchemaKind::LocalTimestampMillis - | SchemaKind::LocalTimestampMicros - | SchemaKind::LocalTimestampNanos => { - check_reader_type(r_type, SchemaKind::Long, w_type) + _ => Err(CompatibilityError::Inconclusive(String::from( + "writers_schema", + ))), } - _ => Err(CompatibilityError::Inconclusive(String::from( - "writers_schema", - ))), } } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - Codec, Reader, Writer, - types::{Record, Value}, - }; - use apache_avro_test_helper::TestResult; - use rstest::*; + #[cfg(test)] + mod tests { + use super::*; + use crate::{ + Codec, Reader, Writer, + types::{Record, Value}, + }; + use apache_avro_test_helper::TestResult; + use rstest::*; - fn int_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"int"}"#).unwrap() - } + fn int_array_schema() -> Schema { + Schema::parse_str(r#"{"type":"array", "items":"int"}"#).unwrap() + } - fn long_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"long"}"#).unwrap() - } + fn long_array_schema() -> Schema { + Schema::parse_str(r#"{"type":"array", "items":"long"}"#).unwrap() + } - fn string_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"string"}"#).unwrap() - } + fn string_array_schema() -> Schema { + Schema::parse_str(r#"{"type":"array", "items":"string"}"#).unwrap() + } - fn int_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"int"}"#).unwrap() - } + fn int_map_schema() -> Schema { + Schema::parse_str(r#"{"type":"map", "values":"int"}"#).unwrap() + } - fn long_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"long"}"#).unwrap() - } + fn long_map_schema() -> Schema { + Schema::parse_str(r#"{"type":"map", "values":"long"}"#).unwrap() + } - fn string_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"string"}"#).unwrap() - } + fn string_map_schema() -> Schema { + Schema::parse_str(r#"{"type":"map", "values":"string"}"#).unwrap() + } - fn enum1_ab_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B"]}"#).unwrap() - } + fn enum1_ab_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B"]}"#).unwrap() + } - fn enum1_abc_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}"#).unwrap() - } + fn enum1_abc_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}"#) + .unwrap() + } - fn enum1_bc_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["B","C"]}"#).unwrap() - } + fn enum1_bc_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["B","C"]}"#).unwrap() + } - fn enum2_ab_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum2", "symbols":["A","B"]}"#).unwrap() - } + fn enum2_ab_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum2", "symbols":["A","B"]}"#).unwrap() + } - fn empty_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[]}"#).unwrap() - } + fn empty_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[]}"#).unwrap() + } - fn empty_record2_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record2", "fields": []}"#).unwrap() - } + fn empty_record2_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record2", "fields": []}"#).unwrap() + } - fn a_int_record1_schema() -> Schema { - Schema::parse_str( - r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}]}"#, - ) - .unwrap() - } + fn a_int_record1_schema() -> Schema { + Schema::parse_str( + r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}]}"#, + ) + .unwrap() + } - fn a_long_record1_schema() -> Schema { - Schema::parse_str( - r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"long"}]}"#, - ) - .unwrap() - } + fn a_long_record1_schema() -> Schema { + Schema::parse_str( + r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"long"}]}"#, + ) + .unwrap() + } - fn a_int_b_int_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}"#).unwrap() - } + fn a_int_b_int_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}"#).unwrap() + } - fn a_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}"#).unwrap() - } + fn a_dint_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}"#).unwrap() + } - fn a_int_b_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}"#).unwrap() - } + fn a_int_b_dint_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}"#).unwrap() + } - fn a_dint_b_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}"#).unwrap() - } + fn a_dint_b_dint_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}"#).unwrap() + } - fn nested_record() -> Schema { - Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}}]}"#).unwrap() - } + fn nested_record() -> Schema { + Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}}]}"#).unwrap() + } - fn nested_optional_record() -> Schema { - Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":["null",{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}],"default":null}]}"#).unwrap() - } + fn nested_optional_record() -> Schema { + Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":["null",{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}],"default":null}]}"#).unwrap() + } - fn int_list_record_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"List", "fields": [{"name": "head", "type": "int"},{"name": "tail", "type": "array", "items": "int"}]}"#).unwrap() - } + fn int_list_record_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"List", "fields": [{"name": "head", "type": "int"},{"name": "tail", "type": "array", "items": "int"}]}"#).unwrap() + } - fn long_list_record_schema() -> Schema { - Schema::parse_str( - r#" + fn long_list_record_schema() -> Schema { + Schema::parse_str( + r#" { "type":"record", "name":"List", "fields": [ {"name": "head", "type": "long"}, {"name": "tail", "type": "array", "items": "long"} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - fn union_schema(schemas: Vec) -> Schema { - let schema_string = schemas - .iter() - .map(|s| s.canonical_form()) - .collect::>() - .join(","); - Schema::parse_str(&format!("[{schema_string}]")).unwrap() - } + fn union_schema(schemas: Vec) -> Schema { + let schema_string = schemas + .iter() + .map(|s| s.canonical_form()) + .collect::>() + .join(","); + Schema::parse_str(&format!("[{schema_string}]")).unwrap() + } - fn empty_union_schema() -> Schema { - union_schema(vec![]) - } + fn empty_union_schema() -> Schema { + union_schema(vec![]) + } - // unused - // fn null_union_schema() -> Schema { union_schema(vec![Schema::Null]) } + // unused + // fn null_union_schema() -> Schema { union_schema(vec![Schema::Null]) } - fn int_union_schema() -> Schema { - union_schema(vec![Schema::Int]) - } + fn int_union_schema() -> Schema { + union_schema(vec![Schema::Int]) + } - fn long_union_schema() -> Schema { - union_schema(vec![Schema::Long]) - } + fn long_union_schema() -> Schema { + union_schema(vec![Schema::Long]) + } - fn string_union_schema() -> Schema { - union_schema(vec![Schema::String]) - } + fn string_union_schema() -> Schema { + union_schema(vec![Schema::String]) + } - fn int_string_union_schema() -> Schema { - union_schema(vec![Schema::Int, Schema::String]) - } + fn int_string_union_schema() -> Schema { + union_schema(vec![Schema::Int, Schema::String]) + } - fn string_int_union_schema() -> Schema { - union_schema(vec![Schema::String, Schema::Int]) - } + fn string_int_union_schema() -> Schema { + union_schema(vec![Schema::String, Schema::Int]) + } - #[test] - fn test_broken() { - assert_eq!( - CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&int_string_union_schema(), &int_union_schema()) - .unwrap_err() - ) - } + #[test] + fn test_broken() { + assert_eq!( + CompatibilityError::MissingUnionElements, + SchemaCompatibility::can_read(&int_string_union_schema(), &int_union_schema()) + .unwrap_err() + ) + } - #[test] - fn test_incompatible_reader_writer_pairs() { - let incompatible_schemas = vec![ - // null - (Schema::Null, Schema::Int), - (Schema::Null, Schema::Long), - // boolean - (Schema::Boolean, Schema::Int), - // int - (Schema::Int, Schema::Null), - (Schema::Int, Schema::Boolean), - (Schema::Int, Schema::Long), - (Schema::Int, Schema::Float), - (Schema::Int, Schema::Double), - // long - (Schema::Long, Schema::Float), - (Schema::Long, Schema::Double), - // float - (Schema::Float, Schema::Double), - // string - (Schema::String, Schema::Boolean), - (Schema::String, Schema::Int), - // bytes - (Schema::Bytes, Schema::Null), - (Schema::Bytes, Schema::Int), - // logical types - (Schema::TimeMicros, Schema::Int), - (Schema::TimestampMillis, Schema::Int), - (Schema::TimestampMicros, Schema::Int), - (Schema::TimestampNanos, Schema::Int), - (Schema::LocalTimestampMillis, Schema::Int), - (Schema::LocalTimestampMicros, Schema::Int), - (Schema::LocalTimestampNanos, Schema::Int), - (Schema::Date, Schema::Long), - (Schema::TimeMillis, Schema::Long), - // array and maps - (int_array_schema(), long_array_schema()), - (int_map_schema(), int_array_schema()), - (int_array_schema(), int_map_schema()), - (int_map_schema(), long_map_schema()), - // enum - (enum1_ab_schema(), enum1_abc_schema()), - (enum1_bc_schema(), enum1_abc_schema()), - (enum1_ab_schema(), enum2_ab_schema()), - (Schema::Int, enum2_ab_schema()), - (enum2_ab_schema(), Schema::Int), - //union - (int_union_schema(), int_string_union_schema()), - (string_union_schema(), int_string_union_schema()), - //record - (empty_record2_schema(), empty_record1_schema()), - (a_int_record1_schema(), empty_record1_schema()), - (a_int_b_dint_record1_schema(), empty_record1_schema()), - (int_list_record_schema(), long_list_record_schema()), - (nested_record(), nested_optional_record()), - ]; - - assert!( - incompatible_schemas - .iter() - .any(|(reader, writer)| SchemaCompatibility::can_read(writer, reader).is_err()) - ); - } + #[test] + fn test_incompatible_reader_writer_pairs() { + let incompatible_schemas = vec![ + // null + (Schema::Null, Schema::Int), + (Schema::Null, Schema::Long), + // boolean + (Schema::Boolean, Schema::Int), + // int + (Schema::Int, Schema::Null), + (Schema::Int, Schema::Boolean), + (Schema::Int, Schema::Long), + (Schema::Int, Schema::Float), + (Schema::Int, Schema::Double), + // long + (Schema::Long, Schema::Float), + (Schema::Long, Schema::Double), + // float + (Schema::Float, Schema::Double), + // string + (Schema::String, Schema::Boolean), + (Schema::String, Schema::Int), + // bytes + (Schema::Bytes, Schema::Null), + (Schema::Bytes, Schema::Int), + // logical types + (Schema::TimeMicros, Schema::Int), + (Schema::TimestampMillis, Schema::Int), + (Schema::TimestampMicros, Schema::Int), + (Schema::TimestampNanos, Schema::Int), + (Schema::LocalTimestampMillis, Schema::Int), + (Schema::LocalTimestampMicros, Schema::Int), + (Schema::LocalTimestampNanos, Schema::Int), + (Schema::Date, Schema::Long), + (Schema::TimeMillis, Schema::Long), + // array and maps + (int_array_schema(), long_array_schema()), + (int_map_schema(), int_array_schema()), + (int_array_schema(), int_map_schema()), + (int_map_schema(), long_map_schema()), + // enum + (enum1_ab_schema(), enum1_abc_schema()), + (enum1_bc_schema(), enum1_abc_schema()), + (enum1_ab_schema(), enum2_ab_schema()), + (Schema::Int, enum2_ab_schema()), + (enum2_ab_schema(), Schema::Int), + //union + (int_union_schema(), int_string_union_schema()), + (string_union_schema(), int_string_union_schema()), + //record + (empty_record2_schema(), empty_record1_schema()), + (a_int_record1_schema(), empty_record1_schema()), + (a_int_b_dint_record1_schema(), empty_record1_schema()), + (int_list_record_schema(), long_list_record_schema()), + (nested_record(), nested_optional_record()), + ]; + + assert!( + incompatible_schemas + .iter() + .any(|(reader, writer)| SchemaCompatibility::can_read(writer, reader).is_err()) + ); + } - #[rstest] - // Record type test - #[case( + #[rstest] + // Record type test + #[case( r#"{"type": "record", "name": "record_a", "fields": [{"type": "long", "name": "date"}]}"#, r#"{"type": "record", "name": "record_a", "fields": [{"type": "long", "name": "date", "default": 18181}]}"# )] - // Fixed type test - #[case( - r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, - r#"{"type": "fixed", "name": "EmployeeId", "size": 16, "default": "u00ffffffffffffx"}"# - )] - // Enum type test - #[case( - r#"{"type": "enum", "name":"Enum1", "symbols": ["A","B"]}"#, - r#"{"type": "enum", "name":"Enum1", "symbols": ["A","B", "C"], "default": "C"}"# - )] - // Map type test - #[case( - r#"{"type": "map", "values": "int"}"#, - r#"{"type": "map", "values": "long"}"# - )] - // Date type - #[case(r#"{"type": "int"}"#, r#"{"type": "int", "logicalType": "date"}"#)] - // time-millis type - #[case( - r#"{"type": "int"}"#, - r#"{"type": "int", "logicalType": "time-millis"}"# - )] - // time-millis type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "time-micros"}"# - )] - // timetimestamp-nanos type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "timestamp-nanos"}"# - )] - // timestamp-millis type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "timestamp-millis"}"# - )] - // timestamp-micros type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "timestamp-micros"}"# - )] - // local-timestamp-millis type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "local-timestamp-millis"}"# - )] - // local-timestamp-micros type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "local-timestamp-micros"}"# - )] - // local-timestamp-nanos type - #[case( - r#"{"type": "long"}"#, - r#"{"type": "long", "logicalType": "local-timestamp-nanos"}"# - )] - // Array type test - #[case( - r#"{"type": "array", "items": "int"}"#, - r#"{"type": "array", "items": "long"}"# - )] - fn test_avro_3950_match_schemas_ok( - #[case] writer_schema_str: &str, - #[case] reader_schema_str: &str, - ) { - let writer_schema = Schema::parse_str(writer_schema_str).unwrap(); - let reader_schema = Schema::parse_str(reader_schema_str).unwrap(); - - assert!(SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).is_ok()); - } + // Fixed type test + #[case( + r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, + r#"{"type": "fixed", "name": "EmployeeId", "size": 16, "default": "u00ffffffffffffx"}"# + )] + // Enum type test + #[case( + r#"{"type": "enum", "name":"Enum1", "symbols": ["A","B"]}"#, + r#"{"type": "enum", "name":"Enum1", "symbols": ["A","B", "C"], "default": "C"}"# + )] + // Map type test + #[case( + r#"{"type": "map", "values": "int"}"#, + r#"{"type": "map", "values": "long"}"# + )] + // Date type + #[case(r#"{"type": "int"}"#, r#"{"type": "int", "logicalType": "date"}"#)] + // time-millis type + #[case( + r#"{"type": "int"}"#, + r#"{"type": "int", "logicalType": "time-millis"}"# + )] + // time-millis type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "time-micros"}"# + )] + // timetimestamp-nanos type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "timestamp-nanos"}"# + )] + // timestamp-millis type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "timestamp-millis"}"# + )] + // timestamp-micros type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "timestamp-micros"}"# + )] + // local-timestamp-millis type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "local-timestamp-millis"}"# + )] + // local-timestamp-micros type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "local-timestamp-micros"}"# + )] + // local-timestamp-nanos type + #[case( + r#"{"type": "long"}"#, + r#"{"type": "long", "logicalType": "local-timestamp-nanos"}"# + )] + // Array type test + #[case( + r#"{"type": "array", "items": "int"}"#, + r#"{"type": "array", "items": "long"}"# + )] + fn test_avro_3950_match_schemas_ok( + #[case] writer_schema_str: &str, + #[case] reader_schema_str: &str, + ) { + let writer_schema = Schema::parse_str(writer_schema_str).unwrap(); + let reader_schema = Schema::parse_str(reader_schema_str).unwrap(); + + assert!(SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).is_ok()); + } - #[rstest] - // Record type test - #[case( + #[rstest] + // Record type test + #[case( r#"{"type": "record", "name":"record_a", "fields": [{"type": "long", "name": "date"}]}"#, r#"{"type": "record", "name":"record_b", "fields": [{"type": "long", "name": "date"}]}"#, CompatibilityError::NameMismatch{writer_name: String::from("record_a"), reader_name: String::from("record_b")} )] - // Fixed type test - #[case( - r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, - r#"{"type": "fixed", "name": "EmployeeId", "size": 20}"#, - CompatibilityError::FixedMismatch - )] - // Enum type test - #[case( + // Fixed type test + #[case( + r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, + r#"{"type": "fixed", "name": "EmployeeId", "size": 20}"#, + CompatibilityError::FixedMismatch + )] + // Enum type test + #[case( r#"{"type": "enum", "name": "Enum1", "symbols": ["A","B"]}"#, r#"{"type": "enum", "name": "Enum2", "symbols": ["A","B"]}"#, CompatibilityError::NameMismatch{writer_name: String::from("Enum1"), reader_name: String::from("Enum2")} )] - // Map type test - #[case( + // Map type test + #[case( r#"{"type":"map", "values": "long"}"#, r#"{"type":"map", "values": "int"}"#, CompatibilityError::TypeExpected {schema_type: String::from("readers_schema"), expected_type: vec![ @@ -856,8 +887,8 @@ mod tests { SchemaKind::LocalTimestampNanos, ]} )] - // Array type test - #[case( + // Array type test + #[case( r#"{"type": "array", "items": "long"}"#, r#"{"type": "array", "items": "int"}"#, CompatibilityError::TypeExpected {schema_type: String::from("readers_schema"), expected_type: vec![ @@ -873,8 +904,8 @@ mod tests { SchemaKind::LocalTimestampNanos, ]} )] - // Date type test - #[case( + // Date type test + #[case( r#"{"type": "string"}"#, r#"{"type": "int", "logicalType": "date"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -883,8 +914,8 @@ mod tests { SchemaKind::Uuid, ]} )] - // time-millis type - #[case( + // time-millis type + #[case( r#"{"type": "string"}"#, r#"{"type": "int", "logicalType": "time-millis"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -893,8 +924,8 @@ mod tests { SchemaKind::Uuid, ]} )] - // time-millis type - #[case( + // time-millis type + #[case( r#"{"type": "int"}"#, r#"{"type": "long", "logicalType": "time-micros"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -906,21 +937,21 @@ mod tests { SchemaKind::TimeMillis ]} )] - // timestamp-nanos type. This test should fail because it is not supported on schema parse_complex - // #[case( - // r#"{"type": "string"}"#, - // r#"{"type": "long", "logicalType": "timestamp-nanos"}"#, - // CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ - // SchemaKind::Int, - // SchemaKind::Long, - // SchemaKind::Float, - // SchemaKind::Double, - // SchemaKind::Date, - // SchemaKind::TimeMillis - // ]} - // )] - // timestamp-millis type - #[case( + // timestamp-nanos type. This test should fail because it is not supported on schema parse_complex + // #[case( + // r#"{"type": "string"}"#, + // r#"{"type": "long", "logicalType": "timestamp-nanos"}"#, + // CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ + // SchemaKind::Int, + // SchemaKind::Long, + // SchemaKind::Float, + // SchemaKind::Double, + // SchemaKind::Date, + // SchemaKind::TimeMillis + // ]} + // )] + // timestamp-millis type + #[case( r#"{"type": "int"}"#, r#"{"type": "long", "logicalType": "timestamp-millis"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -932,8 +963,8 @@ mod tests { SchemaKind::TimeMillis ]} )] - // timestamp-micros type - #[case( + // timestamp-micros type + #[case( r#"{"type": "int"}"#, r#"{"type": "long", "logicalType": "timestamp-micros"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -945,8 +976,8 @@ mod tests { SchemaKind::TimeMillis ]} )] - // local-timestamp-millis type - #[case( + // local-timestamp-millis type + #[case( r#"{"type": "int"}"#, r#"{"type": "long", "logicalType": "local-timestamp-millis"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -958,8 +989,8 @@ mod tests { SchemaKind::TimeMillis ]} )] - // local-timestamp-micros type - #[case( + // local-timestamp-micros type + #[case( r#"{"type": "int"}"#, r#"{"type": "long", "logicalType": "local-timestamp-micros"}"#, CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ @@ -971,446 +1002,456 @@ mod tests { SchemaKind::TimeMillis ]} )] - // local-timestamp-nanos type. This test should fail because it is not supported on schema parse_complex - // #[case( - // r#"{"type": "int"}"#, - // r#"{"type": "long", "logicalType": "local-timestamp-nanos"}"#, - // CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ - // SchemaKind::Int, - // SchemaKind::Long, - // SchemaKind::Float, - // SchemaKind::Double, - // SchemaKind::Date, - // SchemaKind::TimeMillis - // ]} - // )] - // When comparing different types we always get Inconclusive - #[case( + // local-timestamp-nanos type. This test should fail because it is not supported on schema parse_complex + // #[case( + // r#"{"type": "int"}"#, + // r#"{"type": "long", "logicalType": "local-timestamp-nanos"}"#, + // CompatibilityError::TypeExpected{schema_type: String::from("readers_schema"), expected_type: vec![ + // SchemaKind::Int, + // SchemaKind::Long, + // SchemaKind::Float, + // SchemaKind::Double, + // SchemaKind::Date, + // SchemaKind::TimeMillis + // ]} + // )] + // When comparing different types we always get Inconclusive + #[case( r#"{"type": "record", "name":"record_b", "fields": [{"type": "long", "name": "date"}]}"#, r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, CompatibilityError::Inconclusive(String::from("writers_schema")) )] - fn test_avro_3950_match_schemas_error( - #[case] writer_schema_str: &str, - #[case] reader_schema_str: &str, - #[case] expected_error: CompatibilityError, - ) { - let writer_schema = Schema::parse_str(writer_schema_str).unwrap(); - let reader_schema = Schema::parse_str(reader_schema_str).unwrap(); - - assert_eq!( - expected_error, - SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).unwrap_err() - ) - } + fn test_avro_3950_match_schemas_error( + #[case] writer_schema_str: &str, + #[case] reader_schema_str: &str, + #[case] expected_error: CompatibilityError, + ) { + let writer_schema = Schema::parse_str(writer_schema_str).unwrap(); + let reader_schema = Schema::parse_str(reader_schema_str).unwrap(); - #[test] - fn test_compatible_reader_writer_pairs() { - let compatible_schemas = vec![ - (Schema::Null, Schema::Null), - (Schema::Long, Schema::Int), - (Schema::Float, Schema::Int), - (Schema::Float, Schema::Long), - (Schema::Double, Schema::Long), - (Schema::Double, Schema::Int), - (Schema::Double, Schema::Float), - (Schema::String, Schema::Bytes), - (Schema::Bytes, Schema::String), - // logical types - (Schema::Uuid, Schema::Uuid), - (Schema::Uuid, Schema::String), - (Schema::Date, Schema::Int), - (Schema::TimeMillis, Schema::Int), - (Schema::TimeMicros, Schema::Long), - (Schema::TimestampMillis, Schema::Long), - (Schema::TimestampMicros, Schema::Long), - (Schema::TimestampNanos, Schema::Long), - (Schema::LocalTimestampMillis, Schema::Long), - (Schema::LocalTimestampMicros, Schema::Long), - (Schema::LocalTimestampNanos, Schema::Long), - (Schema::String, Schema::Uuid), - (Schema::Int, Schema::Date), - (Schema::Int, Schema::TimeMillis), - (Schema::Long, Schema::TimeMicros), - (Schema::Long, Schema::TimestampMillis), - (Schema::Long, Schema::TimestampMicros), - (Schema::Long, Schema::TimestampNanos), - (Schema::Long, Schema::LocalTimestampMillis), - (Schema::Long, Schema::LocalTimestampMicros), - (Schema::Long, Schema::LocalTimestampNanos), - (int_array_schema(), int_array_schema()), - (long_array_schema(), int_array_schema()), - (int_map_schema(), int_map_schema()), - (long_map_schema(), int_map_schema()), - (enum1_ab_schema(), enum1_ab_schema()), - (enum1_abc_schema(), enum1_ab_schema()), - (empty_union_schema(), empty_union_schema()), - (int_union_schema(), int_union_schema()), - (int_string_union_schema(), string_int_union_schema()), - (int_union_schema(), empty_union_schema()), - (long_union_schema(), int_union_schema()), - (int_union_schema(), Schema::Int), - (Schema::Int, int_union_schema()), - (empty_record1_schema(), empty_record1_schema()), - (empty_record1_schema(), a_int_record1_schema()), - (a_int_record1_schema(), a_int_record1_schema()), - (a_dint_record1_schema(), a_int_record1_schema()), - (a_dint_record1_schema(), a_dint_record1_schema()), - (a_int_record1_schema(), a_dint_record1_schema()), - (a_long_record1_schema(), a_int_record1_schema()), - (a_int_record1_schema(), a_int_b_int_record1_schema()), - (a_dint_record1_schema(), a_int_b_int_record1_schema()), - (a_int_b_dint_record1_schema(), a_int_record1_schema()), - (a_dint_b_dint_record1_schema(), empty_record1_schema()), - (a_dint_b_dint_record1_schema(), a_int_record1_schema()), - (a_int_b_int_record1_schema(), a_dint_b_dint_record1_schema()), - (int_list_record_schema(), int_list_record_schema()), - (long_list_record_schema(), long_list_record_schema()), - (long_list_record_schema(), int_list_record_schema()), - (nested_optional_record(), nested_record()), - ]; - - assert!( - compatible_schemas - .iter() - .all(|(reader, writer)| SchemaCompatibility::can_read(writer, reader).is_ok()) - ); - } + assert_eq!( + expected_error, + SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).unwrap_err() + ) + } + + #[test] + fn test_compatible_reader_writer_pairs() { + let compatible_schemas = vec![ + (Schema::Null, Schema::Null), + (Schema::Long, Schema::Int), + (Schema::Float, Schema::Int), + (Schema::Float, Schema::Long), + (Schema::Double, Schema::Long), + (Schema::Double, Schema::Int), + (Schema::Double, Schema::Float), + (Schema::String, Schema::Bytes), + (Schema::Bytes, Schema::String), + // logical types + (Schema::Uuid, Schema::Uuid), + (Schema::Uuid, Schema::String), + (Schema::Date, Schema::Int), + (Schema::TimeMillis, Schema::Int), + (Schema::TimeMicros, Schema::Long), + (Schema::TimestampMillis, Schema::Long), + (Schema::TimestampMicros, Schema::Long), + (Schema::TimestampNanos, Schema::Long), + (Schema::LocalTimestampMillis, Schema::Long), + (Schema::LocalTimestampMicros, Schema::Long), + (Schema::LocalTimestampNanos, Schema::Long), + (Schema::String, Schema::Uuid), + (Schema::Int, Schema::Date), + (Schema::Int, Schema::TimeMillis), + (Schema::Long, Schema::TimeMicros), + (Schema::Long, Schema::TimestampMillis), + (Schema::Long, Schema::TimestampMicros), + (Schema::Long, Schema::TimestampNanos), + (Schema::Long, Schema::LocalTimestampMillis), + (Schema::Long, Schema::LocalTimestampMicros), + (Schema::Long, Schema::LocalTimestampNanos), + (int_array_schema(), int_array_schema()), + (long_array_schema(), int_array_schema()), + (int_map_schema(), int_map_schema()), + (long_map_schema(), int_map_schema()), + (enum1_ab_schema(), enum1_ab_schema()), + (enum1_abc_schema(), enum1_ab_schema()), + (empty_union_schema(), empty_union_schema()), + (int_union_schema(), int_union_schema()), + (int_string_union_schema(), string_int_union_schema()), + (int_union_schema(), empty_union_schema()), + (long_union_schema(), int_union_schema()), + (int_union_schema(), Schema::Int), + (Schema::Int, int_union_schema()), + (empty_record1_schema(), empty_record1_schema()), + (empty_record1_schema(), a_int_record1_schema()), + (a_int_record1_schema(), a_int_record1_schema()), + (a_dint_record1_schema(), a_int_record1_schema()), + (a_dint_record1_schema(), a_dint_record1_schema()), + (a_int_record1_schema(), a_dint_record1_schema()), + (a_long_record1_schema(), a_int_record1_schema()), + (a_int_record1_schema(), a_int_b_int_record1_schema()), + (a_dint_record1_schema(), a_int_b_int_record1_schema()), + (a_int_b_dint_record1_schema(), a_int_record1_schema()), + (a_dint_b_dint_record1_schema(), empty_record1_schema()), + (a_dint_b_dint_record1_schema(), a_int_record1_schema()), + (a_int_b_int_record1_schema(), a_dint_b_dint_record1_schema()), + (int_list_record_schema(), int_list_record_schema()), + (long_list_record_schema(), long_list_record_schema()), + (long_list_record_schema(), int_list_record_schema()), + (nested_optional_record(), nested_record()), + ]; + + assert!( + compatible_schemas + .iter() + .all(|(reader, writer)| SchemaCompatibility::can_read(writer, reader).is_ok()) + ); + } - fn writer_schema() -> Schema { - Schema::parse_str( - r#" + fn writer_schema() -> Schema { + Schema::parse_str( + r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, {"name":"oldfield2", "type":"string"} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - #[test] - fn test_missing_field() -> TestResult { - let reader_schema = Schema::parse_str( - r#" + #[test] + fn test_missing_field() -> TestResult { + let reader_schema = Schema::parse_str( + r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema,).is_ok()); - assert_eq!( - CompatibilityError::MissingDefaultValue(String::from("oldfield2")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() - ); - - Ok(()) - } + )?; + assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema,).is_ok()); + assert_eq!( + CompatibilityError::MissingDefaultValue(String::from("oldfield2")), + SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + ); + + Ok(()) + } - #[test] - fn test_missing_second_field() -> TestResult { - let reader_schema = Schema::parse_str( - r#" + #[test] + fn test_missing_second_field() -> TestResult { + let reader_schema = Schema::parse_str( + r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield2", "type":"string"} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); - assert_eq!( - CompatibilityError::MissingDefaultValue(String::from("oldfield1")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() - ); - - Ok(()) - } + )?; + assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); + assert_eq!( + CompatibilityError::MissingDefaultValue(String::from("oldfield1")), + SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + ); - #[test] - fn test_all_fields() -> TestResult { - let reader_schema = Schema::parse_str( - r#" + Ok(()) + } + + #[test] + fn test_all_fields() -> TestResult { + let reader_schema = Schema::parse_str( + r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, {"name":"oldfield2", "type":"string"} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); - assert!(SchemaCompatibility::can_read(&reader_schema, &writer_schema()).is_ok()); + )?; + assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); + assert!(SchemaCompatibility::can_read(&reader_schema, &writer_schema()).is_ok()); - Ok(()) - } + Ok(()) + } - #[test] - fn test_new_field_with_default() -> TestResult { - let reader_schema = Schema::parse_str( - r#" + #[test] + fn test_new_field_with_default() -> TestResult { + let reader_schema = Schema::parse_str( + r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, {"name":"newfield1", "type":"int", "default":42} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); - assert_eq!( - CompatibilityError::MissingDefaultValue(String::from("oldfield2")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() - ); - - Ok(()) - } + )?; + assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); + assert_eq!( + CompatibilityError::MissingDefaultValue(String::from("oldfield2")), + SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + ); - #[test] - fn test_new_field() -> TestResult { - let reader_schema = Schema::parse_str( - r#" + Ok(()) + } + + #[test] + fn test_new_field() -> TestResult { + let reader_schema = Schema::parse_str( + r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, {"name":"newfield1", "type":"int"} ]} "#, - )?; - assert_eq!( - CompatibilityError::MissingDefaultValue(String::from("newfield1")), - SchemaCompatibility::can_read(&writer_schema(), &reader_schema).unwrap_err() - ); - assert_eq!( - CompatibilityError::MissingDefaultValue(String::from("oldfield2")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() - ); - - Ok(()) - } + )?; + assert_eq!( + CompatibilityError::MissingDefaultValue(String::from("newfield1")), + SchemaCompatibility::can_read(&writer_schema(), &reader_schema).unwrap_err() + ); + assert_eq!( + CompatibilityError::MissingDefaultValue(String::from("oldfield2")), + SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + ); - #[test] - fn test_array_writer_schema() { - let valid_reader = string_array_schema(); - let invalid_reader = string_map_schema(); + Ok(()) + } - assert!(SchemaCompatibility::can_read(&string_array_schema(), &valid_reader).is_ok()); - assert_eq!( - CompatibilityError::Inconclusive(String::from("writers_schema")), - SchemaCompatibility::can_read(&string_array_schema(), &invalid_reader).unwrap_err() - ); - } + #[test] + fn test_array_writer_schema() { + let valid_reader = string_array_schema(); + let invalid_reader = string_map_schema(); - #[test] - fn test_primitive_writer_schema() { - let valid_reader = Schema::String; - assert!(SchemaCompatibility::can_read(&Schema::String, &valid_reader).is_ok()); - assert_eq!( - CompatibilityError::TypeExpected { - schema_type: String::from("readers_schema"), - expected_type: vec![ - SchemaKind::Int, - SchemaKind::Long, - SchemaKind::Float, - SchemaKind::Double, - SchemaKind::Date, - SchemaKind::TimeMillis - ], - }, - SchemaCompatibility::can_read(&Schema::Int, &Schema::String).unwrap_err() - ); - } + assert!(SchemaCompatibility::can_read(&string_array_schema(), &valid_reader).is_ok()); + assert_eq!( + CompatibilityError::Inconclusive(String::from("writers_schema")), + SchemaCompatibility::can_read(&string_array_schema(), &invalid_reader).unwrap_err() + ); + } - #[test] - fn test_union_reader_writer_subset_incompatibility() { - // reader union schema must contain all writer union branches - let union_writer = union_schema(vec![Schema::Int, Schema::String]); - let union_reader = union_schema(vec![Schema::String]); - - assert_eq!( - CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&union_writer, &union_reader).unwrap_err() - ); - assert!(SchemaCompatibility::can_read(&union_reader, &union_writer).is_ok()); - } + #[test] + fn test_primitive_writer_schema() { + let valid_reader = Schema::String; + assert!(SchemaCompatibility::can_read(&Schema::String, &valid_reader).is_ok()); + assert_eq!( + CompatibilityError::TypeExpected { + schema_type: String::from("readers_schema"), + expected_type: vec![ + SchemaKind::Int, + SchemaKind::Long, + SchemaKind::Float, + SchemaKind::Double, + SchemaKind::Date, + SchemaKind::TimeMillis + ], + }, + SchemaCompatibility::can_read(&Schema::Int, &Schema::String).unwrap_err() + ); + } + + #[test] + fn test_union_reader_writer_subset_incompatibility() { + // reader union schema must contain all writer union branches + let union_writer = union_schema(vec![Schema::Int, Schema::String]); + let union_reader = union_schema(vec![Schema::String]); + + assert_eq!( + CompatibilityError::MissingUnionElements, + SchemaCompatibility::can_read(&union_writer, &union_reader).unwrap_err() + ); + assert!(SchemaCompatibility::can_read(&union_reader, &union_writer).is_ok()); + } - #[test] - fn test_incompatible_record_field() -> TestResult { - let string_schema = Schema::parse_str( - r#" + #[test] + fn test_incompatible_record_field() -> TestResult { + let string_schema = Schema::parse_str( + r#" {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [ {"name":"field1", "type":"string"} ]} "#, - )?; + )?; - let int_schema = Schema::parse_str( - r#" + let int_schema = Schema::parse_str( + r#" {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [ {"name":"field1", "type":"int"} ]} "#, - )?; - - assert_eq!( - CompatibilityError::FieldTypeMismatch( - "field1".to_owned(), - Box::new(CompatibilityError::TypeExpected { - schema_type: "readers_schema".to_owned(), - expected_type: vec![SchemaKind::String, SchemaKind::Bytes, SchemaKind::Uuid] - }) - ), - SchemaCompatibility::can_read(&string_schema, &int_schema).unwrap_err() - ); - - Ok(()) - } + )?; - #[test] - fn test_enum_symbols() -> TestResult { - let enum_schema1 = Schema::parse_str( - r#" + assert_eq!( + CompatibilityError::FieldTypeMismatch( + "field1".to_owned(), + Box::new(CompatibilityError::TypeExpected { + schema_type: "readers_schema".to_owned(), + expected_type: vec![ + SchemaKind::String, + SchemaKind::Bytes, + SchemaKind::Uuid + ] + }) + ), + SchemaCompatibility::can_read(&string_schema, &int_schema).unwrap_err() + ); + + Ok(()) + } + + #[test] + fn test_enum_symbols() -> TestResult { + let enum_schema1 = Schema::parse_str( + r#" {"type":"enum", "name":"MyEnum", "symbols":["A","B"]} "#, - )?; - let enum_schema2 = - Schema::parse_str(r#"{"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}"#)?; - assert_eq!( - CompatibilityError::MissingSymbols, - SchemaCompatibility::can_read(&enum_schema2, &enum_schema1).unwrap_err() - ); - assert!(SchemaCompatibility::can_read(&enum_schema1, &enum_schema2).is_ok()); - - Ok(()) - } + )?; + let enum_schema2 = + Schema::parse_str(r#"{"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}"#)?; + assert_eq!( + CompatibilityError::MissingSymbols, + SchemaCompatibility::can_read(&enum_schema2, &enum_schema1).unwrap_err() + ); + assert!(SchemaCompatibility::can_read(&enum_schema1, &enum_schema2).is_ok()); - fn point_2d_schema() -> Schema { - Schema::parse_str( - r#" + Ok(()) + } + + fn point_2d_schema() -> Schema { + Schema::parse_str( + r#" {"type":"record", "name":"Point2D", "fields":[ {"name":"x", "type":"double"}, {"name":"y", "type":"double"} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - fn point_2d_fullname_schema() -> Schema { - Schema::parse_str( - r#" + fn point_2d_fullname_schema() -> Schema { + Schema::parse_str( + r#" {"type":"record", "name":"Point", "namespace":"written", "fields":[ {"name":"x", "type":"double"}, {"name":"y", "type":"double"} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - fn point_3d_no_default_schema() -> Schema { - Schema::parse_str( - r#" + fn point_3d_no_default_schema() -> Schema { + Schema::parse_str( + r#" {"type":"record", "name":"Point", "fields":[ {"name":"x", "type":"double"}, {"name":"y", "type":"double"}, {"name":"z", "type":"double"} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - fn point_3d_schema() -> Schema { - Schema::parse_str( - r#" + fn point_3d_schema() -> Schema { + Schema::parse_str( + r#" {"type":"record", "name":"Point3D", "fields":[ {"name":"x", "type":"double"}, {"name":"y", "type":"double"}, {"name":"z", "type":"double", "default": 0.0} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - fn point_3d_match_name_schema() -> Schema { - Schema::parse_str( - r#" + fn point_3d_match_name_schema() -> Schema { + Schema::parse_str( + r#" {"type":"record", "name":"Point", "fields":[ {"name":"x", "type":"double"}, {"name":"y", "type":"double"}, {"name":"z", "type":"double", "default": 0.0} ]} "#, - ) - .unwrap() - } + ) + .unwrap() + } - #[test] - fn test_union_resolution_no_structure_match() { - // short name match, but no structure match - let read_schema = union_schema(vec![Schema::Null, point_3d_no_default_schema()]); - assert_eq!( - CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).unwrap_err() - ); - } + #[test] + fn test_union_resolution_no_structure_match() { + // short name match, but no structure match + let read_schema = union_schema(vec![Schema::Null, point_3d_no_default_schema()]); + assert_eq!( + CompatibilityError::MissingUnionElements, + SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + .unwrap_err() + ); + } - #[test] - fn test_union_resolution_first_structure_match_2d() { - // multiple structure matches with no name matches - let read_schema = union_schema(vec![ - Schema::Null, - point_3d_no_default_schema(), - point_2d_schema(), - point_3d_schema(), - ]); - assert_eq!( - CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).unwrap_err() - ); - } + #[test] + fn test_union_resolution_first_structure_match_2d() { + // multiple structure matches with no name matches + let read_schema = union_schema(vec![ + Schema::Null, + point_3d_no_default_schema(), + point_2d_schema(), + point_3d_schema(), + ]); + assert_eq!( + CompatibilityError::MissingUnionElements, + SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + .unwrap_err() + ); + } - #[test] - fn test_union_resolution_first_structure_match_3d() { - // multiple structure matches with no name matches - let read_schema = union_schema(vec![ - Schema::Null, - point_3d_no_default_schema(), - point_3d_schema(), - point_2d_schema(), - ]); - assert_eq!( - CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).unwrap_err() - ); - } + #[test] + fn test_union_resolution_first_structure_match_3d() { + // multiple structure matches with no name matches + let read_schema = union_schema(vec![ + Schema::Null, + point_3d_no_default_schema(), + point_3d_schema(), + point_2d_schema(), + ]); + assert_eq!( + CompatibilityError::MissingUnionElements, + SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + .unwrap_err() + ); + } - #[test] - fn test_union_resolution_named_structure_match() { - // multiple structure matches with a short name match - let read_schema = union_schema(vec![ - Schema::Null, - point_2d_schema(), - point_3d_match_name_schema(), - point_3d_schema(), - ]); - assert_eq!( - CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).unwrap_err() - ); - } + #[test] + fn test_union_resolution_named_structure_match() { + // multiple structure matches with a short name match + let read_schema = union_schema(vec![ + Schema::Null, + point_2d_schema(), + point_3d_match_name_schema(), + point_3d_schema(), + ]); + assert_eq!( + CompatibilityError::MissingUnionElements, + SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + .unwrap_err() + ); + } - #[test] - fn test_union_resolution_full_name_match() { - // there is a full name match that should be chosen - let read_schema = union_schema(vec![ - Schema::Null, - point_2d_schema(), - point_3d_match_name_schema(), - point_3d_schema(), - point_2d_fullname_schema(), - ]); - assert!(SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).is_ok()); - } + #[test] + fn test_union_resolution_full_name_match() { + // there is a full name match that should be chosen + let read_schema = union_schema(vec![ + Schema::Null, + point_2d_schema(), + point_3d_match_name_schema(), + point_3d_schema(), + point_2d_fullname_schema(), + ]); + assert!( + SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).is_ok() + ); + } - #[test] - fn test_avro_3772_enum_default() -> TestResult { - let writer_raw_schema = r#" + #[test] + fn test_avro_3772_enum_default() -> TestResult { + let writer_raw_schema = r#" { "type": "record", "name": "test", @@ -1430,7 +1471,7 @@ mod tests { } "#; - let reader_raw_schema = r#" + let reader_raw_schema = r#" { "type": "record", "name": "test", @@ -1449,32 +1490,32 @@ mod tests { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema)?; - let reader_schema = Schema::parse_str(reader_raw_schema)?; - let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); - let mut record = Record::new(writer.schema()).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - record.put("c", "clubs"); - writer.append(record).unwrap(); - let input = writer.into_inner()?; - let mut reader = Reader::with_schema(&reader_schema, &input[..])?; - assert_eq!( - reader.next().unwrap().unwrap(), - Value::Record(vec![ - ("a".to_string(), Value::Long(27)), - ("b".to_string(), Value::String("foo".to_string())), - ("c".to_string(), Value::Enum(1, "spades".to_string())), - ]) - ); - assert!(reader.next().is_none()); - - Ok(()) - } + let writer_schema = Schema::parse_str(writer_raw_schema)?; + let reader_schema = Schema::parse_str(reader_raw_schema)?; + let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); + let mut record = Record::new(writer.schema()).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + record.put("c", "clubs"); + writer.append(record).unwrap(); + let input = writer.into_inner()?; + let mut reader = Reader::with_schema(&reader_schema, &input[..])?; + assert_eq!( + reader.next().unwrap().unwrap(), + Value::Record(vec![ + ("a".to_string(), Value::Long(27)), + ("b".to_string(), Value::String("foo".to_string())), + ("c".to_string(), Value::Enum(1, "spades".to_string())), + ]) + ); + assert!(reader.next().is_none()); + + Ok(()) + } - #[test] - fn test_avro_3772_enum_default_less_symbols() -> TestResult { - let writer_raw_schema = r#" + #[test] + fn test_avro_3772_enum_default_less_symbols() -> TestResult { + let writer_raw_schema = r#" { "type": "record", "name": "test", @@ -1494,7 +1535,7 @@ mod tests { } "#; - let reader_raw_schema = r#" + let reader_raw_schema = r#" { "type": "record", "name": "test", @@ -1513,34 +1554,34 @@ mod tests { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema)?; - let reader_schema = Schema::parse_str(reader_raw_schema)?; - let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); - let mut record = Record::new(writer.schema()).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - record.put("c", "hearts"); - writer.append(record).unwrap(); - let input = writer.into_inner()?; - let mut reader = Reader::with_schema(&reader_schema, &input[..])?; - assert_eq!( - reader.next().unwrap().unwrap(), - Value::Record(vec![ - ("a".to_string(), Value::Long(27)), - ("b".to_string(), Value::String("foo".to_string())), - ("c".to_string(), Value::Enum(0, "hearts".to_string())), - ]) - ); - assert!(reader.next().is_none()); - - Ok(()) - } + let writer_schema = Schema::parse_str(writer_raw_schema)?; + let reader_schema = Schema::parse_str(reader_raw_schema)?; + let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); + let mut record = Record::new(writer.schema()).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + record.put("c", "hearts"); + writer.append(record).unwrap(); + let input = writer.into_inner()?; + let mut reader = Reader::with_schema(&reader_schema, &input[..])?; + assert_eq!( + reader.next().unwrap().unwrap(), + Value::Record(vec![ + ("a".to_string(), Value::Long(27)), + ("b".to_string(), Value::String("foo".to_string())), + ("c".to_string(), Value::Enum(0, "hearts".to_string())), + ]) + ); + assert!(reader.next().is_none()); - #[test] - fn avro_3894_take_aliases_into_account_when_serializing_for_schema_compatibility() -> TestResult - { - let schema_v1 = Schema::parse_str( - r#" + Ok(()) + } + + #[test] + fn avro_3894_take_aliases_into_account_when_serializing_for_schema_compatibility() + -> TestResult { + let schema_v1 = Schema::parse_str( + r#" { "type": "record", "name": "Conference", @@ -1550,10 +1591,10 @@ mod tests { {"type": "long", "name": "date"} ] }"#, - )?; + )?; - let schema_v2 = Schema::parse_str( - r#" + let schema_v2 = Schema::parse_str( + r#" { "type": "record", "name": "Conference", @@ -1563,17 +1604,17 @@ mod tests { {"type": "long", "name": "date", "aliases" : [ "time" ]} ] }"#, - )?; + )?; - assert!(SchemaCompatibility::mutual_read(&schema_v1, &schema_v2).is_ok()); + assert!(SchemaCompatibility::mutual_read(&schema_v1, &schema_v2).is_ok()); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3917_take_aliases_into_account_for_schema_compatibility() -> TestResult { - let schema_v1 = Schema::parse_str( - r#" + #[test] + fn avro_3917_take_aliases_into_account_for_schema_compatibility() -> TestResult { + let schema_v1 = Schema::parse_str( + r#" { "type": "record", "name": "Conference", @@ -1583,10 +1624,10 @@ mod tests { {"type": "long", "name": "date", "aliases" : [ "time" ]} ] }"#, - )?; + )?; - let schema_v2 = Schema::parse_str( - r#" + let schema_v2 = Schema::parse_str( + r#" { "type": "record", "name": "Conference", @@ -1596,24 +1637,24 @@ mod tests { {"type": "long", "name": "time"} ] }"#, - )?; + )?; - assert!(SchemaCompatibility::can_read(&schema_v2, &schema_v1).is_ok()); - assert_eq!( - CompatibilityError::MissingDefaultValue(String::from("time")), - SchemaCompatibility::can_read(&schema_v1, &schema_v2).unwrap_err() - ); + assert!(SchemaCompatibility::can_read(&schema_v2, &schema_v1).is_ok()); + assert_eq!( + CompatibilityError::MissingDefaultValue(String::from("time")), + SchemaCompatibility::can_read(&schema_v1, &schema_v2).unwrap_err() + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3898_record_schemas_match_by_unqualified_name() -> TestResult { - let schemas = [ - // Record schemas - ( - Schema::parse_str( - r#"{ + #[test] + fn test_avro_3898_record_schemas_match_by_unqualified_name() -> TestResult { + let schemas = [ + // Record schemas + ( + Schema::parse_str( + r#"{ "type": "record", "name": "Statistics", "fields": [ @@ -1623,9 +1664,9 @@ mod tests { { "name": "max", "type": "int", "default": 0 } ] }"#, - )?, - Schema::parse_str( - r#"{ + )?, + Schema::parse_str( + r#"{ "type": "record", "name": "Statistics", "namespace": "my.namespace", @@ -1636,59 +1677,59 @@ mod tests { { "name": "average", "type": "int", "default": 0} ] }"#, - )?, - ), - // Enum schemas - ( - Schema::parse_str( - r#"{ + )?, + ), + // Enum schemas + ( + Schema::parse_str( + r#"{ "type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs"] }"#, - )?, - Schema::parse_str( - r#"{ + )?, + Schema::parse_str( + r#"{ "type": "enum", "name": "Suit", "namespace": "my.namespace", "symbols": ["diamonds", "spades", "clubs", "hearts"] }"#, - )?, - ), - // Fixed schemas - ( - Schema::parse_str( - r#"{ + )?, + ), + // Fixed schemas + ( + Schema::parse_str( + r#"{ "type": "fixed", "name": "EmployeeId", "size": 16 }"#, - )?, - Schema::parse_str( - r#"{ + )?, + Schema::parse_str( + r#"{ "type": "fixed", "name": "EmployeeId", "namespace": "my.namespace", "size": 16 }"#, - )?, - ), - ]; + )?, + ), + ]; - for (schema_1, schema_2) in schemas { - assert!(SchemaCompatibility::can_read(&schema_1, &schema_2).is_ok()); - } + for (schema_1, schema_2) in schemas { + assert!(SchemaCompatibility::can_read(&schema_1, &schema_2).is_ok()); + } - Ok(()) - } + Ok(()) + } - #[test] - fn test_can_read_compatibility_errors() -> TestResult { - let schemas = [ - ( - Schema::parse_str( - r#"{ + #[test] + fn test_can_read_compatibility_errors() -> TestResult { + let schemas = [ + ( + Schema::parse_str( + r#"{ "type": "record", "name": "StatisticsMap", "fields": [ @@ -1696,9 +1737,9 @@ mod tests { {"name": "success", "type": {"type": "map", "values": "int"}} ] }"#, - )?, - Schema::parse_str( - r#"{ + )?, + Schema::parse_str( + r#"{ "type": "record", "name": "StatisticsMap", "fields": [ @@ -1706,49 +1747,49 @@ mod tests { {"name": "success", "type": ["null", {"type": "map", "values": "int"}], "default": null} ] }"#, - )?, - "Incompatible schemata! Field 'success' in reader schema does not match the type in the writer schema", - ), - ( - Schema::parse_str( - r#"{ + )?, + "Incompatible schemata! Field 'success' in reader schema does not match the type in the writer schema", + ), + ( + Schema::parse_str( + r#"{ "type": "record", "name": "StatisticsArray", "fields": [ {"name": "max_values", "type": {"type": "array", "items": "int"}} ] }"#, - )?, - Schema::parse_str( - r#"{ + )?, + Schema::parse_str( + r#"{ "type": "record", "name": "StatisticsArray", "fields": [ {"name": "max_values", "type": ["null", {"type": "array", "items": "int"}], "default": null} ] }"#, - )?, - "Incompatible schemata! Field 'max_values' in reader schema does not match the type in the writer schema", - ), - ]; + )?, + "Incompatible schemata! Field 'max_values' in reader schema does not match the type in the writer schema", + ), + ]; + + for (schema_1, schema_2, error) in schemas { + assert!(SchemaCompatibility::can_read(&schema_1, &schema_2).is_ok()); + assert_eq!( + error, + SchemaCompatibility::can_read(&schema_2, &schema_1) + .unwrap_err() + .to_string() + ); + } - for (schema_1, schema_2, error) in schemas { - assert!(SchemaCompatibility::can_read(&schema_1, &schema_2).is_ok()); - assert_eq!( - error, - SchemaCompatibility::can_read(&schema_2, &schema_1) - .unwrap_err() - .to_string() - ); + Ok(()) } - Ok(()) - } - - #[test] - fn avro_3974_can_read_schema_references() -> TestResult { - let schema_strs = vec![ - r#"{ + #[test] + fn avro_3974_can_read_schema_references() -> TestResult { + let schema_strs = vec![ + r#"{ "type": "record", "name": "Child", "namespace": "avro", @@ -1760,7 +1801,7 @@ mod tests { ] } "#, - r#"{ + r#"{ "type": "record", "name": "Parent", "namespace": "avro", @@ -1772,11 +1813,12 @@ mod tests { ] } "#, - ]; + ]; - let schemas = Schema::parse_list(schema_strs).unwrap(); - SchemaCompatibility::can_read(&schemas[1], &schemas[1])?; + let schemas = Schema::parse_list(schema_strs).unwrap(); + SchemaCompatibility::can_read(&schemas[1], &schemas[1])?; - Ok(()) + Ok(()) + } } } diff --git a/avro/src/schema_equality.rs b/avro/src/schema_equality.rs index b884a0f3..9609e956 100644 --- a/avro/src/schema_equality.rs +++ b/avro/src/schema_equality.rs @@ -15,254 +15,270 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - Schema, - schema::{ +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead + Unpin => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod schema_equality { + use crate::schema::tokio::{ ArraySchema, DecimalSchema, EnumSchema, FixedSchema, MapSchema, RecordField, RecordSchema, - UnionSchema, - }, -}; -use log::{debug, error}; -use std::{fmt::Debug, sync::OnceLock}; - -/// A trait that compares two schemata for equality. -/// To register a custom one use [set_schemata_equality_comparator]. -pub trait SchemataEq: Debug + Send + Sync { - /// Compares two schemata for equality. - fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool; -} + Schema, UnionSchema, + }; + use log::{debug, error}; + use std::{fmt::Debug, sync::OnceLock}; + + /// A trait that compares two schemata for equality. + /// To register a custom one use [set_schemata_equality_comparator]. + pub trait SchemataEq: Debug + Send + Sync { + /// Compares two schemata for equality. + fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool; + } -/// Compares two schemas according to the Avro specification by using -/// their canonical forms. -/// See -#[derive(Debug)] -pub struct SpecificationEq; -impl SchemataEq for SpecificationEq { - fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { - schema_one.canonical_form() == schema_two.canonical_form() + /// Compares two schemas according to the Avro specification by using + /// their canonical forms. + /// See + #[derive(Debug)] + pub struct SpecificationEq; + impl SchemataEq for SpecificationEq { + fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { + schema_one.canonical_form() == schema_two.canonical_form() + } } -} -/// Compares two schemas for equality field by field, using only the fields that -/// are used to construct their canonical forms. -/// See -#[derive(Debug)] -pub struct StructFieldEq { - /// Whether to include custom attributes in the comparison. - /// The custom attributes are not used to construct the canonical form of the schema! - pub include_attributes: bool, -} + /// Compares two schemas for equality field by field, using only the fields that + /// are used to construct their canonical forms. + /// See + #[derive(Debug)] + pub struct StructFieldEq { + /// Whether to include custom attributes in the comparison. + /// The custom attributes are not used to construct the canonical form of the schema! + pub include_attributes: bool, + } -impl SchemataEq for StructFieldEq { - fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { - macro_rules! compare_primitive { - ($primitive:ident) => { - if let Schema::$primitive = schema_one { - if let Schema::$primitive = schema_two { - return true; + impl SchemataEq for StructFieldEq { + fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { + macro_rules! compare_primitive { + ($primitive:ident) => { + if let Schema::$primitive = schema_one { + if let Schema::$primitive = schema_two { + return true; + } + return false; } - return false; - } - }; - } + }; + } - if schema_one.name() != schema_two.name() { - return false; - } + if schema_one.name() != schema_two.name() { + return false; + } - compare_primitive!(Null); - compare_primitive!(Boolean); - compare_primitive!(Int); - compare_primitive!(Int); - compare_primitive!(Long); - compare_primitive!(Float); - compare_primitive!(Double); - compare_primitive!(Bytes); - compare_primitive!(String); - compare_primitive!(Uuid); - compare_primitive!(BigDecimal); - compare_primitive!(Date); - compare_primitive!(Duration); - compare_primitive!(TimeMicros); - compare_primitive!(TimeMillis); - compare_primitive!(TimestampMicros); - compare_primitive!(TimestampMillis); - compare_primitive!(TimestampNanos); - compare_primitive!(LocalTimestampMicros); - compare_primitive!(LocalTimestampMillis); - compare_primitive!(LocalTimestampNanos); - - if self.include_attributes - && schema_one.custom_attributes() != schema_two.custom_attributes() - { - return false; - } + compare_primitive!(Null); + compare_primitive!(Boolean); + compare_primitive!(Int); + compare_primitive!(Int); + compare_primitive!(Long); + compare_primitive!(Float); + compare_primitive!(Double); + compare_primitive!(Bytes); + compare_primitive!(String); + compare_primitive!(Uuid); + compare_primitive!(BigDecimal); + compare_primitive!(Date); + compare_primitive!(Duration); + compare_primitive!(TimeMicros); + compare_primitive!(TimeMillis); + compare_primitive!(TimestampMicros); + compare_primitive!(TimestampMillis); + compare_primitive!(TimestampNanos); + compare_primitive!(LocalTimestampMicros); + compare_primitive!(LocalTimestampMillis); + compare_primitive!(LocalTimestampNanos); + + if self.include_attributes + && schema_one.custom_attributes() != schema_two.custom_attributes() + { + return false; + } - if let Schema::Record(RecordSchema { - fields: fields_one, .. - }) = schema_one - { if let Schema::Record(RecordSchema { - fields: fields_two, .. - }) = schema_two + fields: fields_one, .. + }) = schema_one { - return self.compare_fields(fields_one, fields_two); + if let Schema::Record(RecordSchema { + fields: fields_two, .. + }) = schema_two + { + return self.compare_fields(fields_one, fields_two); + } + return false; } - return false; - } - if let Schema::Enum(EnumSchema { - symbols: symbols_one, - .. - }) = schema_one - { if let Schema::Enum(EnumSchema { - symbols: symbols_two, + symbols: symbols_one, .. - }) = schema_two + }) = schema_one { - return symbols_one == symbols_two; + if let Schema::Enum(EnumSchema { + symbols: symbols_two, + .. + }) = schema_two + { + return symbols_one == symbols_two; + } + return false; } - return false; - } - if let Schema::Fixed(FixedSchema { size: size_one, .. }) = schema_one { - if let Schema::Fixed(FixedSchema { size: size_two, .. }) = schema_two { - return size_one == size_two; + if let Schema::Fixed(FixedSchema { size: size_one, .. }) = schema_one { + if let Schema::Fixed(FixedSchema { size: size_two, .. }) = schema_two { + return size_one == size_two; + } + return false; } - return false; - } - if let Schema::Union(UnionSchema { - schemas: schemas_one, - .. - }) = schema_one - { if let Schema::Union(UnionSchema { - schemas: schemas_two, + schemas: schemas_one, .. - }) = schema_two + }) = schema_one { - return schemas_one.len() == schemas_two.len() - && schemas_one - .iter() - .zip(schemas_two.iter()) - .all(|(s1, s2)| self.compare(s1, s2)); + if let Schema::Union(UnionSchema { + schemas: schemas_two, + .. + }) = schema_two + { + return schemas_one.len() == schemas_two.len() + && schemas_one + .iter() + .zip(schemas_two.iter()) + .all(|(s1, s2)| self.compare(s1, s2)); + } + return false; } - return false; - } - if let Schema::Decimal(DecimalSchema { - precision: precision_one, - scale: scale_one, - .. - }) = schema_one - { if let Schema::Decimal(DecimalSchema { - precision: precision_two, - scale: scale_two, + precision: precision_one, + scale: scale_one, .. - }) = schema_two + }) = schema_one { - return precision_one == precision_two && scale_one == scale_two; + if let Schema::Decimal(DecimalSchema { + precision: precision_two, + scale: scale_two, + .. + }) = schema_two + { + return precision_one == precision_two && scale_one == scale_two; + } + return false; } - return false; - } - if let Schema::Array(ArraySchema { - items: items_one, .. - }) = schema_one - { if let Schema::Array(ArraySchema { - items: items_two, .. - }) = schema_two + items: items_one, .. + }) = schema_one { - return items_one == items_two; + if let Schema::Array(ArraySchema { + items: items_two, .. + }) = schema_two + { + return items_one == items_two; + } + return false; } - return false; - } - if let Schema::Map(MapSchema { - types: types_one, .. - }) = schema_one - { if let Schema::Map(MapSchema { - types: types_two, .. - }) = schema_two + types: types_one, .. + }) = schema_one { - return self.compare(types_one, types_two); + if let Schema::Map(MapSchema { + types: types_two, .. + }) = schema_two + { + return self.compare(types_one, types_two); + } + return false; } - return false; - } - if let Schema::Ref { name: name_one } = schema_one { - if let Schema::Ref { name: name_two } = schema_two { - return name_one == name_two; + if let Schema::Ref { name: name_one } = schema_one { + if let Schema::Ref { name: name_two } = schema_two { + return name_one == name_two; + } + return false; } - return false; - } - error!( - "This is a bug in schema_equality.rs! The following schemata types are not checked! \ + error!( + "This is a bug in schema_equality.rs! The following schemata types are not checked! \ Please report it to the Avro library maintainers! \ \n{schema_one:?}\n\n{schema_two:?}" - ); - false + ); + false + } } -} -impl StructFieldEq { - fn compare_fields(&self, fields_one: &[RecordField], fields_two: &[RecordField]) -> bool { - fields_one.len() == fields_two.len() - && fields_one - .iter() - .zip(fields_two.iter()) - .all(|(f1, f2)| self.compare(&f1.schema, &f2.schema)) + impl StructFieldEq { + fn compare_fields(&self, fields_one: &[RecordField], fields_two: &[RecordField]) -> bool { + fields_one.len() == fields_two.len() + && fields_one + .iter() + .zip(fields_two.iter()) + .all(|(f1, f2)| self.compare(&f1.schema, &f2.schema)) + } } -} -static SCHEMATA_COMPARATOR_ONCE: OnceLock> = OnceLock::new(); - -/// Sets a custom schemata equality comparator. -/// -/// Returns a unit if the registration was successful or the already -/// registered comparator if the registration failed. -/// -/// **Note**: This function must be called before parsing any schema because this will -/// register the default comparator and the registration is one time only! -pub fn set_schemata_equality_comparator( - comparator: Box, -) -> Result<(), Box> { - debug!("Setting a custom schemata equality comparator: {comparator:?}."); - SCHEMATA_COMPARATOR_ONCE.set(comparator) -} + static SCHEMATA_COMPARATOR_ONCE: OnceLock> = OnceLock::new(); + + /// Sets a custom schemata equality comparator. + /// + /// Returns a unit if the registration was successful or the already + /// registered comparator if the registration failed. + /// + /// **Note**: This function must be called before parsing any schema because this will + /// register the default comparator and the registration is one time only! + pub fn set_schemata_equality_comparator( + comparator: Box, + ) -> Result<(), Box> { + debug!("Setting a custom schemata equality comparator: {comparator:?}."); + SCHEMATA_COMPARATOR_ONCE.set(comparator) + } -pub(crate) fn compare_schemata(schema_one: &Schema, schema_two: &Schema) -> bool { - SCHEMATA_COMPARATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default schemata equality comparator: SpecificationEq.",); - Box::new(StructFieldEq { - include_attributes: false, + pub(crate) fn compare_schemata(schema_one: &Schema, schema_two: &Schema) -> bool { + SCHEMATA_COMPARATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default schemata equality comparator: SpecificationEq.",); + Box::new(StructFieldEq { + include_attributes: false, + }) }) - }) - .compare(schema_one, schema_two) -} + .compare(schema_one, schema_two) + } -#[cfg(test)] -#[allow(non_snake_case)] -mod tests { - use super::*; - use crate::schema::{Name, RecordFieldOrder}; - use apache_avro_test_helper::TestResult; - use serde_json::Value; - use std::collections::BTreeMap; - - const SPECIFICATION_EQ: SpecificationEq = SpecificationEq; - const STRUCT_FIELD_EQ: StructFieldEq = StructFieldEq { - include_attributes: false, - }; + #[cfg(test)] + #[allow(non_snake_case)] + mod tests { + use super::*; + use crate::schema::{Name, RecordFieldOrder}; + use apache_avro_test_helper::TestResult; + use serde_json::Value; + use std::collections::BTreeMap; + + const SPECIFICATION_EQ: SpecificationEq = SpecificationEq; + const STRUCT_FIELD_EQ: StructFieldEq = StructFieldEq { + include_attributes: false, + }; - macro_rules! test_primitives { + macro_rules! test_primitives { ($primitive:ident) => { paste::item! { #[test] @@ -275,313 +291,314 @@ mod tests { }; } - test_primitives!(Null); - test_primitives!(Boolean); - test_primitives!(Int); - test_primitives!(Long); - test_primitives!(Float); - test_primitives!(Double); - test_primitives!(Bytes); - test_primitives!(String); - test_primitives!(Uuid); - test_primitives!(BigDecimal); - test_primitives!(Date); - test_primitives!(Duration); - test_primitives!(TimeMicros); - test_primitives!(TimeMillis); - test_primitives!(TimestampMicros); - test_primitives!(TimestampMillis); - test_primitives!(TimestampNanos); - test_primitives!(LocalTimestampMicros); - test_primitives!(LocalTimestampMillis); - test_primitives!(LocalTimestampNanos); - - #[test] - fn test_avro_3939_compare_named_schemata_with_different_names() { - let schema_one = Schema::Ref { - name: Name::from("name1"), - }; + test_primitives!(Null); + test_primitives!(Boolean); + test_primitives!(Int); + test_primitives!(Long); + test_primitives!(Float); + test_primitives!(Double); + test_primitives!(Bytes); + test_primitives!(String); + test_primitives!(Uuid); + test_primitives!(BigDecimal); + test_primitives!(Date); + test_primitives!(Duration); + test_primitives!(TimeMicros); + test_primitives!(TimeMillis); + test_primitives!(TimestampMicros); + test_primitives!(TimestampMillis); + test_primitives!(TimestampNanos); + test_primitives!(LocalTimestampMicros); + test_primitives!(LocalTimestampMillis); + test_primitives!(LocalTimestampNanos); + + #[test] + fn test_avro_3939_compare_named_schemata_with_different_names() { + let schema_one = Schema::Ref { + name: Name::from("name1"), + }; - let schema_two = Schema::Ref { - name: Name::from("name2"), - }; + let schema_two = Schema::Ref { + name: Name::from("name2"), + }; - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - assert!(!specification_eq_res); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!(!struct_field_eq_res); + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + assert!(!specification_eq_res); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!(!struct_field_eq_res); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_schemata_not_including_attributes() { - let schema_one = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), - ); - let schema_two = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), - ); - // STRUCT_FIELD_EQ does not include attributes ! - assert!(STRUCT_FIELD_EQ.compare(&schema_one, &schema_two)); - } + #[test] + fn test_avro_3939_compare_schemata_not_including_attributes() { + let schema_one = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), + ); + let schema_two = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), + ); + // STRUCT_FIELD_EQ does not include attributes ! + assert!(STRUCT_FIELD_EQ.compare(&schema_one, &schema_two)); + } - #[test] - fn test_avro_3939_compare_schemata_including_attributes() { - let struct_field_eq = StructFieldEq { - include_attributes: true, - }; - let schema_one = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), - ); - let schema_two = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), - ); - assert!(!struct_field_eq.compare(&schema_one, &schema_two)); - } + #[test] + fn test_avro_3939_compare_schemata_including_attributes() { + let struct_field_eq = StructFieldEq { + include_attributes: true, + }; + let schema_one = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), + ); + let schema_two = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), + ); + assert!(!struct_field_eq.compare(&schema_one, &schema_two)); + } - #[test] - fn test_avro_3939_compare_map_schemata() { - let schema_one = Schema::map(Schema::Boolean); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::map(Schema::Boolean); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Map failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Map failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_map_schemata() { + let schema_one = Schema::map(Schema::Boolean); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::map(Schema::Boolean); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Map failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Map failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_array_schemata() { - let schema_one = Schema::array(Schema::Boolean); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::array(Schema::Boolean); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Array failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Array failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_array_schemata() { + let schema_one = Schema::array(Schema::Boolean); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::array(Schema::Boolean); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Array failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Array failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_decimal_schemata() { - let schema_one = Schema::Decimal(DecimalSchema { - precision: 10, - scale: 2, - inner: Box::new(Schema::Bytes), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Decimal(DecimalSchema { - precision: 10, - scale: 2, - inner: Box::new(Schema::Bytes), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Decimal failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Decimal failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_decimal_schemata() { + let schema_one = Schema::Decimal(DecimalSchema { + precision: 10, + scale: 2, + inner: Box::new(Schema::Bytes), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Decimal(DecimalSchema { + precision: 10, + scale: 2, + inner: Box::new(Schema::Bytes), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Decimal failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Decimal failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_fixed_schemata() { - let schema_one = Schema::Fixed(FixedSchema { - name: Name::from("fixed"), - doc: None, - size: 10, - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Fixed(FixedSchema { - name: Name::from("fixed"), - doc: None, - size: 10, - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Fixed failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Fixed failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_fixed_schemata() { + let schema_one = Schema::Fixed(FixedSchema { + name: Name::from("fixed"), + doc: None, + size: 10, + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - #[test] - fn test_avro_3939_compare_enum_schemata() { - let schema_one = Schema::Enum(EnumSchema { - name: Name::from("enum"), - doc: None, - symbols: vec!["A".to_string(), "B".to_string()], - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Enum(EnumSchema { - name: Name::from("enum"), - doc: None, - symbols: vec!["A".to_string(), "B".to_string()], - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Enum failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Enum failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + let schema_two = Schema::Fixed(FixedSchema { + name: Name::from("fixed"), + doc: None, + size: 10, + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Fixed failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Fixed failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_ref_schemata() { - let schema_one = Schema::Ref { - name: Name::from("ref"), - }; - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + #[test] + fn test_avro_3939_compare_enum_schemata() { + let schema_one = Schema::Enum(EnumSchema { + name: Name::from("enum"), + doc: None, + symbols: vec!["A".to_string(), "B".to_string()], + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - let schema_two = Schema::Ref { - name: Name::from("ref"), - }; + let schema_two = Schema::Enum(EnumSchema { + name: Name::from("enum"), + doc: None, + symbols: vec!["A".to_string(), "B".to_string()], + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Enum failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Enum failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Ref failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Ref failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_ref_schemata() { + let schema_one = Schema::Ref { + name: Name::from("ref"), + }; + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Ref { + name: Name::from("ref"), + }; + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Ref failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Ref failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_record_schemata() { - let schema_one = Schema::Record(RecordSchema { - name: Name::from("record"), - doc: None, - fields: vec![RecordField { - name: "field".to_string(), + #[test] + fn test_avro_3939_compare_record_schemata() { + let schema_one = Schema::Record(RecordSchema { + name: Name::from("record"), doc: None, - default: None, - schema: Schema::Boolean, - order: RecordFieldOrder::Ignore, + fields: vec![RecordField { + name: "field".to_string(), + doc: None, + default: None, + schema: Schema::Boolean, + order: RecordFieldOrder::Ignore, + aliases: None, + custom_attributes: BTreeMap::new(), + position: 0, + }], aliases: None, - custom_attributes: BTreeMap::new(), - position: 0, - }], - aliases: None, - attributes: BTreeMap::new(), - lookup: Default::default(), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Record(RecordSchema { - name: Name::from("record"), - doc: None, - fields: vec![RecordField { - name: "field".to_string(), + attributes: BTreeMap::new(), + lookup: Default::default(), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Record(RecordSchema { + name: Name::from("record"), doc: None, - default: None, - schema: Schema::Boolean, - order: RecordFieldOrder::Ignore, + fields: vec![RecordField { + name: "field".to_string(), + doc: None, + default: None, + schema: Schema::Boolean, + order: RecordFieldOrder::Ignore, + aliases: None, + custom_attributes: BTreeMap::new(), + position: 0, + }], aliases: None, - custom_attributes: BTreeMap::new(), - position: 0, - }], - aliases: None, - attributes: BTreeMap::new(), - lookup: Default::default(), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Record failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Record failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + attributes: BTreeMap::new(), + lookup: Default::default(), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Record failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Record failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_union_schemata() -> TestResult { - let schema_one = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Union failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Union failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - Ok(()) + #[test] + fn test_avro_3939_compare_union_schemata() -> TestResult { + let schema_one = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Union failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Union failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + Ok(()) + } } } diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 8e6e595e..6a3d1a48 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -16,1031 +16,1054 @@ // under the License. //! Logic for serde-compatible serialization. -use crate::{ - Error, - bytes::{BytesType, SER_BYTES_TYPE}, - types::Value, -}; -use serde::{Serialize, ser}; -use std::{collections::HashMap, iter::once}; - -#[derive(Clone, Default)] -pub struct Serializer {} - -pub struct SeqSerializer { - items: Vec, -} -pub struct SeqVariantSerializer<'a> { - index: u32, - variant: &'a str, - items: Vec, -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + headers::tokio => headers::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + types::tokio => types::sync, + #[tokio::test] => #[test] + ); + } +)] +mod ser { + + use crate::{ + Error, + bytes::{BytesType, SER_BYTES_TYPE}, + types::tokio::Value, + }; + use serde::{Serialize, ser}; + use std::{collections::HashMap, iter::once}; + + #[derive(Clone, Default)] + pub struct Serializer {} + + pub struct SeqSerializer { + items: Vec, + } + + pub struct SeqVariantSerializer<'a> { + index: u32, + variant: &'a str, + items: Vec, + } -pub struct MapSerializer { - indices: HashMap, - values: Vec, -} + pub struct MapSerializer { + indices: HashMap, + values: Vec, + } -pub struct StructSerializer { - fields: Vec<(String, Value)>, -} + pub struct StructSerializer { + fields: Vec<(String, Value)>, + } -pub struct StructVariantSerializer<'a> { - index: u32, - variant: &'a str, - fields: Vec<(String, Value)>, -} + pub struct StructVariantSerializer<'a> { + index: u32, + variant: &'a str, + fields: Vec<(String, Value)>, + } -impl SeqSerializer { - pub fn new(len: Option) -> SeqSerializer { - let items = match len { - Some(len) => Vec::with_capacity(len), - None => Vec::new(), - }; + impl SeqSerializer { + pub fn new(len: Option) -> SeqSerializer { + let items = match len { + Some(len) => Vec::with_capacity(len), + None => Vec::new(), + }; - SeqSerializer { items } + SeqSerializer { items } + } } -} -impl<'a> SeqVariantSerializer<'a> { - pub fn new(index: u32, variant: &'a str, len: Option) -> SeqVariantSerializer<'a> { - let items = match len { - Some(len) => Vec::with_capacity(len), - None => Vec::new(), - }; - SeqVariantSerializer { - index, - variant, - items, + impl<'a> SeqVariantSerializer<'a> { + pub fn new(index: u32, variant: &'a str, len: Option) -> SeqVariantSerializer<'a> { + let items = match len { + Some(len) => Vec::with_capacity(len), + None => Vec::new(), + }; + SeqVariantSerializer { + index, + variant, + items, + } } } -} -impl MapSerializer { - pub fn new(len: Option) -> MapSerializer { - let (indices, values) = match len { - Some(len) => (HashMap::with_capacity(len), Vec::with_capacity(len)), - None => (HashMap::new(), Vec::new()), - }; + impl MapSerializer { + pub fn new(len: Option) -> MapSerializer { + let (indices, values) = match len { + Some(len) => (HashMap::with_capacity(len), Vec::with_capacity(len)), + None => (HashMap::new(), Vec::new()), + }; - MapSerializer { indices, values } + MapSerializer { indices, values } + } } -} -impl StructSerializer { - pub fn new(len: usize) -> StructSerializer { - StructSerializer { - fields: Vec::with_capacity(len), + impl StructSerializer { + pub fn new(len: usize) -> StructSerializer { + StructSerializer { + fields: Vec::with_capacity(len), + } } } -} -impl<'a> StructVariantSerializer<'a> { - pub fn new(index: u32, variant: &'a str, len: usize) -> StructVariantSerializer<'a> { - StructVariantSerializer { - index, - variant, - fields: Vec::with_capacity(len), + impl<'a> StructVariantSerializer<'a> { + pub fn new(index: u32, variant: &'a str, len: usize) -> StructVariantSerializer<'a> { + StructVariantSerializer { + index, + variant, + fields: Vec::with_capacity(len), + } } } -} -impl<'b> ser::Serializer for &'b mut Serializer { - type Ok = Value; - type Error = Error; - type SerializeSeq = SeqSerializer; - type SerializeTuple = SeqSerializer; - type SerializeTupleStruct = SeqSerializer; - type SerializeTupleVariant = SeqVariantSerializer<'b>; - type SerializeMap = MapSerializer; - type SerializeStruct = StructSerializer; - type SerializeStructVariant = StructVariantSerializer<'b>; - - fn serialize_bool(self, v: bool) -> Result { - Ok(Value::Boolean(v)) - } + impl<'b> ser::Serializer for &'b mut Serializer { + type Ok = Value; + type Error = Error; + type SerializeSeq = SeqSerializer; + type SerializeTuple = SeqSerializer; + type SerializeTupleStruct = SeqSerializer; + type SerializeTupleVariant = SeqVariantSerializer<'b>; + type SerializeMap = MapSerializer; + type SerializeStruct = StructSerializer; + type SerializeStructVariant = StructVariantSerializer<'b>; - fn serialize_i8(self, v: i8) -> Result { - self.serialize_i32(i32::from(v)) - } + fn serialize_bool(self, v: bool) -> Result { + Ok(Value::Boolean(v)) + } - fn serialize_i16(self, v: i16) -> Result { - self.serialize_i32(i32::from(v)) - } + fn serialize_i8(self, v: i8) -> Result { + self.serialize_i32(i32::from(v)) + } - fn serialize_i32(self, v: i32) -> Result { - Ok(Value::Int(v)) - } + fn serialize_i16(self, v: i16) -> Result { + self.serialize_i32(i32::from(v)) + } - fn serialize_i64(self, v: i64) -> Result { - Ok(Value::Long(v)) - } + fn serialize_i32(self, v: i32) -> Result { + Ok(Value::Int(v)) + } - fn serialize_u8(self, v: u8) -> Result { - self.serialize_i32(i32::from(v)) - } + fn serialize_i64(self, v: i64) -> Result { + Ok(Value::Long(v)) + } - fn serialize_u16(self, v: u16) -> Result { - self.serialize_i32(i32::from(v)) - } + fn serialize_u8(self, v: u8) -> Result { + self.serialize_i32(i32::from(v)) + } - fn serialize_u32(self, v: u32) -> Result { - if v <= i32::MAX as u32 { - self.serialize_i32(v as i32) - } else { - self.serialize_i64(i64::from(v)) + fn serialize_u16(self, v: u16) -> Result { + self.serialize_i32(i32::from(v)) } - } - fn serialize_u64(self, v: u64) -> Result { - if v <= i64::MAX as u64 { - self.serialize_i64(v as i64) - } else { - Err(ser::Error::custom("u64 is too large")) + fn serialize_u32(self, v: u32) -> Result { + if v <= i32::MAX as u32 { + self.serialize_i32(v as i32) + } else { + self.serialize_i64(i64::from(v)) + } } - } - fn serialize_f32(self, v: f32) -> Result { - Ok(Value::Float(v)) - } + fn serialize_u64(self, v: u64) -> Result { + if v <= i64::MAX as u64 { + self.serialize_i64(v as i64) + } else { + Err(ser::Error::custom("u64 is too large")) + } + } - fn serialize_f64(self, v: f64) -> Result { - Ok(Value::Double(v)) - } + fn serialize_f32(self, v: f32) -> Result { + Ok(Value::Float(v)) + } - fn serialize_char(self, v: char) -> Result { - self.serialize_str(&once(v).collect::()) - } + fn serialize_f64(self, v: f64) -> Result { + Ok(Value::Double(v)) + } - fn serialize_str(self, v: &str) -> Result { - Ok(Value::String(v.to_owned())) - } + fn serialize_char(self, v: char) -> Result { + self.serialize_str(&once(v).collect::()) + } - fn serialize_bytes(self, v: &[u8]) -> Result { - match SER_BYTES_TYPE.get() { - BytesType::Bytes => Ok(Value::Bytes(v.to_owned())), - BytesType::Fixed => Ok(Value::Fixed(v.len(), v.to_owned())), + fn serialize_str(self, v: &str) -> Result { + Ok(Value::String(v.to_owned())) } - } - fn serialize_none(self) -> Result { - Ok(Value::from(None::)) - } + fn serialize_bytes(self, v: &[u8]) -> Result { + match SER_BYTES_TYPE.get() { + BytesType::Bytes => Ok(Value::Bytes(v.to_owned())), + BytesType::Fixed => Ok(Value::Fixed(v.len(), v.to_owned())), + } + } - fn serialize_some(self, value: &T) -> Result - where - T: Serialize + ?Sized, - { - let v = value.serialize(&mut Serializer::default())?; - Ok(Value::from(Some(v))) - } + fn serialize_none(self) -> Result { + Ok(Value::from(None::)) + } - fn serialize_unit(self) -> Result { - Ok(Value::Null) - } + fn serialize_some(self, value: &T) -> Result + where + T: Serialize + ?Sized, + { + let v = value.serialize(&mut Serializer::default())?; + Ok(Value::from(Some(v))) + } - fn serialize_unit_struct(self, _: &'static str) -> Result { - self.serialize_unit() - } + fn serialize_unit(self) -> Result { + Ok(Value::Null) + } - fn serialize_unit_variant( - self, - _: &'static str, - _variant_index: u32, - variant: &'static str, - ) -> Result { - Ok(Value::String(variant.to_string())) - } + fn serialize_unit_struct(self, _: &'static str) -> Result { + self.serialize_unit() + } - fn serialize_newtype_struct( - self, - _: &'static str, - value: &T, - ) -> Result - where - T: Serialize + ?Sized, - { - value.serialize(self) - } + fn serialize_unit_variant( + self, + _: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result { + Ok(Value::String(variant.to_string())) + } - fn serialize_newtype_variant( - self, - _: &'static str, - index: u32, - variant: &'static str, - value: &T, - ) -> Result - where - T: Serialize + ?Sized, - { - Ok(Value::Record(vec![ - ("type".to_owned(), Value::Enum(index, variant.to_owned())), - ( - "value".to_owned(), - Value::Union(index, Box::new(value.serialize(self)?)), - ), - ])) - } + fn serialize_newtype_struct( + self, + _: &'static str, + value: &T, + ) -> Result + where + T: Serialize + ?Sized, + { + value.serialize(self) + } - fn serialize_seq(self, len: Option) -> Result { - Ok(SeqSerializer::new(len)) - } + fn serialize_newtype_variant( + self, + _: &'static str, + index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: Serialize + ?Sized, + { + Ok(Value::Record(vec![ + ("type".to_owned(), Value::Enum(index, variant.to_owned())), + ( + "value".to_owned(), + Value::Union(index, Box::new(value.serialize(self)?)), + ), + ])) + } - fn serialize_tuple(self, len: usize) -> Result { - self.serialize_seq(Some(len)) - } + fn serialize_seq(self, len: Option) -> Result { + Ok(SeqSerializer::new(len)) + } - fn serialize_tuple_struct( - self, - _: &'static str, - len: usize, - ) -> Result { - self.serialize_seq(Some(len)) - } + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_seq(Some(len)) + } - fn serialize_tuple_variant( - self, - _: &'static str, - index: u32, - variant: &'static str, - len: usize, - ) -> Result { - Ok(SeqVariantSerializer::new(index, variant, Some(len))) - } + fn serialize_tuple_struct( + self, + _: &'static str, + len: usize, + ) -> Result { + self.serialize_seq(Some(len)) + } - fn serialize_map(self, len: Option) -> Result { - Ok(MapSerializer::new(len)) - } + fn serialize_tuple_variant( + self, + _: &'static str, + index: u32, + variant: &'static str, + len: usize, + ) -> Result { + Ok(SeqVariantSerializer::new(index, variant, Some(len))) + } - fn serialize_struct( - self, - _: &'static str, - len: usize, - ) -> Result { - Ok(StructSerializer::new(len)) - } + fn serialize_map(self, len: Option) -> Result { + Ok(MapSerializer::new(len)) + } - fn serialize_struct_variant( - self, - _: &'static str, - index: u32, - variant: &'static str, - len: usize, - ) -> Result { - Ok(StructVariantSerializer::new(index, variant, len)) - } + fn serialize_struct( + self, + _: &'static str, + len: usize, + ) -> Result { + Ok(StructSerializer::new(len)) + } - fn is_human_readable(&self) -> bool { - crate::util::is_human_readable() - } -} + fn serialize_struct_variant( + self, + _: &'static str, + index: u32, + variant: &'static str, + len: usize, + ) -> Result { + Ok(StructVariantSerializer::new(index, variant, len)) + } -impl ser::SerializeSeq for SeqSerializer { - type Ok = Value; - type Error = Error; - - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - self.items - .push(value.serialize(&mut Serializer::default())?); - Ok(()) + fn is_human_readable(&self) -> bool { + crate::util::is_human_readable() + } } - fn end(self) -> Result { - Ok(Value::Array(self.items)) - } -} + impl ser::SerializeSeq for SeqSerializer { + type Ok = Value; + type Error = Error; -impl ser::SerializeTuple for SeqSerializer { - type Ok = Value; - type Error = Error; + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + self.items + .push(value.serialize(&mut Serializer::default())?); + Ok(()) + } - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - ser::SerializeSeq::serialize_element(self, value) + fn end(self) -> Result { + Ok(Value::Array(self.items)) + } } - fn end(self) -> Result { - ser::SerializeSeq::end(self) - } -} + impl ser::SerializeTuple for SeqSerializer { + type Ok = Value; + type Error = Error; -impl ser::SerializeTupleStruct for SeqSerializer { - type Ok = Value; - type Error = Error; + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + ser::SerializeSeq::serialize_element(self, value) + } - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - ser::SerializeSeq::serialize_element(self, value) + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } } - fn end(self) -> Result { - ser::SerializeSeq::end(self) - } -} + impl ser::SerializeTupleStruct for SeqSerializer { + type Ok = Value; + type Error = Error; -impl ser::SerializeSeq for SeqVariantSerializer<'_> { - type Ok = Value; - type Error = Error; - - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - self.items.push(Value::Union( - self.index, - Box::new(value.serialize(&mut Serializer::default())?), - )); - Ok(()) - } + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + ser::SerializeSeq::serialize_element(self, value) + } - fn end(self) -> Result { - Ok(Value::Record(vec![ - ( - "type".to_owned(), - Value::Enum(self.index, self.variant.to_owned()), - ), - ("value".to_owned(), Value::Array(self.items)), - ])) + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } } -} -impl ser::SerializeTupleVariant for SeqVariantSerializer<'_> { - type Ok = Value; - type Error = Error; + impl ser::SerializeSeq for SeqVariantSerializer<'_> { + type Ok = Value; + type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - ser::SerializeSeq::serialize_element(self, value) - } + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + self.items.push(Value::Union( + self.index, + Box::new(value.serialize(&mut Serializer::default())?), + )); + Ok(()) + } - fn end(self) -> Result { - ser::SerializeSeq::end(self) + fn end(self) -> Result { + Ok(Value::Record(vec![ + ( + "type".to_owned(), + Value::Enum(self.index, self.variant.to_owned()), + ), + ("value".to_owned(), Value::Array(self.items)), + ])) + } } -} -impl ser::SerializeMap for MapSerializer { - type Ok = Value; - type Error = Error; + impl ser::SerializeTupleVariant for SeqVariantSerializer<'_> { + type Ok = Value; + type Error = Error; - fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - let key = key.serialize(&mut Serializer::default())?; + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + ser::SerializeSeq::serialize_element(self, value) + } - if let Value::String(key) = key { - self.indices.insert(key, self.values.len()); - Ok(()) - } else { - Err(ser::Error::custom("map key is not a string")) + fn end(self) -> Result { + ser::SerializeSeq::end(self) } } - fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - self.values - .push(value.serialize(&mut Serializer::default())?); - Ok(()) - } + impl ser::SerializeMap for MapSerializer { + type Ok = Value; + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + let key = key.serialize(&mut Serializer::default())?; - fn end(self) -> Result { - let mut items = HashMap::new(); - for (key, index) in self.indices { - if let Some(value) = self.values.get(index) { - items.insert(key, value.clone()); + if let Value::String(key) = key { + self.indices.insert(key, self.values.len()); + Ok(()) + } else { + Err(ser::Error::custom("map key is not a string")) } } - Ok(Value::Map(items)) - } -} + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + self.values + .push(value.serialize(&mut Serializer::default())?); + Ok(()) + } -impl ser::SerializeStruct for StructSerializer { - type Ok = Value; - type Error = Error; - - fn serialize_field(&mut self, name: &'static str, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - self.fields.push(( - name.to_owned(), - value.serialize(&mut Serializer::default())?, - )); - Ok(()) - } + fn end(self) -> Result { + let mut items = HashMap::new(); + for (key, index) in self.indices { + if let Some(value) = self.values.get(index) { + items.insert(key, value.clone()); + } + } - fn end(self) -> Result { - Ok(Value::Record(self.fields)) + Ok(Value::Map(items)) + } } -} -impl ser::SerializeStructVariant for StructVariantSerializer<'_> { - type Ok = Value; - type Error = Error; - - fn serialize_field(&mut self, name: &'static str, value: &T) -> Result<(), Self::Error> - where - T: Serialize + ?Sized, - { - self.fields.push(( - name.to_owned(), - value.serialize(&mut Serializer::default())?, - )); - Ok(()) - } + impl ser::SerializeStruct for StructSerializer { + type Ok = Value; + type Error = Error; - fn end(self) -> Result { - Ok(Value::Record(vec![ - ( - "type".to_owned(), - Value::Enum(self.index, self.variant.to_owned()), - ), - ( - "value".to_owned(), - Value::Union(self.index, Box::new(Value::Record(self.fields))), - ), - ])) + fn serialize_field(&mut self, name: &'static str, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + self.fields.push(( + name.to_owned(), + value.serialize(&mut Serializer::default())?, + )); + Ok(()) + } + + fn end(self) -> Result { + Ok(Value::Record(self.fields)) + } } -} -/// Interpret a serializeable instance as a `Value`. -/// -/// This conversion can fail if the value is not valid as per the Avro specification. -/// e.g: HashMap with non-string keys -pub fn to_value(value: S) -> Result { - let mut serializer = Serializer::default(); - value.serialize(&mut serializer) -} + impl ser::SerializeStructVariant for StructVariantSerializer<'_> { + type Ok = Value; + type Error = Error; -#[cfg(test)] -mod tests { - use super::*; - use crate::Decimal; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; - use serde::{Deserialize, Serialize}; - use serial_test::serial; - use std::sync::atomic::Ordering; - - #[derive(Debug, Deserialize, Serialize, Clone)] - struct Test { - a: i64, - b: String, - decimal: Decimal, - } + fn serialize_field(&mut self, name: &'static str, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + self.fields.push(( + name.to_owned(), + value.serialize(&mut Serializer::default())?, + )); + Ok(()) + } - #[derive(Debug, Deserialize, Serialize)] - struct TestInner { - a: Test, - b: i32, + fn end(self) -> Result { + Ok(Value::Record(vec![ + ( + "type".to_owned(), + Value::Enum(self.index, self.variant.to_owned()), + ), + ( + "value".to_owned(), + Value::Union(self.index, Box::new(Value::Record(self.fields))), + ), + ])) + } } - #[derive(Debug, Deserialize, Serialize)] - struct TestUnitExternalEnum { - a: UnitExternalEnum, - } + /// Interpret a serializeable instance as a `Value`. + /// + /// This conversion can fail if the value is not valid as per the Avro specification. + /// e.g: HashMap with non-string keys + pub fn to_value(value: S) -> Result { + let mut serializer = Serializer::default(); + value.serialize(&mut serializer) + } + + #[cfg(test)] + mod tests { + use super::*; + use crate::Decimal; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; + use serde::{Deserialize, Serialize}; + use serial_test::serial; + use std::sync::atomic::Ordering; + + #[derive(Debug, Deserialize, Serialize, Clone)] + struct Test { + a: i64, + b: String, + decimal: Decimal, + } - #[derive(Debug, Deserialize, Serialize)] - enum UnitExternalEnum { - Val1, - Val2, - } + #[derive(Debug, Deserialize, Serialize)] + struct TestInner { + a: Test, + b: i32, + } - #[derive(Debug, Deserialize, Serialize)] - struct TestUnitInternalEnum { - a: UnitInternalEnum, - } + #[derive(Debug, Deserialize, Serialize)] + struct TestUnitExternalEnum { + a: UnitExternalEnum, + } - #[derive(Debug, Deserialize, Serialize)] - #[serde(tag = "t")] - enum UnitInternalEnum { - Val1, - Val2, - } + #[derive(Debug, Deserialize, Serialize)] + enum UnitExternalEnum { + Val1, + Val2, + } - #[derive(Debug, Deserialize, Serialize)] - struct TestUnitAdjacentEnum { - a: UnitAdjacentEnum, - } + #[derive(Debug, Deserialize, Serialize)] + struct TestUnitInternalEnum { + a: UnitInternalEnum, + } - #[derive(Debug, Deserialize, Serialize)] - #[serde(tag = "t", content = "v")] - enum UnitAdjacentEnum { - Val1, - Val2, - } + #[derive(Debug, Deserialize, Serialize)] + #[serde(tag = "t")] + enum UnitInternalEnum { + Val1, + Val2, + } - #[derive(Debug, Deserialize, Serialize)] - struct TestUnitUntaggedEnum { - a: UnitUntaggedEnum, - } + #[derive(Debug, Deserialize, Serialize)] + struct TestUnitAdjacentEnum { + a: UnitAdjacentEnum, + } - #[derive(Debug, Deserialize, Serialize)] - #[serde(untagged)] - enum UnitUntaggedEnum { - Val1, - Val2, - } + #[derive(Debug, Deserialize, Serialize)] + #[serde(tag = "t", content = "v")] + enum UnitAdjacentEnum { + Val1, + Val2, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestSingleValueExternalEnum { - a: SingleValueExternalEnum, - } + #[derive(Debug, Deserialize, Serialize)] + struct TestUnitUntaggedEnum { + a: UnitUntaggedEnum, + } - #[derive(Debug, Serialize, Deserialize)] - enum SingleValueExternalEnum { - Double(f64), - String(String), - } + #[derive(Debug, Deserialize, Serialize)] + #[serde(untagged)] + enum UnitUntaggedEnum { + Val1, + Val2, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestSingleValueInternalEnum { - a: SingleValueInternalEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestSingleValueExternalEnum { + a: SingleValueExternalEnum, + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(tag = "t")] - enum SingleValueInternalEnum { - Double(f64), - String(String), - } + #[derive(Debug, Serialize, Deserialize)] + enum SingleValueExternalEnum { + Double(f64), + String(String), + } - #[derive(Debug, Serialize, Deserialize)] - struct TestSingleValueAdjacentEnum { - a: SingleValueAdjacentEnum, - } - #[derive(Debug, Serialize, Deserialize)] - #[serde(tag = "t", content = "v")] - enum SingleValueAdjacentEnum { - Double(f64), - String(String), - } + #[derive(Debug, Serialize, Deserialize)] + struct TestSingleValueInternalEnum { + a: SingleValueInternalEnum, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestSingleValueUntaggedEnum { - a: SingleValueUntaggedEnum, - } + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "t")] + enum SingleValueInternalEnum { + Double(f64), + String(String), + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(untagged)] - enum SingleValueUntaggedEnum { - Double(f64), - String(String), - } + #[derive(Debug, Serialize, Deserialize)] + struct TestSingleValueAdjacentEnum { + a: SingleValueAdjacentEnum, + } + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "t", content = "v")] + enum SingleValueAdjacentEnum { + Double(f64), + String(String), + } - #[derive(Debug, Serialize, Deserialize)] - struct TestStructExternalEnum { - a: StructExternalEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestSingleValueUntaggedEnum { + a: SingleValueUntaggedEnum, + } - #[derive(Debug, Serialize, Deserialize)] - enum StructExternalEnum { - Val1 { x: f32, y: f32 }, - Val2 { x: f32, y: f32 }, - } + #[derive(Debug, Serialize, Deserialize)] + #[serde(untagged)] + enum SingleValueUntaggedEnum { + Double(f64), + String(String), + } - #[derive(Debug, Serialize, Deserialize)] - struct TestStructInternalEnum { - a: StructInternalEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestStructExternalEnum { + a: StructExternalEnum, + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(tag = "type")] - enum StructInternalEnum { - Val1 { x: f32, y: f32 }, - Val2 { x: f32, y: f32 }, - } + #[derive(Debug, Serialize, Deserialize)] + enum StructExternalEnum { + Val1 { x: f32, y: f32 }, + Val2 { x: f32, y: f32 }, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestStructAdjacentEnum { - a: StructAdjacentEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestStructInternalEnum { + a: StructInternalEnum, + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(tag = "t", content = "v")] - enum StructAdjacentEnum { - Val1 { x: f32, y: f32 }, - Val2 { x: f32, y: f32 }, - } + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "type")] + enum StructInternalEnum { + Val1 { x: f32, y: f32 }, + Val2 { x: f32, y: f32 }, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestStructUntaggedEnum { - a: StructUntaggedEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestStructAdjacentEnum { + a: StructAdjacentEnum, + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(untagged)] - enum StructUntaggedEnum { - Val1 { x: f32, y: f32 }, - Val2 { x: f32, y: f32, z: f32 }, - } + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "t", content = "v")] + enum StructAdjacentEnum { + Val1 { x: f32, y: f32 }, + Val2 { x: f32, y: f32 }, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestTupleExternalEnum { - a: TupleExternalEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestStructUntaggedEnum { + a: StructUntaggedEnum, + } - #[derive(Debug, Serialize, Deserialize)] - enum TupleExternalEnum { - Val1(f32, f32), - Val2(f32, f32, f32), - } + #[derive(Debug, Serialize, Deserialize)] + #[serde(untagged)] + enum StructUntaggedEnum { + Val1 { x: f32, y: f32 }, + Val2 { x: f32, y: f32, z: f32 }, + } - // Tuple Internal Enum cannot be instantiated + #[derive(Debug, Serialize, Deserialize)] + struct TestTupleExternalEnum { + a: TupleExternalEnum, + } - #[derive(Debug, Serialize, Deserialize)] - struct TestTupleAdjacentEnum { - a: TupleAdjacentEnum, - } + #[derive(Debug, Serialize, Deserialize)] + enum TupleExternalEnum { + Val1(f32, f32), + Val2(f32, f32, f32), + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(tag = "t", content = "v")] - enum TupleAdjacentEnum { - Val1(f32, f32), - Val2(f32, f32, f32), - } + // Tuple Internal Enum cannot be instantiated - #[derive(Debug, Serialize, Deserialize)] - struct TestTupleUntaggedEnum { - a: TupleUntaggedEnum, - } + #[derive(Debug, Serialize, Deserialize)] + struct TestTupleAdjacentEnum { + a: TupleAdjacentEnum, + } - #[derive(Debug, Serialize, Deserialize)] - #[serde(untagged)] - enum TupleUntaggedEnum { - Val1(f32, f32), - Val2(f32, f32, f32), - } + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "t", content = "v")] + enum TupleAdjacentEnum { + Val1(f32, f32), + Val2(f32, f32, f32), + } + + #[derive(Debug, Serialize, Deserialize)] + struct TestTupleUntaggedEnum { + a: TupleUntaggedEnum, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(untagged)] + enum TupleUntaggedEnum { + Val1(f32, f32), + Val2(f32, f32, f32), + } + + #[test] + fn test_to_value() -> TestResult { + let test = Test { + a: 27, + b: "foo".to_owned(), + decimal: Decimal::from(vec![1, 24]), + }; + let expected = Value::Record(vec![ + ("a".to_owned(), Value::Long(27)), + ("b".to_owned(), Value::String("foo".to_owned())), + ("decimal".to_owned(), Value::Bytes(vec![1, 24])), + ]); + + assert_eq!(to_value(test.clone())?, expected); + + let test_inner = TestInner { a: test, b: 35 }; + + let expected_inner = Value::Record(vec![ + ( + "a".to_owned(), + Value::Record(vec![ + ("a".to_owned(), Value::Long(27)), + ("b".to_owned(), Value::String("foo".to_owned())), + ("decimal".to_owned(), Value::Bytes(vec![1, 24])), + ]), + ), + ("b".to_owned(), Value::Int(35)), + ]); + + assert_eq!(to_value(test_inner)?, expected_inner); + + Ok(()) + } - #[test] - fn test_to_value() -> TestResult { - let test = Test { - a: 27, - b: "foo".to_owned(), - decimal: Decimal::from(vec![1, 24]), - }; - let expected = Value::Record(vec![ - ("a".to_owned(), Value::Long(27)), - ("b".to_owned(), Value::String("foo".to_owned())), - ("decimal".to_owned(), Value::Bytes(vec![1, 24])), - ]); - - assert_eq!(to_value(test.clone())?, expected); - - let test_inner = TestInner { a: test, b: 35 }; - - let expected_inner = Value::Record(vec![ - ( + #[test] + fn test_to_value_unit_enum() -> TestResult { + let test = TestUnitExternalEnum { + a: UnitExternalEnum::Val1, + }; + + let expected = Value::Record(vec![("a".to_owned(), Value::String("Val1".to_owned()))]); + + assert_eq!( + to_value(test)?, + expected, + "Error serializing unit external enum" + ); + + let test = TestUnitInternalEnum { + a: UnitInternalEnum::Val1, + }; + + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "Error serializing unit internal enum" + ); + + let test = TestUnitAdjacentEnum { + a: UnitAdjacentEnum::Val1, + }; + + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "Error serializing unit adjacent enum" + ); + + let test = TestUnitUntaggedEnum { + a: UnitUntaggedEnum::Val1, + }; + + let expected = Value::Record(vec![("a".to_owned(), Value::Null)]); + + assert_eq!( + to_value(test)?, + expected, + "Error serializing unit untagged enum" + ); + + Ok(()) + } + + #[test] + fn test_to_value_single_value_enum() -> TestResult { + let test = TestSingleValueExternalEnum { + a: SingleValueExternalEnum::Double(64.0), + }; + + let expected = Value::Record(vec![( "a".to_owned(), Value::Record(vec![ - ("a".to_owned(), Value::Long(27)), - ("b".to_owned(), Value::String("foo".to_owned())), - ("decimal".to_owned(), Value::Bytes(vec![1, 24])), + ("type".to_owned(), Value::Enum(0, "Double".to_owned())), + ( + "value".to_owned(), + Value::Union(0, Box::new(Value::Double(64.0))), + ), ]), - ), - ("b".to_owned(), Value::Int(35)), - ]); + )]); - assert_eq!(to_value(test_inner)?, expected_inner); + assert_eq!( + to_value(test)?, + expected, + "Error serializing single value external enum" + ); - Ok(()) - } + // It is not possible to serialize an internal Single Value enum... + let test = TestSingleValueInternalEnum { + a: SingleValueInternalEnum::Double(64.0), + }; - #[test] - fn test_to_value_unit_enum() -> TestResult { - let test = TestUnitExternalEnum { - a: UnitExternalEnum::Val1, - }; - - let expected = Value::Record(vec![("a".to_owned(), Value::String("Val1".to_owned()))]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing unit external enum" - ); - - let test = TestUnitInternalEnum { - a: UnitInternalEnum::Val1, - }; - - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing unit internal enum" - ); - - let test = TestUnitAdjacentEnum { - a: UnitAdjacentEnum::Val1, - }; - - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![("t".to_owned(), Value::String("Val1".to_owned()))]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing unit adjacent enum" - ); - - let test = TestUnitUntaggedEnum { - a: UnitUntaggedEnum::Val1, - }; - - let expected = Value::Record(vec![("a".to_owned(), Value::Null)]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing unit untagged enum" - ); - - Ok(()) - } + assert!(to_value(test).is_err(), "{}", true); - #[test] - fn test_to_value_single_value_enum() -> TestResult { - let test = TestSingleValueExternalEnum { - a: SingleValueExternalEnum::Double(64.0), - }; + let test = TestSingleValueAdjacentEnum { + a: SingleValueAdjacentEnum::Double(64.0), + }; - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::Enum(0, "Double".to_owned())), - ( - "value".to_owned(), - Value::Union(0, Box::new(Value::Double(64.0))), - ), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing single value external enum" - ); - - // It is not possible to serialize an internal Single Value enum... - let test = TestSingleValueInternalEnum { - a: SingleValueInternalEnum::Double(64.0), - }; - - assert!(to_value(test).is_err(), "{}", true); - - let test = TestSingleValueAdjacentEnum { - a: SingleValueAdjacentEnum::Double(64.0), - }; - - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("t".to_owned(), Value::String("Double".to_owned())), - ("v".to_owned(), Value::Double(64.0)), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing single value adjacent enum" - ); - - let test = TestSingleValueUntaggedEnum { - a: SingleValueUntaggedEnum::Double(64.0), - }; - - let expected = Value::Record(vec![("a".to_owned(), Value::Double(64.0))]); - - assert_eq!( - to_value(test)?, - expected, - "Error serializing single value untagged enum" - ); - - Ok(()) - } + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("t".to_owned(), Value::String("Double".to_owned())), + ("v".to_owned(), Value::Double(64.0)), + ]), + )]); - #[test] - fn test_to_value_struct_enum() -> TestResult { - let test = TestStructExternalEnum { - a: StructExternalEnum::Val1 { x: 1.0, y: 2.0 }, - }; - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::Enum(0, "Val1".to_owned())), - ( - "value".to_owned(), - Value::Union( - 0, - Box::new(Value::Record(vec![ + assert_eq!( + to_value(test)?, + expected, + "Error serializing single value adjacent enum" + ); + + let test = TestSingleValueUntaggedEnum { + a: SingleValueUntaggedEnum::Double(64.0), + }; + + let expected = Value::Record(vec![("a".to_owned(), Value::Double(64.0))]); + + assert_eq!( + to_value(test)?, + expected, + "Error serializing single value untagged enum" + ); + + Ok(()) + } + + #[test] + fn test_to_value_struct_enum() -> TestResult { + let test = TestStructExternalEnum { + a: StructExternalEnum::Val1 { x: 1.0, y: 2.0 }, + }; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::Enum(0, "Val1".to_owned())), + ( + "value".to_owned(), + Value::Union( + 0, + Box::new(Value::Record(vec![ + ("x".to_owned(), Value::Float(1.0)), + ("y".to_owned(), Value::Float(2.0)), + ])), + ), + ), + ]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "error serializing struct external enum" + ); + + // I don't think that this is feasible in avro + + let test = TestStructInternalEnum { + a: StructInternalEnum::Val1 { x: 1.0, y: 2.0 }, + }; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::String("Val1".to_owned())), + ("x".to_owned(), Value::Float(1.0)), + ("y".to_owned(), Value::Float(2.0)), + ]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "error serializing struct internal enum" + ); + + let test = TestStructAdjacentEnum { + a: StructAdjacentEnum::Val1 { x: 1.0, y: 2.0 }, + }; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("t".to_owned(), Value::String("Val1".to_owned())), + ( + "v".to_owned(), + Value::Record(vec![ ("x".to_owned(), Value::Float(1.0)), ("y".to_owned(), Value::Float(2.0)), - ])), + ]), ), - ), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing struct external enum" - ); - - // I don't think that this is feasible in avro - - let test = TestStructInternalEnum { - a: StructInternalEnum::Val1 { x: 1.0, y: 2.0 }, - }; - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::String("Val1".to_owned())), - ("x".to_owned(), Value::Float(1.0)), - ("y".to_owned(), Value::Float(2.0)), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing struct internal enum" - ); - - let test = TestStructAdjacentEnum { - a: StructAdjacentEnum::Val1 { x: 1.0, y: 2.0 }, - }; - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("t".to_owned(), Value::String("Val1".to_owned())), - ( - "v".to_owned(), - Value::Record(vec![ - ("x".to_owned(), Value::Float(1.0)), - ("y".to_owned(), Value::Float(2.0)), - ]), - ), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing struct adjacent enum" - ); - - let test = TestStructUntaggedEnum { - a: StructUntaggedEnum::Val1 { x: 1.0, y: 2.0 }, - }; - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("x".to_owned(), Value::Float(1.0)), - ("y".to_owned(), Value::Float(2.0)), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing struct untagged enum" - ); - - let test = TestStructUntaggedEnum { - a: StructUntaggedEnum::Val2 { - x: 1.0, - y: 2.0, - z: 3.0, - }, - }; - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("x".to_owned(), Value::Float(1.0)), - ("y".to_owned(), Value::Float(2.0)), - ("z".to_owned(), Value::Float(3.0)), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing struct untagged enum variant" - ); - - Ok(()) - } + ]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "error serializing struct adjacent enum" + ); + + let test = TestStructUntaggedEnum { + a: StructUntaggedEnum::Val1 { x: 1.0, y: 2.0 }, + }; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("x".to_owned(), Value::Float(1.0)), + ("y".to_owned(), Value::Float(2.0)), + ]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "error serializing struct untagged enum" + ); + + let test = TestStructUntaggedEnum { + a: StructUntaggedEnum::Val2 { + x: 1.0, + y: 2.0, + z: 3.0, + }, + }; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("x".to_owned(), Value::Float(1.0)), + ("y".to_owned(), Value::Float(2.0)), + ("z".to_owned(), Value::Float(3.0)), + ]), + )]); - #[test] - fn test_to_value_tuple_enum() -> TestResult { - let test = TestTupleExternalEnum { - a: TupleExternalEnum::Val2(1.0, 2.0, 3.0), - }; + assert_eq!( + to_value(test)?, + expected, + "error serializing struct untagged enum variant" + ); - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("type".to_owned(), Value::Enum(1, "Val2".to_owned())), - ( - "value".to_owned(), - Value::Array(vec![ - Value::Union(1, Box::new(Value::Float(1.0))), - Value::Union(1, Box::new(Value::Float(2.0))), - Value::Union(1, Box::new(Value::Float(3.0))), - ]), - ), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing tuple external enum" - ); - - let test = TestTupleAdjacentEnum { - a: TupleAdjacentEnum::Val1(1.0, 2.0), - }; - - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Record(vec![ - ("t".to_owned(), Value::String("Val1".to_owned())), - ( - "v".to_owned(), - Value::Array(vec![Value::Float(1.0), Value::Float(2.0)]), - ), - ]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing tuple adjacent enum" - ); - - let test = TestTupleUntaggedEnum { - a: TupleUntaggedEnum::Val1(1.0, 2.0), - }; - - let expected = Value::Record(vec![( - "a".to_owned(), - Value::Array(vec![Value::Float(1.0), Value::Float(2.0)]), - )]); - - assert_eq!( - to_value(test)?, - expected, - "error serializing tuple untagged enum" - ); - - Ok(()) - } + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] - fn avro_3747_human_readable_false() { - use serde::ser::Serializer as SerdeSerializer; + #[test] + fn test_to_value_tuple_enum() -> TestResult { + let test = TestTupleExternalEnum { + a: TupleExternalEnum::Val2(1.0, 2.0, 3.0), + }; - crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("type".to_owned(), Value::Enum(1, "Val2".to_owned())), + ( + "value".to_owned(), + Value::Array(vec![ + Value::Union(1, Box::new(Value::Float(1.0))), + Value::Union(1, Box::new(Value::Float(2.0))), + Value::Union(1, Box::new(Value::Float(3.0))), + ]), + ), + ]), + )]); - let ser = &mut Serializer {}; + assert_eq!( + to_value(test)?, + expected, + "error serializing tuple external enum" + ); - assert_eq!(ser.is_human_readable(), false); - } + let test = TestTupleAdjacentEnum { + a: TupleAdjacentEnum::Val1(1.0, 2.0), + }; - #[test] - #[serial(serde_is_human_readable)] - fn avro_3747_human_readable_true() { - use serde::ser::Serializer as SerdeSerializer; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Record(vec![ + ("t".to_owned(), Value::String("Val1".to_owned())), + ( + "v".to_owned(), + Value::Array(vec![Value::Float(1.0), Value::Float(2.0)]), + ), + ]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "error serializing tuple adjacent enum" + ); - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let test = TestTupleUntaggedEnum { + a: TupleUntaggedEnum::Val1(1.0, 2.0), + }; - let ser = &mut Serializer {}; + let expected = Value::Record(vec![( + "a".to_owned(), + Value::Array(vec![Value::Float(1.0), Value::Float(2.0)]), + )]); + + assert_eq!( + to_value(test)?, + expected, + "error serializing tuple untagged enum" + ); + + Ok(()) + } - assert!(ser.is_human_readable()); + #[test] + #[serial(serde_is_human_readable)] + fn avro_3747_human_readable_false() { + use serde::ser::Serializer as SerdeSerializer; + + crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); + + let ser = &mut Serializer {}; + + assert_eq!(ser.is_human_readable(), false); + } + + #[test] + #[serial(serde_is_human_readable)] + fn avro_3747_human_readable_true() { + use serde::ser::Serializer as SerdeSerializer; + + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + + let ser = &mut Serializer {}; + + assert!(ser.is_human_readable()); + } } } diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index e62d21d1..f26d8ed0 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -18,1364 +18,1440 @@ //! Logic for serde-compatible schema-aware serialization //! which writes directly to a `Write` stream -use crate::{ - bigdecimal::big_decimal_as_bytes, - encode::{encode_int, encode_long}, - error::{Details, Error}, - schema::{Name, NamesRef, Namespace, RecordField, RecordSchema, Schema}, -}; -use bigdecimal::BigDecimal; -use serde::{Serialize, ser}; -use std::{borrow::Cow, io::Write, str::FromStr}; - -const COLLECTION_SERIALIZER_ITEM_LIMIT: usize = 1024; -const COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY: usize = 32; -const SINGLE_VALUE_INIT_BUFFER_SIZE: usize = 128; - -/// The sequence serializer for [`SchemaAwareWriteSerializer`]. -/// [`SchemaAwareWriteSerializeSeq`] may break large arrays up into multiple blocks to avoid having -/// to obtain the length of the entire array before being able to write any data to the underlying -/// [`std::fmt::Write`] stream. (See the -/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) -/// section of the Avro spec for more info.) -pub struct SchemaAwareWriteSerializeSeq<'a, 's, W: Write> { - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - item_schema: &'s Schema, - item_buffer_size: usize, - item_buffers: Vec>, - bytes_written: usize, -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + #[tokio::test] => #[test] + ); + } +)] +mod bigdecimal { -impl<'a, 's, W: Write> SchemaAwareWriteSerializeSeq<'a, 's, W> { - fn new( + use crate::{ + bigdecimal::tokio::big_decimal_as_bytes, + encode::tokio::{encode_int, encode_long}, + error::{Details, Error}, + schema::tokio::{Name, NamesRef, Namespace, RecordField, RecordSchema, Schema}, + }; + use bigdecimal::BigDecimal; + use serde::{Serialize, ser}; + use std::{borrow::Cow, io::Write, str::FromStr}; + + const COLLECTION_SERIALIZER_ITEM_LIMIT: usize = 1024; + const COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY: usize = 32; + const SINGLE_VALUE_INIT_BUFFER_SIZE: usize = 128; + + /// The sequence serializer for [`SchemaAwareWriteSerializer`]. + /// [`SchemaAwareWriteSerializeSeq`] may break large arrays up into multiple blocks to avoid having + /// to obtain the length of the entire array before being able to write any data to the underlying + /// [`std::fmt::Write`] stream. (See the + /// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) + /// section of the Avro spec for more info.) + pub struct SchemaAwareWriteSerializeSeq<'a, 's, W: Write> { ser: &'a mut SchemaAwareWriteSerializer<'s, W>, item_schema: &'s Schema, - len: Option, - ) -> SchemaAwareWriteSerializeSeq<'a, 's, W> { - SchemaAwareWriteSerializeSeq { - ser, - item_schema, - item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, - item_buffers: Vec::with_capacity( - len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), - ), - bytes_written: 0, + item_buffer_size: usize, + item_buffers: Vec>, + bytes_written: usize, + } + + impl<'a, 's, W: Write> SchemaAwareWriteSerializeSeq<'a, 's, W> { + fn new( + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + item_schema: &'s Schema, + len: Option, + ) -> SchemaAwareWriteSerializeSeq<'a, 's, W> { + SchemaAwareWriteSerializeSeq { + ser, + item_schema, + item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, + item_buffers: Vec::with_capacity( + len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), + ), + bytes_written: 0, + } } - } - fn write_buffered_items(&mut self) -> Result<(), Error> { - if !self.item_buffers.is_empty() { - self.bytes_written += - encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; - for item in self.item_buffers.drain(..) { - self.bytes_written += self - .ser - .writer - .write(item.as_slice()) - .map_err(Details::WriteBytes)?; + fn write_buffered_items(&mut self) -> Result<(), Error> { + if !self.item_buffers.is_empty() { + self.bytes_written += + encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; + for item in self.item_buffers.drain(..) { + self.bytes_written += self + .ser + .writer + .write(item.as_slice()) + .map_err(Details::WriteBytes)?; + } } + + Ok(()) } - Ok(()) - } + fn serialize_element(&mut self, value: &T) -> Result<(), Error> { + let mut item_buffer: Vec = Vec::with_capacity(self.item_buffer_size); + let mut item_ser = SchemaAwareWriteSerializer::new( + &mut item_buffer, + self.item_schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + value.serialize(&mut item_ser)?; - fn serialize_element(&mut self, value: &T) -> Result<(), Error> { - let mut item_buffer: Vec = Vec::with_capacity(self.item_buffer_size); - let mut item_ser = SchemaAwareWriteSerializer::new( - &mut item_buffer, - self.item_schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - value.serialize(&mut item_ser)?; + self.item_buffer_size = std::cmp::max(self.item_buffer_size, item_buffer.len() + 16); - self.item_buffer_size = std::cmp::max(self.item_buffer_size, item_buffer.len() + 16); + self.item_buffers.push(item_buffer); - self.item_buffers.push(item_buffer); + if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { + self.write_buffered_items()?; + } - if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { - self.write_buffered_items()?; + Ok(()) } - Ok(()) - } - - fn end(mut self) -> Result { - self.write_buffered_items()?; - self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; + fn end(mut self) -> Result { + self.write_buffered_items()?; + self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; - Ok(self.bytes_written) + Ok(self.bytes_written) + } } -} -impl ser::SerializeSeq for SchemaAwareWriteSerializeSeq<'_, '_, W> { - type Ok = usize; - type Error = Error; + impl ser::SerializeSeq for SchemaAwareWriteSerializeSeq<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - self.serialize_element(&value) - } + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + self.serialize_element(&value) + } - fn end(self) -> Result { - self.end() + fn end(self) -> Result { + self.end() + } } -} -impl ser::SerializeTuple for SchemaAwareWriteSerializeSeq<'_, '_, W> { - type Ok = usize; - type Error = Error; + impl ser::SerializeTuple for SchemaAwareWriteSerializeSeq<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - ser::SerializeSeq::serialize_element(self, value) - } + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) + } - fn end(self) -> Result { - ser::SerializeSeq::end(self) + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } } -} - -/// The map serializer for [`SchemaAwareWriteSerializer`]. -/// [`SchemaAwareWriteSerializeMap`] may break large maps up into multiple blocks to avoid having to -/// obtain the size of the entire map before being able to write any data to the underlying -/// [`std::fmt::Write`] stream. (See the -/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) -/// section of the Avro spec for more info.) -pub struct SchemaAwareWriteSerializeMap<'a, 's, W: Write> { - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - item_schema: &'s Schema, - item_buffer_size: usize, - item_buffers: Vec>, - bytes_written: usize, -} -impl<'a, 's, W: Write> SchemaAwareWriteSerializeMap<'a, 's, W> { - fn new( + /// The map serializer for [`SchemaAwareWriteSerializer`]. + /// [`SchemaAwareWriteSerializeMap`] may break large maps up into multiple blocks to avoid having to + /// obtain the size of the entire map before being able to write any data to the underlying + /// [`std::fmt::Write`] stream. (See the + /// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) + /// section of the Avro spec for more info.) + pub struct SchemaAwareWriteSerializeMap<'a, 's, W: Write> { ser: &'a mut SchemaAwareWriteSerializer<'s, W>, item_schema: &'s Schema, - len: Option, - ) -> SchemaAwareWriteSerializeMap<'a, 's, W> { - SchemaAwareWriteSerializeMap { - ser, - item_schema, - item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, - item_buffers: Vec::with_capacity( - len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), - ), - bytes_written: 0, + item_buffer_size: usize, + item_buffers: Vec>, + bytes_written: usize, + } + + impl<'a, 's, W: Write> SchemaAwareWriteSerializeMap<'a, 's, W> { + fn new( + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + item_schema: &'s Schema, + len: Option, + ) -> SchemaAwareWriteSerializeMap<'a, 's, W> { + SchemaAwareWriteSerializeMap { + ser, + item_schema, + item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, + item_buffers: Vec::with_capacity( + len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), + ), + bytes_written: 0, + } } - } - fn write_buffered_items(&mut self) -> Result<(), Error> { - if !self.item_buffers.is_empty() { - self.bytes_written += - encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; - for item in self.item_buffers.drain(..) { - self.bytes_written += self - .ser - .writer - .write(item.as_slice()) - .map_err(Details::WriteBytes)?; + fn write_buffered_items(&mut self) -> Result<(), Error> { + if !self.item_buffers.is_empty() { + self.bytes_written += + encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; + for item in self.item_buffers.drain(..) { + self.bytes_written += self + .ser + .writer + .write(item.as_slice()) + .map_err(Details::WriteBytes)?; + } } - } - Ok(()) + Ok(()) + } } -} -impl ser::SerializeMap for SchemaAwareWriteSerializeMap<'_, '_, W> { - type Ok = usize; - type Error = Error; - - fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - let mut element_buffer: Vec = Vec::with_capacity(self.item_buffer_size); - let string_schema = Schema::String; - let mut key_ser = SchemaAwareWriteSerializer::new( - &mut element_buffer, - &string_schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - key.serialize(&mut key_ser)?; - - self.item_buffers.push(element_buffer); - - Ok(()) - } + impl ser::SerializeMap for SchemaAwareWriteSerializeMap<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - let last_index = self.item_buffers.len() - 1; - let element_buffer = &mut self.item_buffers[last_index]; - let mut val_ser = SchemaAwareWriteSerializer::new( - element_buffer, - self.item_schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - value.serialize(&mut val_ser)?; - - self.item_buffer_size = std::cmp::max(self.item_buffer_size, element_buffer.len() + 16); - - if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { - self.write_buffered_items()?; + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + let mut element_buffer: Vec = Vec::with_capacity(self.item_buffer_size); + let string_schema = Schema::String; + let mut key_ser = SchemaAwareWriteSerializer::new( + &mut element_buffer, + &string_schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + key.serialize(&mut key_ser)?; + + self.item_buffers.push(element_buffer); + + Ok(()) } - Ok(()) - } + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + let last_index = self.item_buffers.len() - 1; + let element_buffer = &mut self.item_buffers[last_index]; + let mut val_ser = SchemaAwareWriteSerializer::new( + element_buffer, + self.item_schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + value.serialize(&mut val_ser)?; - fn end(mut self) -> Result { - self.write_buffered_items()?; - self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; + self.item_buffer_size = std::cmp::max(self.item_buffer_size, element_buffer.len() + 16); - Ok(self.bytes_written) - } -} + if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { + self.write_buffered_items()?; + } -/// The struct serializer for [`SchemaAwareWriteSerializer`], which can serialize Avro records. -/// [`SchemaAwareWriteSerializeStruct`] can accept fields out of order, but doing so incurs a -/// performance penalty, since it requires [`SchemaAwareWriteSerializeStruct`] to buffer serialized -/// values in order to write them to the stream in order. -pub struct SchemaAwareWriteSerializeStruct<'a, 's, W: Write> { - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - record_schema: &'s RecordSchema, - bytes_written: usize, -} + Ok(()) + } + + fn end(mut self) -> Result { + self.write_buffered_items()?; + self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; + + Ok(self.bytes_written) + } + } -impl<'a, 's, W: Write> SchemaAwareWriteSerializeStruct<'a, 's, W> { - fn new( + /// The struct serializer for [`SchemaAwareWriteSerializer`], which can serialize Avro records. + /// [`SchemaAwareWriteSerializeStruct`] can accept fields out of order, but doing so incurs a + /// performance penalty, since it requires [`SchemaAwareWriteSerializeStruct`] to buffer serialized + /// values in order to write them to the stream in order. + pub struct SchemaAwareWriteSerializeStruct<'a, 's, W: Write> { ser: &'a mut SchemaAwareWriteSerializer<'s, W>, record_schema: &'s RecordSchema, - ) -> SchemaAwareWriteSerializeStruct<'a, 's, W> { - SchemaAwareWriteSerializeStruct { - ser, - record_schema, - bytes_written: 0, + bytes_written: usize, + } + + impl<'a, 's, W: Write> SchemaAwareWriteSerializeStruct<'a, 's, W> { + fn new( + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + record_schema: &'s RecordSchema, + ) -> SchemaAwareWriteSerializeStruct<'a, 's, W> { + SchemaAwareWriteSerializeStruct { + ser, + record_schema, + bytes_written: 0, + } } - } - fn serialize_next_field(&mut self, field: &RecordField, value: &T) -> Result<(), Error> - where - T: ?Sized + ser::Serialize, - { - // If we receive fields in order, write them directly to the main writer - let mut value_ser = SchemaAwareWriteSerializer::new( - &mut *self.ser.writer, - &field.schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - self.bytes_written += value.serialize(&mut value_ser)?; - - Ok(()) - } + fn serialize_next_field(&mut self, field: &RecordField, value: &T) -> Result<(), Error> + where + T: ?Sized + ser::Serialize, + { + // If we receive fields in order, write them directly to the main writer + let mut value_ser = SchemaAwareWriteSerializer::new( + &mut *self.ser.writer, + &field.schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + self.bytes_written += value.serialize(&mut value_ser)?; + + Ok(()) + } - fn end(self) -> Result { - Ok(self.bytes_written) + fn end(self) -> Result { + Ok(self.bytes_written) + } } -} -impl ser::SerializeStruct for SchemaAwareWriteSerializeStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; - - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - let record_field = self - .record_schema - .lookup - .get(key) - .and_then(|idx| self.record_schema.fields.get(*idx)); - - match record_field { - Some(field) => { - // self.item_count += 1; - self.serialize_next_field(field, value).map_err(|e| { - Details::SerializeRecordFieldWithSchema { - field_name: key, - record_schema: Schema::Record(self.record_schema.clone()), - error: Box::new(e), - } - .into() - }) + impl ser::SerializeStruct for SchemaAwareWriteSerializeStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + let record_field = self + .record_schema + .lookup + .get(key) + .and_then(|idx| self.record_schema.fields.get(*idx)); + + match record_field { + Some(field) => { + // self.item_count += 1; + self.serialize_next_field(field, value).map_err(|e| { + Details::SerializeRecordFieldWithSchema { + field_name: key, + record_schema: Schema::Record(self.record_schema.clone()), + error: Box::new(e), + } + .into() + }) + } + None => Err(Details::FieldName(String::from(key)).into()), } - None => Err(Details::FieldName(String::from(key)).into()), } - } - fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> { - let skipped_field = self - .record_schema - .lookup - .get(key) - .and_then(|idx| self.record_schema.fields.get(*idx)); - - if let Some(skipped_field) = skipped_field { - // self.item_count += 1; - skipped_field - .default - .serialize(&mut SchemaAwareWriteSerializer::new( - self.ser.writer, - &skipped_field.schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ))?; - } else { - return Err(Details::GetField(key.to_string()).into()); + fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> { + let skipped_field = self + .record_schema + .lookup + .get(key) + .and_then(|idx| self.record_schema.fields.get(*idx)); + + if let Some(skipped_field) = skipped_field { + // self.item_count += 1; + skipped_field + .default + .serialize(&mut SchemaAwareWriteSerializer::new( + self.ser.writer, + &skipped_field.schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ))?; + } else { + return Err(Details::GetField(key.to_string()).into()); + } + + Ok(()) } - Ok(()) + fn end(self) -> Result { + self.end() + } } - fn end(self) -> Result { - self.end() - } -} + impl ser::SerializeStructVariant for SchemaAwareWriteSerializeStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; -impl ser::SerializeStructVariant for SchemaAwareWriteSerializeStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeStruct::serialize_field(self, key, value) + } - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - ser::SerializeStruct::serialize_field(self, key, value) + fn end(self) -> Result { + ser::SerializeStruct::end(self) + } } - fn end(self) -> Result { - ser::SerializeStruct::end(self) + /// The tuple struct serializer for [`SchemaAwareWriteSerializer`]. + /// [`SchemaAwareWriteSerializeTupleStruct`] can serialize to an Avro array, record, or big-decimal. + /// When serializing to a record, fields must be provided in the correct order, since no names are provided. + pub enum SchemaAwareWriteSerializeTupleStruct<'a, 's, W: Write> { + Record(SchemaAwareWriteSerializeStruct<'a, 's, W>), + Array(SchemaAwareWriteSerializeSeq<'a, 's, W>), } -} - -/// The tuple struct serializer for [`SchemaAwareWriteSerializer`]. -/// [`SchemaAwareWriteSerializeTupleStruct`] can serialize to an Avro array, record, or big-decimal. -/// When serializing to a record, fields must be provided in the correct order, since no names are provided. -pub enum SchemaAwareWriteSerializeTupleStruct<'a, 's, W: Write> { - Record(SchemaAwareWriteSerializeStruct<'a, 's, W>), - Array(SchemaAwareWriteSerializeSeq<'a, 's, W>), -} -impl SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { - fn serialize_field(&mut self, value: &T) -> Result<(), Error> - where - T: ?Sized + ser::Serialize, - { - use SchemaAwareWriteSerializeTupleStruct::*; - match self { - Record(_record_ser) => { - unimplemented!("Tuple struct serialization to record is not supported!"); + impl SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { + fn serialize_field(&mut self, value: &T) -> Result<(), Error> + where + T: ?Sized + ser::Serialize, + { + use SchemaAwareWriteSerializeTupleStruct::*; + match self { + Record(_record_ser) => { + unimplemented!("Tuple struct serialization to record is not supported!"); + } + Array(array_ser) => array_ser.serialize_element(&value), } - Array(array_ser) => array_ser.serialize_element(&value), } - } - fn end(self) -> Result { - use SchemaAwareWriteSerializeTupleStruct::*; - match self { - Record(record_ser) => record_ser.end(), - Array(array_ser) => array_ser.end(), + fn end(self) -> Result { + use SchemaAwareWriteSerializeTupleStruct::*; + match self { + Record(record_ser) => record_ser.end(), + Array(array_ser) => array_ser.end(), + } } } -} -impl ser::SerializeTupleStruct for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; + impl ser::SerializeTupleStruct for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - self.serialize_field(&value) - } + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + self.serialize_field(&value) + } - fn end(self) -> Result { - self.end() + fn end(self) -> Result { + self.end() + } } -} -impl ser::SerializeTupleVariant for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; + impl ser::SerializeTupleVariant for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - self.serialize_field(&value) - } + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + self.serialize_field(&value) + } - fn end(self) -> Result { - self.end() + fn end(self) -> Result { + self.end() + } } -} -/// A [`serde::ser::Serializer`] implementation that serializes directly to a [`std::fmt::Write`] -/// using the provided schema. If [`SchemaAwareWriteSerializer`] isn't able to match the incoming -/// data with its schema, it will return an error. -/// A [`SchemaAwareWriteSerializer`] instance can be re-used to serialize multiple values matching -/// the schema to its [`std::fmt::Write`] stream. -pub struct SchemaAwareWriteSerializer<'s, W: Write> { - writer: &'s mut W, - root_schema: &'s Schema, - names: &'s NamesRef<'s>, - enclosing_namespace: Namespace, -} - -impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { - /// Create a new [`SchemaAwareWriteSerializer`]. - /// - /// `writer` is the [`std::fmt::Write`] stream to be written to. - /// - /// `schema` is the schema of the value to be written. - /// - /// `names` is the mapping of schema names to schemas, to be used for type reference lookups - /// - /// `enclosing_namespace` is the enclosing namespace to be used for type reference lookups - pub fn new( + /// A [`serde::ser::Serializer`] implementation that serializes directly to a [`std::fmt::Write`] + /// using the provided schema. If [`SchemaAwareWriteSerializer`] isn't able to match the incoming + /// data with its schema, it will return an error. + /// A [`SchemaAwareWriteSerializer`] instance can be re-used to serialize multiple values matching + /// the schema to its [`std::fmt::Write`] stream. + pub struct SchemaAwareWriteSerializer<'s, W: Write> { writer: &'s mut W, - schema: &'s Schema, + root_schema: &'s Schema, names: &'s NamesRef<'s>, enclosing_namespace: Namespace, - ) -> SchemaAwareWriteSerializer<'s, W> { - SchemaAwareWriteSerializer { - writer, - root_schema: schema, - names, - enclosing_namespace, - } } - fn get_ref_schema(&self, name: &'s Name) -> Result<&'s Schema, Error> { - let full_name = match name.namespace { - Some(_) => Cow::Borrowed(name), - None => Cow::Owned(Name { - name: name.name.clone(), - namespace: self.enclosing_namespace.clone(), - }), - }; + impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { + /// Create a new [`SchemaAwareWriteSerializer`]. + /// + /// `writer` is the [`std::fmt::Write`] stream to be written to. + /// + /// `schema` is the schema of the value to be written. + /// + /// `names` is the mapping of schema names to schemas, to be used for type reference lookups + /// + /// `enclosing_namespace` is the enclosing namespace to be used for type reference lookups + pub fn new( + writer: &'s mut W, + schema: &'s Schema, + names: &'s NamesRef<'s>, + enclosing_namespace: Namespace, + ) -> SchemaAwareWriteSerializer<'s, W> { + SchemaAwareWriteSerializer { + writer, + root_schema: schema, + names, + enclosing_namespace, + } + } - let ref_schema = self.names.get(full_name.as_ref()).copied(); + fn get_ref_schema(&self, name: &'s Name) -> Result<&'s Schema, Error> { + let full_name = match name.namespace { + Some(_) => Cow::Borrowed(name), + None => Cow::Owned(Name { + name: name.name.clone(), + namespace: self.enclosing_namespace.clone(), + }), + }; - ref_schema.ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) - } + let ref_schema = self.names.get(full_name.as_ref()).copied(); + + ref_schema + .ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) + } - fn write_bytes(&mut self, bytes: &[u8]) -> Result { - let mut bytes_written: usize = 0; + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let mut bytes_written: usize = 0; - bytes_written += encode_long(bytes.len() as i64, &mut self.writer)?; - bytes_written += self.writer.write(bytes).map_err(Details::WriteBytes)?; + bytes_written += encode_long(bytes.len() as i64, &mut self.writer)?; + bytes_written += self.writer.write(bytes).map_err(Details::WriteBytes)?; - Ok(bytes_written) - } + Ok(bytes_written) + } - fn serialize_bool_with_schema(&mut self, value: bool, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "bool", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_bool_with_schema( + &mut self, + value: bool, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "bool", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Boolean => self - .writer - .write(&[u8::from(value)]) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Boolean => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_bool_with_schema(value, variant_schema); + match schema { + Schema::Boolean => self + .writer + .write(&[u8::from(value)]) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Boolean => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_bool_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "No matching Schema::Bool found in {:?}", + union_schema.schemas + ))) } - Err(create_error(format!( - "No matching Schema::Bool found in {:?}", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected {expected}. Got: Bool"))), } - expected => Err(create_error(format!("Expected {expected}. Got: Bool"))), } - } - fn serialize_i32_with_schema(&mut self, value: i32, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "int (i8 | i16 | i32)", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_i32_with_schema( + &mut self, + value: i32, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "int (i8 | i16 | i32)", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => encode_int(value, &mut self.writer), - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_i32_with_schema(value, variant_schema); + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + encode_int(value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_i32_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching int-like schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Cannot find a matching int-like schema in {union_schema:?}" - ))) + expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), } - expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), } - } - fn serialize_i64_with_schema(&mut self, value: i64, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "i64", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_i64_with_schema( + &mut self, + value: i64, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "i64", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - let int_value = - i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_int(int_value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value, &mut self.writer), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_i64_with_schema(value, variant_schema); + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + let int_value = + i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_int(int_value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value, &mut self.writer), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_i64_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching int/long-like schema in {:?}", + union_schema.schemas + ))) } - Err(create_error(format!( - "Cannot find a matching int/long-like schema in {:?}", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), } - } - fn serialize_u8_with_schema(&mut self, value: u8, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "u8", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_u8_with_schema(&mut self, value: u8, schema: &Schema) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "u8", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - encode_int(value as i32, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), - Schema::Bytes => self.write_bytes(&[value]), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos - | Schema::Bytes => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_u8_with_schema(value, variant_schema); + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + encode_int(value as i32, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), + Schema::Bytes => self.write_bytes(&[value]), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos + | Schema::Bytes => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_u8_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching Int-like, Long-like or Bytes schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Cannot find a matching Int-like, Long-like or Bytes schema in {union_schema:?}" - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: Int"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: Int"))), } - } - fn serialize_u32_with_schema(&mut self, value: u32, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "unsigned int (u16 | u32)", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_u32_with_schema( + &mut self, + value: u32, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "unsigned int (u16 | u32)", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - let int_value = - i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_int(int_value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_u32_with_schema(value, variant_schema); + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + let int_value = + i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_int(int_value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_u32_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching Int-like or Long-like schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Cannot find a matching Int-like or Long-like schema in {union_schema:?}" - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), } - } - fn serialize_u64_with_schema(&mut self, value: u64, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "u64", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_u64_with_schema( + &mut self, + value: u64, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "u64", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - let int_value = - i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_int(int_value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - let long_value = - i64::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_long(long_value, &mut self.writer) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_u64_with_schema(value, variant_schema); + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + let int_value = + i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_int(int_value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + let long_value = + i64::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_long(long_value, &mut self.writer) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_u64_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching Int-like or Long-like schema in {:?}", + union_schema.schemas + ))) } - Err(create_error(format!( - "Cannot find a matching Int-like or Long-like schema in {:?}", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), } - expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), } - } - fn serialize_f32_with_schema(&mut self, value: f32, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "f32", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_f32_with_schema( + &mut self, + value: f32, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "f32", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Float => self - .writer - .write(&value.to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Double => self - .writer - .write(&(value as f64).to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Float | Schema::Double => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_f32_with_schema(value, variant_schema); + match schema { + Schema::Float => self + .writer + .write(&value.to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Double => self + .writer + .write(&(value as f64).to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Float | Schema::Double => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_f32_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a Float schema in {:?}", + union_schema.schemas + ))) } - Err(create_error(format!( - "Cannot find a Float schema in {:?}", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: Float"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: Float"))), } - } - fn serialize_f64_with_schema(&mut self, value: f64, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "f64", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_f64_with_schema( + &mut self, + value: f64, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "f64", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Float => self - .writer - .write(&(value as f32).to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Double => self - .writer - .write(&value.to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Float | Schema::Double => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_f64_with_schema(value, variant_schema); + match schema { + Schema::Float => self + .writer + .write(&(value as f32).to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Double => self + .writer + .write(&value.to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Float | Schema::Double => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_f64_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a Double schema in {:?}", + union_schema.schemas + ))) } - Err(create_error(format!( - "Cannot find a Double schema in {:?}", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: Double"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: Double"))), } - } - - fn serialize_char_with_schema(&mut self, value: char, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "char", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; - match schema { - Schema::String | Schema::Bytes => self.write_bytes(String::from(value).as_bytes()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::String | Schema::Bytes => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_char_with_schema(value, variant_schema); + fn serialize_char_with_schema( + &mut self, + value: char, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "char", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::String | Schema::Bytes => self.write_bytes(String::from(value).as_bytes()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::String | Schema::Bytes => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_char_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching String or Bytes schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Cannot find a matching String or Bytes schema in {union_schema:?}" - ))) + expected => Err(create_error(format!("Expected {expected}. Got: char"))), } - expected => Err(create_error(format!("Expected {expected}. Got: char"))), } - } - - fn serialize_str_with_schema(&mut self, value: &str, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "string", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; - match schema { - Schema::String | Schema::Bytes | Schema::Uuid => self.write_bytes(value.as_bytes()), - Schema::BigDecimal => { - // If we get a string for a `BigDecimal` type, expect a display string representation, such as "12.75" - let decimal_val = - BigDecimal::from_str(value).map_err(|e| create_error(e.to_string()))?; - let decimal_bytes = big_decimal_as_bytes(&decimal_val)?; - self.write_bytes(decimal_bytes.as_slice()) - } - Schema::Fixed(fixed_schema) => { - if value.len() == fixed_schema.size { - self.writer - .write(value.as_bytes()) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - Err(create_error(format!( - "Fixed schema size ({}) does not match the value length ({})", - fixed_schema.size, - value.len() - ))) + fn serialize_str_with_schema( + &mut self, + value: &str, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "string", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::String | Schema::Bytes | Schema::Uuid => self.write_bytes(value.as_bytes()), + Schema::BigDecimal => { + // If we get a string for a `BigDecimal` type, expect a display string representation, such as "12.75" + let decimal_val = + BigDecimal::from_str(value).map_err(|e| create_error(e.to_string()))?; + let decimal_bytes = big_decimal_as_bytes(&decimal_val)?; + self.write_bytes(decimal_bytes.as_slice()) } - } - Schema::Ref { name } => { - let ref_schema = self.get_ref_schema(name)?; - self.serialize_str_with_schema(value, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::String - | Schema::Bytes - | Schema::Uuid - | Schema::Fixed(_) - | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_str_with_schema(value, variant_schema); + Schema::Fixed(fixed_schema) => { + if value.len() == fixed_schema.size { + self.writer + .write(value.as_bytes()) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + Err(create_error(format!( + "Fixed schema size ({}) does not match the value length ({})", + fixed_schema.size, + value.len() + ))) + } + } + Schema::Ref { name } => { + let ref_schema = self.get_ref_schema(name)?; + self.serialize_str_with_schema(value, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::String + | Schema::Bytes + | Schema::Uuid + | Schema::Fixed(_) + | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_str_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Expected one of the union variants {:?}. Got: String", + union_schema.schemas + ))) } - Err(create_error(format!( - "Expected one of the union variants {:?}. Got: String", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: String"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: String"))), } - } - fn serialize_bytes_with_schema( - &mut self, - value: &[u8], - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - use std::fmt::Write; - let mut v_str = String::with_capacity(value.len()); - for b in value { - if write!(&mut v_str, "{b:x}").is_err() { - v_str.push_str("??"); + fn serialize_bytes_with_schema( + &mut self, + value: &[u8], + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + use std::fmt::Write; + let mut v_str = String::with_capacity(value.len()); + for b in value { + if write!(&mut v_str, "{b:x}").is_err() { + v_str.push_str("??"); + } } - } - Error::new(Details::SerializeValueWithSchema { - value_type: "bytes", - value: format!("{v_str}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + Error::new(Details::SerializeValueWithSchema { + value_type: "bytes", + value: format!("{v_str}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::String | Schema::Bytes | Schema::Uuid | Schema::BigDecimal => { - self.write_bytes(value) - } - Schema::Fixed(fixed_schema) => { - if value.len() == fixed_schema.size { - self.writer - .write(value) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - Err(create_error(format!( - "Fixed schema size ({}) does not match the value length ({})", - fixed_schema.size, - value.len() - ))) - } - } - Schema::Duration => { - if value.len() == 12 { - self.writer - .write(value) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - Err(create_error(format!( - "Duration length must be 12! Got ({})", - value.len() - ))) + match schema { + Schema::String | Schema::Bytes | Schema::Uuid | Schema::BigDecimal => { + self.write_bytes(value) } - } - Schema::Decimal(decimal_schema) => match decimal_schema.inner.as_ref() { - Schema::Bytes => self.write_bytes(value), - Schema::Fixed(fixed_schema) => match fixed_schema.size.checked_sub(value.len()) { - Some(pad) => { - let pad_val = match value.len() { - 0 => 0, - _ => value[0], - }; - let padding = vec![pad_val; pad]; - self.writer - .write(padding.as_slice()) - .map_err(Details::WriteBytes)?; + Schema::Fixed(fixed_schema) => { + if value.len() == fixed_schema.size { self.writer .write(value) .map_err(|e| Details::WriteBytes(e).into()) + } else { + Err(create_error(format!( + "Fixed schema size ({}) does not match the value length ({})", + fixed_schema.size, + value.len() + ))) } - None => Err(Details::CompareFixedSizes { - size: fixed_schema.size, - n: value.len(), + } + Schema::Duration => { + if value.len() == 12 { + self.writer + .write(value) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + Err(create_error(format!( + "Duration length must be 12! Got ({})", + value.len() + ))) } - .into()), + } + Schema::Decimal(decimal_schema) => match decimal_schema.inner.as_ref() { + Schema::Bytes => self.write_bytes(value), + Schema::Fixed(fixed_schema) => match fixed_schema.size.checked_sub(value.len()) + { + Some(pad) => { + let pad_val = match value.len() { + 0 => 0, + _ => value[0], + }; + let padding = vec![pad_val; pad]; + self.writer + .write(padding.as_slice()) + .map_err(Details::WriteBytes)?; + self.writer + .write(value) + .map_err(|e| Details::WriteBytes(e).into()) + } + None => Err(Details::CompareFixedSizes { + size: fixed_schema.size, + n: value.len(), + } + .into()), + }, + unsupported => Err(create_error(format!( + "Decimal schema's inner should be Bytes or Fixed schema. Got: {unsupported}" + ))), }, - unsupported => Err(create_error(format!( - "Decimal schema's inner should be Bytes or Fixed schema. Got: {unsupported}" - ))), - }, - Schema::Ref { name } => { - let ref_schema = self.get_ref_schema(name)?; - self.serialize_bytes_with_schema(value, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::String - | Schema::Bytes - | Schema::Uuid - | Schema::BigDecimal - | Schema::Fixed(_) - | Schema::Duration - | Schema::Decimal(_) - | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_bytes_with_schema(value, variant_schema); + Schema::Ref { name } => { + let ref_schema = self.get_ref_schema(name)?; + self.serialize_bytes_with_schema(value, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::String + | Schema::Bytes + | Schema::Uuid + | Schema::BigDecimal + | Schema::Fixed(_) + | Schema::Duration + | Schema::Decimal(_) + | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_bytes_with_schema(value, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal or Ref schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Cannot find a matching String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal or Ref schema in {union_schema:?}" - ))) + unsupported => Err(create_error(format!( + "Expected String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal, Ref or Union schema. Got: {unsupported}" + ))), } - unsupported => Err(create_error(format!( - "Expected String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal, Ref or Union schema. Got: {unsupported}" - ))), } - } - fn serialize_none_with_schema(&mut self, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "none", - value: format!("None. Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Null => Ok(0), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Null => { - return encode_int(i as i32, &mut *self.writer); + fn serialize_none_with_schema(&mut self, schema: &Schema) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "none", + value: format!("None. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Null => Ok(0), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Null => { + return encode_int(i as i32, &mut *self.writer); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching Null schema in {:?}", + union_schema.schemas + ))) } - Err(create_error(format!( - "Cannot find a matching Null schema in {:?}", - union_schema.schemas - ))) + expected => Err(create_error(format!("Expected: {expected}. Got: Null"))), } - expected => Err(create_error(format!("Expected: {expected}. Got: Null"))), } - } - - fn serialize_some_with_schema(&mut self, value: &T, schema: &Schema) -> Result - where - T: ?Sized + ser::Serialize, - { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "some", - value: format!("Some(?). Cause: {cause}"), - schema: schema.clone(), - }) - }; - match schema { - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Null => { /* skip */ } - _ => { - encode_long(i as i64, &mut *self.writer)?; - let mut variant_ser = SchemaAwareWriteSerializer::new( - &mut *self.writer, - variant_schema, - self.names, - self.enclosing_namespace.clone(), - ); - return value.serialize(&mut variant_ser); - } - } - } - Err(create_error(format!( - "Cannot find a matching Null schema in {:?}", - union_schema.schemas - ))) + fn serialize_some_with_schema( + &mut self, + value: &T, + schema: &Schema, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "some", + value: format!("Some(?). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Null => { /* skip */ } + _ => { + encode_long(i as i64, &mut *self.writer)?; + let mut variant_ser = SchemaAwareWriteSerializer::new( + &mut *self.writer, + variant_schema, + self.names, + self.enclosing_namespace.clone(), + ); + return value.serialize(&mut variant_ser); + } + } + } + Err(create_error(format!( + "Cannot find a matching Null schema in {:?}", + union_schema.schemas + ))) + } + _ => value.serialize(self), } - _ => value.serialize(self), } - } - fn serialize_unit_struct_with_schema( - &mut self, - name: &'static str, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "unit struct", - value: format!("{name}. Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Record(sch) => match sch.fields.len() { - 0 => Ok(0), - too_many => Err(create_error(format!( - "Too many fields: {too_many}. Expected: 0" - ))), - }, - Schema::Null => Ok(0), - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_unit_struct_with_schema(name, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Record(record_schema) if record_schema.fields.is_empty() => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_unit_struct_with_schema(name, variant_schema); - } - Schema::Null | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_unit_struct_with_schema(name, variant_schema); + fn serialize_unit_struct_with_schema( + &mut self, + name: &'static str, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "unit struct", + value: format!("{name}. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Record(sch) => match sch.fields.len() { + 0 => Ok(0), + too_many => Err(create_error(format!( + "Too many fields: {too_many}. Expected: 0" + ))), + }, + Schema::Null => Ok(0), + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_unit_struct_with_schema(name, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Record(record_schema) if record_schema.fields.is_empty() => { + encode_int(i as i32, &mut *self.writer)?; + return self + .serialize_unit_struct_with_schema(name, variant_schema); + } + Schema::Null | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self + .serialize_unit_struct_with_schema(name, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Cannot find a matching Null schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Cannot find a matching Null schema in {union_schema:?}" - ))) + unsupported => Err(create_error(format!( + "Expected Null or Union schema. Got: {unsupported}" + ))), } - unsupported => Err(create_error(format!( - "Expected Null or Union schema. Got: {unsupported}" - ))), } - } - fn serialize_unit_variant_with_schema( - &mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "unit variant", - value: format!("{name}::{variant} (index={variant_index}). Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_unit_variant_with_schema( + &mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "unit variant", + value: format!("{name}::{variant} (index={variant_index}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Enum(enum_schema) => { + if variant_index as usize >= enum_schema.symbols.len() { + return Err(create_error(format!( + "Variant index out of bounds: {}. The Enum schema has '{}' symbols", + variant_index, + enum_schema.symbols.len() + ))); + } - match schema { - Schema::Enum(enum_schema) => { - if variant_index as usize >= enum_schema.symbols.len() { - return Err(create_error(format!( - "Variant index out of bounds: {}. The Enum schema has '{}' symbols", - variant_index, - enum_schema.symbols.len() - ))); + encode_int(variant_index as i32, &mut self.writer) } + Schema::Union(union_schema) => { + if variant_index as usize >= union_schema.schemas.len() { + return Err(create_error(format!( + "Variant index out of bounds: {}. The union schema has '{}' schemas", + variant_index, + union_schema.schemas.len() + ))); + } - encode_int(variant_index as i32, &mut self.writer) - } - Schema::Union(union_schema) => { - if variant_index as usize >= union_schema.schemas.len() { - return Err(create_error(format!( - "Variant index out of bounds: {}. The union schema has '{}' schemas", + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_unit_struct_with_schema( + name, + &union_schema.schemas[variant_index as usize], + ) + } + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_unit_variant_with_schema( + name, variant_index, - union_schema.schemas.len() - ))); + variant, + ref_schema, + ) } - - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_unit_struct_with_schema( - name, - &union_schema.schemas[variant_index as usize], - ) - } - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_unit_variant_with_schema(name, variant_index, variant, ref_schema) + unsupported => Err(create_error(format!( + "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref" + ))), } - unsupported => Err(create_error(format!( - "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref" - ))), } - } - fn serialize_newtype_struct_with_schema( - &mut self, - _name: &'static str, - value: &T, - schema: &Schema, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - let mut inner_ser = SchemaAwareWriteSerializer::new( - &mut *self.writer, - schema, - self.names, - self.enclosing_namespace.clone(), - ); - // Treat any newtype struct as a transparent wrapper around the contained type - value.serialize(&mut inner_ser) - } - - fn serialize_newtype_variant_with_schema( - &mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - value: &T, - schema: &Schema, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "newtype variant", - value: format!("{name}::{variant}(?) (index={variant_index}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Union(union_schema) => { - let variant_schema = union_schema - .schemas - .get(variant_index as usize) - .ok_or_else(|| { - create_error(format!( - "No variant schema at position {variant_index} for {union_schema:?}" - )) - })?; + fn serialize_newtype_struct_with_schema( + &mut self, + _name: &'static str, + value: &T, + schema: &Schema, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let mut inner_ser = SchemaAwareWriteSerializer::new( + &mut *self.writer, + schema, + self.names, + self.enclosing_namespace.clone(), + ); + // Treat any newtype struct as a transparent wrapper around the contained type + value.serialize(&mut inner_ser) + } - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_newtype_struct_with_schema(variant, value, variant_schema) + fn serialize_newtype_variant_with_schema( + &mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + schema: &Schema, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "newtype variant", + value: format!("{name}::{variant}(?) (index={variant_index}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Union(union_schema) => { + let variant_schema = union_schema + .schemas + .get(variant_index as usize) + .ok_or_else(|| { + create_error(format!( + "No variant schema at position {variant_index} for {union_schema:?}" + )) + })?; + + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_newtype_struct_with_schema(variant, value, variant_schema) + } + _ => Err(create_error(format!( + "Expected Union schema. Got: {schema}" + ))), } - _ => Err(create_error(format!( - "Expected Union schema. Got: {schema}" - ))), } - } - - fn serialize_seq_with_schema<'a>( - &'a mut self, - len: Option, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - let len_str = len - .map(|l| format!("{l}")) - .unwrap_or_else(|| String::from("?")); - - Error::new(Details::SerializeValueWithSchema { - value_type: "sequence", - value: format!("sequence (len={len_str}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - match schema { - Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( - self, - array_schema.items.as_ref(), - len, - )), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Array(_) => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_seq_with_schema(len, variant_schema); + fn serialize_seq_with_schema<'a>( + &'a mut self, + len: Option, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + let len_str = len + .map(|l| format!("{l}")) + .unwrap_or_else(|| String::from("?")); + + Error::new(Details::SerializeValueWithSchema { + value_type: "sequence", + value: format!("sequence (len={len_str}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( + self, + array_schema.items.as_ref(), + len, + )), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Array(_) => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_seq_with_schema(len, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Expected Array schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Expected Array schema in {union_schema:?}" - ))) + _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), } - _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), } - } - fn serialize_tuple_with_schema<'a>( - &'a mut self, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "tuple", - value: format!("tuple (len={len}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( - self, - array_schema.items.as_ref(), - Some(len), - )), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Array(_) => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_tuple_with_schema(len, variant_schema); + fn serialize_tuple_with_schema<'a>( + &'a mut self, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "tuple", + value: format!("tuple (len={len}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( + self, + array_schema.items.as_ref(), + Some(len), + )), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Array(_) => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_tuple_with_schema(len, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Expected Array schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Expected Array schema in {union_schema:?}" - ))) + _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), } - _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), } - } - - fn serialize_tuple_struct_with_schema<'a>( - &'a mut self, - name: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "tuple struct", - value: format!( - "{name}({}). Cause: {cause}", - vec!["?"; len].as_slice().join(",") - ), - schema: schema.clone(), - }) - }; - match schema { - Schema::Array(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Array( - SchemaAwareWriteSerializeSeq::new(self, &sch.items, Some(len)), - )), - Schema::Record(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Record( - SchemaAwareWriteSerializeStruct::new(self, sch), - )), - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_tuple_struct_with_schema(name, len, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Record(inner) => { - if inner.fields.len() == len { + fn serialize_tuple_struct_with_schema<'a>( + &'a mut self, + name: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "tuple struct", + value: format!( + "{name}({}). Cause: {cause}", + vec!["?"; len].as_slice().join(",") + ), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Array(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Array( + SchemaAwareWriteSerializeSeq::new(self, &sch.items, Some(len)), + )), + Schema::Record(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Record( + SchemaAwareWriteSerializeStruct::new(self, sch), + )), + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_tuple_struct_with_schema(name, len, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Record(inner) => { + if inner.fields.len() == len { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_tuple_struct_with_schema( + name, + len, + variant_schema, + ); + } + } + Schema::Array(_) | Schema::Ref { name: _ } => { encode_int(i as i32, &mut *self.writer)?; return self.serialize_tuple_struct_with_schema( name, @@ -1383,50 +1459,41 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { variant_schema, ); } + _ => { /* skip */ } } - Schema::Array(_) | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_tuple_struct_with_schema( - name, - len, - variant_schema, - ); - } - _ => { /* skip */ } } + Err(create_error(format!( + "Expected Record, Array or Ref schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Expected Record, Array or Ref schema in {union_schema:?}" - ))) + _ => Err(create_error(format!( + "Expected Record, Array, Ref or Union schema. Got: {schema}" + ))), } - _ => Err(create_error(format!( - "Expected Record, Array, Ref or Union schema. Got: {schema}" - ))), } - } - fn serialize_tuple_variant_with_schema<'a>( - &'a mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "tuple variant", - value: format!( - "{name}::{variant}({}) (index={variant_index}). Cause: {cause}", - vec!["?"; len].as_slice().join(",") - ), - schema: schema.clone(), - }) - }; + fn serialize_tuple_variant_with_schema<'a>( + &'a mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "tuple variant", + value: format!( + "{name}::{variant}({}) (index={variant_index}). Cause: {cause}", + vec!["?"; len].as_slice().join(",") + ), + schema: schema.clone(), + }) + }; - match schema { - Schema::Union(union_schema) => { - let variant_schema = union_schema + match schema { + Schema::Union(union_schema) => { + let variant_schema = union_schema .schemas .get(variant_index as usize) .ok_or_else(|| { @@ -1435,125 +1502,133 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { )) })?; - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_tuple_struct_with_schema(variant, len, variant_schema) + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_tuple_struct_with_schema(variant, len, variant_schema) + } + _ => Err(create_error(format!( + "Expected Union schema. Got: {schema}" + ))), } - _ => Err(create_error(format!( - "Expected Union schema. Got: {schema}" - ))), } - } - fn serialize_map_with_schema<'a>( - &'a mut self, - len: Option, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - let len_str = len - .map(|l| format!("{l}")) - .unwrap_or_else(|| String::from("?")); - - Error::new(Details::SerializeValueWithSchema { - value_type: "map", - value: format!("map (size={len_str}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Map(map_schema) => Ok(SchemaAwareWriteSerializeMap::new( - self, - map_schema.types.as_ref(), - len, - )), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Map(_) => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_map_with_schema(len, variant_schema); + fn serialize_map_with_schema<'a>( + &'a mut self, + len: Option, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + let len_str = len + .map(|l| format!("{l}")) + .unwrap_or_else(|| String::from("?")); + + Error::new(Details::SerializeValueWithSchema { + value_type: "map", + value: format!("map (size={len_str}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Map(map_schema) => Ok(SchemaAwareWriteSerializeMap::new( + self, + map_schema.types.as_ref(), + len, + )), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Map(_) => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_map_with_schema(len, variant_schema); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Expected a Map schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Expected a Map schema in {union_schema:?}" - ))) + _ => Err(create_error(format!( + "Expected Map or Union schema. Got: {schema}" + ))), } - _ => Err(create_error(format!( - "Expected Map or Union schema. Got: {schema}" - ))), } - } - fn serialize_struct_with_schema<'a>( - &'a mut self, - name: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "struct", - value: format!("{name}{{ ... }}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_struct_with_schema<'a>( + &'a mut self, + name: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "struct", + value: format!("{name}{{ ... }}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Record(record_schema) => { - Ok(SchemaAwareWriteSerializeStruct::new(self, record_schema)) - } - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_struct_with_schema(name, len, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Record(inner) - if inner.fields.len() == len && inner.name.name == name => - { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_struct_with_schema(name, len, variant_schema); - } - Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_struct_with_schema(name, len, variant_schema); + match schema { + Schema::Record(record_schema) => { + Ok(SchemaAwareWriteSerializeStruct::new(self, record_schema)) + } + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_struct_with_schema(name, len, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Record(inner) + if inner.fields.len() == len && inner.name.name == name => + { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_struct_with_schema( + name, + len, + variant_schema, + ); + } + Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_struct_with_schema( + name, + len, + variant_schema, + ); + } + _ => { /* skip */ } } - _ => { /* skip */ } } + Err(create_error(format!( + "Expected Record or Ref schema in {union_schema:?}" + ))) } - Err(create_error(format!( - "Expected Record or Ref schema in {union_schema:?}" - ))) + _ => Err(create_error(format!( + "Expected Record, Ref or Union schema. Got: {schema}" + ))), } - _ => Err(create_error(format!( - "Expected Record, Ref or Union schema. Got: {schema}" - ))), } - } - fn serialize_struct_variant_with_schema<'a>( - &'a mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "struct variant", - value: format!("{name}::{variant}{{ ... }} (size={len}. Cause: {cause})"), - schema: schema.clone(), - }) - }; + fn serialize_struct_variant_with_schema<'a>( + &'a mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "struct variant", + value: format!("{name}::{variant}{{ ... }} (size={len}. Cause: {cause})"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Union(union_schema) => { - let variant_schema = union_schema + match schema { + Schema::Union(union_schema) => { + let variant_schema = union_schema .schemas .get(variant_index as usize) .ok_or_else(|| { @@ -1562,388 +1637,397 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { )) })?; - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_struct_with_schema(variant, len, variant_schema) + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_struct_with_schema(variant, len, variant_schema) + } + _ => Err(create_error(format!( + "Expected Union schema. Got: {schema}" + ))), } - _ => Err(create_error(format!( - "Expected Union schema. Got: {schema}" - ))), } } -} -impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s, W> { - type Ok = usize; - type Error = Error; - type SerializeSeq = SchemaAwareWriteSerializeSeq<'a, 's, W>; - type SerializeTuple = SchemaAwareWriteSerializeSeq<'a, 's, W>; - type SerializeTupleStruct = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; - type SerializeTupleVariant = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; - type SerializeMap = SchemaAwareWriteSerializeMap<'a, 's, W>; - type SerializeStruct = SchemaAwareWriteSerializeStruct<'a, 's, W>; - type SerializeStructVariant = SchemaAwareWriteSerializeStruct<'a, 's, W>; - - fn serialize_bool(self, v: bool) -> Result { - self.serialize_bool_with_schema(v, self.root_schema) - } + impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s, W> { + type Ok = usize; + type Error = Error; + type SerializeSeq = SchemaAwareWriteSerializeSeq<'a, 's, W>; + type SerializeTuple = SchemaAwareWriteSerializeSeq<'a, 's, W>; + type SerializeTupleStruct = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; + type SerializeTupleVariant = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; + type SerializeMap = SchemaAwareWriteSerializeMap<'a, 's, W>; + type SerializeStruct = SchemaAwareWriteSerializeStruct<'a, 's, W>; + type SerializeStructVariant = SchemaAwareWriteSerializeStruct<'a, 's, W>; - fn serialize_i8(self, v: i8) -> Result { - self.serialize_i32(v as i32) - } + fn serialize_bool(self, v: bool) -> Result { + self.serialize_bool_with_schema(v, self.root_schema) + } - fn serialize_i16(self, v: i16) -> Result { - self.serialize_i32(v as i32) - } + fn serialize_i8(self, v: i8) -> Result { + self.serialize_i32(v as i32) + } - fn serialize_i32(self, v: i32) -> Result { - self.serialize_i32_with_schema(v, self.root_schema) - } + fn serialize_i16(self, v: i16) -> Result { + self.serialize_i32(v as i32) + } - fn serialize_i64(self, v: i64) -> Result { - self.serialize_i64_with_schema(v, self.root_schema) - } + fn serialize_i32(self, v: i32) -> Result { + self.serialize_i32_with_schema(v, self.root_schema) + } - fn serialize_u8(self, v: u8) -> Result { - self.serialize_u8_with_schema(v, self.root_schema) - } + fn serialize_i64(self, v: i64) -> Result { + self.serialize_i64_with_schema(v, self.root_schema) + } - fn serialize_u16(self, v: u16) -> Result { - self.serialize_u32(v as u32) - } + fn serialize_u8(self, v: u8) -> Result { + self.serialize_u8_with_schema(v, self.root_schema) + } - fn serialize_u32(self, v: u32) -> Result { - self.serialize_u32_with_schema(v, self.root_schema) - } + fn serialize_u16(self, v: u16) -> Result { + self.serialize_u32(v as u32) + } - fn serialize_u64(self, v: u64) -> Result { - self.serialize_u64_with_schema(v, self.root_schema) - } + fn serialize_u32(self, v: u32) -> Result { + self.serialize_u32_with_schema(v, self.root_schema) + } - fn serialize_f32(self, v: f32) -> Result { - self.serialize_f32_with_schema(v, self.root_schema) - } + fn serialize_u64(self, v: u64) -> Result { + self.serialize_u64_with_schema(v, self.root_schema) + } - fn serialize_f64(self, v: f64) -> Result { - self.serialize_f64_with_schema(v, self.root_schema) - } + fn serialize_f32(self, v: f32) -> Result { + self.serialize_f32_with_schema(v, self.root_schema) + } - fn serialize_char(self, v: char) -> Result { - self.serialize_char_with_schema(v, self.root_schema) - } + fn serialize_f64(self, v: f64) -> Result { + self.serialize_f64_with_schema(v, self.root_schema) + } - fn serialize_str(self, v: &str) -> Result { - self.serialize_str_with_schema(v, self.root_schema) - } + fn serialize_char(self, v: char) -> Result { + self.serialize_char_with_schema(v, self.root_schema) + } - fn serialize_bytes(self, v: &[u8]) -> Result { - self.serialize_bytes_with_schema(v, self.root_schema) - } + fn serialize_str(self, v: &str) -> Result { + self.serialize_str_with_schema(v, self.root_schema) + } - fn serialize_none(self) -> Result { - self.serialize_none_with_schema(self.root_schema) - } + fn serialize_bytes(self, v: &[u8]) -> Result { + self.serialize_bytes_with_schema(v, self.root_schema) + } - fn serialize_some(self, value: &T) -> Result - where - T: ?Sized + ser::Serialize, - { - self.serialize_some_with_schema(value, self.root_schema) - } + fn serialize_none(self) -> Result { + self.serialize_none_with_schema(self.root_schema) + } - fn serialize_unit(self) -> Result { - self.serialize_none() - } + fn serialize_some(self, value: &T) -> Result + where + T: ?Sized + ser::Serialize, + { + self.serialize_some_with_schema(value, self.root_schema) + } - fn serialize_unit_struct(self, name: &'static str) -> Result { - self.serialize_unit_struct_with_schema(name, self.root_schema) - } + fn serialize_unit(self) -> Result { + self.serialize_none() + } - fn serialize_unit_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - ) -> Result { - self.serialize_unit_variant_with_schema(name, variant_index, variant, self.root_schema) - } + fn serialize_unit_struct(self, name: &'static str) -> Result { + self.serialize_unit_struct_with_schema(name, self.root_schema) + } - fn serialize_newtype_struct( - self, - name: &'static str, - value: &T, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - self.serialize_newtype_struct_with_schema(name, value, self.root_schema) - } + fn serialize_unit_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + ) -> Result { + self.serialize_unit_variant_with_schema(name, variant_index, variant, self.root_schema) + } - fn serialize_newtype_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - value: &T, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - self.serialize_newtype_variant_with_schema( - name, - variant_index, - variant, - value, - self.root_schema, - ) - } + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + self.serialize_newtype_struct_with_schema(name, value, self.root_schema) + } - fn serialize_seq(self, len: Option) -> Result { - self.serialize_seq_with_schema(len, self.root_schema) - } + fn serialize_newtype_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + self.serialize_newtype_variant_with_schema( + name, + variant_index, + variant, + value, + self.root_schema, + ) + } - fn serialize_tuple(self, len: usize) -> Result { - self.serialize_tuple_with_schema(len, self.root_schema) - } + fn serialize_seq(self, len: Option) -> Result { + self.serialize_seq_with_schema(len, self.root_schema) + } - fn serialize_tuple_struct( - self, - name: &'static str, - len: usize, - ) -> Result { - self.serialize_tuple_struct_with_schema(name, len, self.root_schema) - } + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_tuple_with_schema(len, self.root_schema) + } - fn serialize_tuple_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - ) -> Result { - self.serialize_tuple_variant_with_schema( - name, - variant_index, - variant, - len, - self.root_schema, - ) - } + fn serialize_tuple_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + self.serialize_tuple_struct_with_schema(name, len, self.root_schema) + } - fn serialize_map(self, len: Option) -> Result { - self.serialize_map_with_schema(len, self.root_schema) - } + fn serialize_tuple_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.serialize_tuple_variant_with_schema( + name, + variant_index, + variant, + len, + self.root_schema, + ) + } - fn serialize_struct( - self, - name: &'static str, - len: usize, - ) -> Result { - self.serialize_struct_with_schema(name, len, self.root_schema) - } + fn serialize_map(self, len: Option) -> Result { + self.serialize_map_with_schema(len, self.root_schema) + } - fn serialize_struct_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - ) -> Result { - self.serialize_struct_variant_with_schema( - name, - variant_index, - variant, - len, - self.root_schema, - ) - } + fn serialize_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + self.serialize_struct_with_schema(name, len, self.root_schema) + } - fn is_human_readable(&self) -> bool { - crate::util::is_human_readable() + fn serialize_struct_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.serialize_struct_variant_with_schema( + name, + variant_index, + variant, + len, + self.root_schema, + ) + } + + fn is_human_readable(&self) -> bool { + crate::util::is_human_readable() + } } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - Days, Duration, Millis, Months, decimal::Decimal, error::Details, schema::ResolvedSchema, - }; - use apache_avro_test_helper::TestResult; - use bigdecimal::BigDecimal; - use num_bigint::{BigInt, Sign}; - use serde::Serialize; - use serde_bytes::{ByteArray, Bytes}; - use serial_test::serial; - use std::{ - collections::{BTreeMap, HashMap}, - marker::PhantomData, - sync::atomic::Ordering, - }; - use uuid::Uuid; + #[cfg(test)] + mod tests { + use super::*; + use crate::{ + Days, Duration, Millis, Months, decimal::Decimal, error::Details, + schema::ResolvedSchema, + }; + use apache_avro_test_helper::TestResult; + use bigdecimal::BigDecimal; + use num_bigint::{BigInt, Sign}; + use serde::Serialize; + use serde_bytes::{ByteArray, Bytes}; + use serial_test::serial; + use std::{ + collections::{BTreeMap, HashMap}, + marker::PhantomData, + sync::atomic::Ordering, + }; + use uuid::Uuid; - #[test] - fn test_serialize_null() -> TestResult { - let schema = Schema::Null; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + #[test] + fn test_serialize_null() -> TestResult { + let schema = Schema::Null; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - ().serialize(&mut serializer)?; - None::<()>.serialize(&mut serializer)?; - None::.serialize(&mut serializer)?; - None::.serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + ().serialize(&mut serializer)?; + None::<()>.serialize(&mut serializer)?; + None::.serialize(&mut serializer)?; + None::.serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - assert_eq!(buffer.as_slice(), Vec::::new().as_slice()); + assert_eq!(buffer.as_slice(), Vec::::new().as_slice()); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_bool() -> TestResult { - let schema = Schema::Boolean; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + #[test] + fn test_serialize_bool() -> TestResult { + let schema = Schema::Boolean; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - true.serialize(&mut serializer)?; - false.serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + true.serialize(&mut serializer)?; + false.serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - assert_eq!(buffer.as_slice(), &[1, 0]); + assert_eq!(buffer.as_slice(), &[1, 0]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_int() -> TestResult { - let schema = Schema::Int; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 4u8.serialize(&mut serializer)?; - 31u16.serialize(&mut serializer)?; - 13u32.serialize(&mut serializer)?; - 7i8.serialize(&mut serializer)?; - (-57i16).serialize(&mut serializer)?; - 129i32.serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); - - assert_eq!(buffer.as_slice(), &[8, 62, 26, 14, 113, 130, 2]); - - Ok(()) - } + #[test] + fn test_serialize_int() -> TestResult { + let schema = Schema::Int; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - #[test] - fn test_serialize_long() -> TestResult { - let schema = Schema::Long; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 4u8.serialize(&mut serializer)?; - 31u16.serialize(&mut serializer)?; - 13u32.serialize(&mut serializer)?; - 291u64.serialize(&mut serializer)?; - 7i8.serialize(&mut serializer)?; - (-57i16).serialize(&mut serializer)?; - 129i32.serialize(&mut serializer)?; - (-432i64).serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); - - assert_eq!( - buffer.as_slice(), - &[8, 62, 26, 198, 4, 14, 113, 130, 2, 223, 6] - ); - - Ok(()) - } + 4u8.serialize(&mut serializer)?; + 31u16.serialize(&mut serializer)?; + 13u32.serialize(&mut serializer)?; + 7i8.serialize(&mut serializer)?; + (-57i16).serialize(&mut serializer)?; + 129i32.serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - #[test] - fn test_serialize_float() -> TestResult { - let schema = Schema::Float; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!(buffer.as_slice(), &[8, 62, 26, 14, 113, 130, 2]); - 4.7f32.serialize(&mut serializer)?; - (-14.1f64).serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + Ok(()) + } - assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); + #[test] + fn test_serialize_long() -> TestResult { + let schema = Schema::Long; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - Ok(()) - } + 4u8.serialize(&mut serializer)?; + 31u16.serialize(&mut serializer)?; + 13u32.serialize(&mut serializer)?; + 291u64.serialize(&mut serializer)?; + 7i8.serialize(&mut serializer)?; + (-57i16).serialize(&mut serializer)?; + 129i32.serialize(&mut serializer)?; + (-432i64).serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - #[test] - fn test_serialize_double() -> TestResult { - let schema = Schema::Float; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!( + buffer.as_slice(), + &[8, 62, 26, 198, 4, 14, 113, 130, 2, 223, 6] + ); - 4.7f32.serialize(&mut serializer)?; - (-14.1f64).serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + Ok(()) + } - assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); + #[test] + fn test_serialize_float() -> TestResult { + let schema = Schema::Float; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - Ok(()) - } + 4.7f32.serialize(&mut serializer)?; + (-14.1f64).serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - #[test] - fn test_serialize_bytes() -> TestResult { - let schema = Schema::Bytes; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 'a'.serialize(&mut serializer)?; - "test".serialize(&mut serializer)?; - Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; - assert!(().serialize(&mut serializer).is_err()); - assert!(PhantomData::.serialize(&mut serializer).is_err()); - - assert_eq!( - buffer.as_slice(), - &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] - ); - - Ok(()) - } + assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); - #[test] - fn test_serialize_string() -> TestResult { - let schema = Schema::String; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 'a'.serialize(&mut serializer)?; - "test".serialize(&mut serializer)?; - Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; - assert!(().serialize(&mut serializer).is_err()); - assert!(PhantomData::.serialize(&mut serializer).is_err()); - - assert_eq!( - buffer.as_slice(), - &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] - ); - - Ok(()) - } + Ok(()) + } + + #[test] + fn test_serialize_double() -> TestResult { + let schema = Schema::Float; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 4.7f32.serialize(&mut serializer)?; + (-14.1f64).serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); + + assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); + + Ok(()) + } + + #[test] + fn test_serialize_bytes() -> TestResult { + let schema = Schema::Bytes; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - #[test] - fn test_serialize_record() -> TestResult { - let schema = Schema::parse_str( - r#"{ + 'a'.serialize(&mut serializer)?; + "test".serialize(&mut serializer)?; + Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; + assert!(().serialize(&mut serializer).is_err()); + assert!(PhantomData::.serialize(&mut serializer).is_err()); + + assert_eq!( + buffer.as_slice(), + &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] + ); + + Ok(()) + } + + #[test] + fn test_serialize_string() -> TestResult { + let schema = Schema::String; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 'a'.serialize(&mut serializer)?; + "test".serialize(&mut serializer)?; + Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; + assert!(().serialize(&mut serializer).is_err()); + assert!(PhantomData::.serialize(&mut serializer).is_err()); + + assert_eq!( + buffer.as_slice(), + &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] + ); + + Ok(()) + } + + #[test] + fn test_serialize_record() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "record", "name": "TestRecord", "fields": [ @@ -1951,438 +2035,447 @@ mod tests { {"name": "intField", "type": "int"} ] }"#, - )?; + )?; - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct GoodTestRecord { - string_field: String, - int_field: i32, - } + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct GoodTestRecord { + string_field: String, + int_field: i32, + } - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct BadTestRecord { - foo_string_field: String, - bar_int_field: i32, - } + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct BadTestRecord { + foo_string_field: String, + bar_int_field: i32, + } - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let good_record = GoodTestRecord { - string_field: String::from("test"), - int_field: 10, - }; - good_record.serialize(&mut serializer)?; + let good_record = GoodTestRecord { + string_field: String::from("test"), + int_field: 10, + }; + good_record.serialize(&mut serializer)?; - let bad_record = BadTestRecord { - foo_string_field: String::from("test"), - bar_int_field: 10, - }; - assert!(bad_record.serialize(&mut serializer).is_err()); + let bad_record = BadTestRecord { + foo_string_field: String::from("test"), + bar_int_field: 10, + }; + assert!(bad_record.serialize(&mut serializer).is_err()); - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - assert_eq!(buffer.as_slice(), &[8, b't', b'e', b's', b't', 20]); + assert_eq!(buffer.as_slice(), &[8, b't', b'e', b's', b't', 20]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_empty_record() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_empty_record() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "record", "name": "EmptyRecord", "fields": [] }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - #[derive(Serialize)] - struct EmptyRecord; - EmptyRecord.serialize(&mut serializer)?; + #[derive(Serialize)] + struct EmptyRecord; + EmptyRecord.serialize(&mut serializer)?; - #[derive(Serialize)] - struct NonEmptyRecord { - foo: String, - } - let record = NonEmptyRecord { - foo: "bar".to_string(), - }; - match record - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::FieldName(field_name)) if field_name == "foo" => (), - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } + #[derive(Serialize)] + struct NonEmptyRecord { + foo: String, + } + let record = NonEmptyRecord { + foo: "bar".to_string(), + }; + match record + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::FieldName(field_name)) if field_name == "foo" => (), + unexpected => panic!("Expected an error. Got: {unexpected:?}"), + } - match ().serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); // serialize_unit() delegates to serialize_none() - assert_eq!(value, "None. Cause: Expected: Record. Got: Null"); - assert_eq!(schema, schema); + match ().serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); // serialize_unit() delegates to serialize_none() + assert_eq!(value, "None. Cause: Expected: Record. Got: Null"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.len(), 0); + assert_eq!(buffer.len(), 0); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_enum() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_enum() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "enum", "name": "Suit", "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] }"#, - )?; - - #[derive(Serialize)] - enum Suit { - Spades, - Hearts, - Diamonds, - Clubs, - } - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - Suit::Spades.serialize(&mut serializer)?; - Suit::Hearts.serialize(&mut serializer)?; - Suit::Diamonds.serialize(&mut serializer)?; - Suit::Clubs.serialize(&mut serializer)?; - match None::<()> - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); - assert_eq!(value, "None. Cause: Expected: Enum. Got: Null"); - assert_eq!(schema, schema); + )?; + + #[derive(Serialize)] + enum Suit { + Spades, + Hearts, + Diamonds, + Clubs, } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[0, 2, 4, 6]); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - Ok(()) - } + Suit::Spades.serialize(&mut serializer)?; + Suit::Hearts.serialize(&mut serializer)?; + Suit::Diamonds.serialize(&mut serializer)?; + Suit::Clubs.serialize(&mut serializer)?; + match None::<()> + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); + assert_eq!(value, "None. Cause: Expected: Enum. Got: Null"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), + } + + assert_eq!(buffer.as_slice(), &[0, 2, 4, 6]); - #[test] - fn test_serialize_array() -> TestResult { - let schema = Schema::parse_str( - r#"{ + Ok(()) + } + + #[test] + fn test_serialize_array() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "array", "items": "long" }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let arr: Vec = vec![10, 5, 400]; - arr.serialize(&mut serializer)?; + let arr: Vec = vec![10, 5, 400]; + arr.serialize(&mut serializer)?; - match vec![1_f32] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "1. Cause: Expected: Long. Got: Float"); - assert_eq!(schema, schema); + match vec![1_f32] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "1. Cause: Expected: Long. Got: Float"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0]); + assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_map() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_map() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "map", "values": "long" }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let mut map: BTreeMap = BTreeMap::new(); - map.insert(String::from("item1"), 10); - map.insert(String::from("item2"), 400); + let mut map: BTreeMap = BTreeMap::new(); + map.insert(String::from("item1"), 10); + map.insert(String::from("item2"), 400); - map.serialize(&mut serializer)?; + map.serialize(&mut serializer)?; - let mut map: BTreeMap = BTreeMap::new(); - map.insert(String::from("item1"), "value1"); - match map.serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "string"); - assert_eq!(value, "value1. Cause: Expected: Long. Got: String"); - assert_eq!(schema, schema); + let mut map: BTreeMap = BTreeMap::new(); + map.insert(String::from("item1"), "value1"); + match map.serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "string"); + assert_eq!(value, "value1. Cause: Expected: Long. Got: String"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!( - buffer.as_slice(), - &[ - 4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, 6, - 0 - ] - ); + assert_eq!( + buffer.as_slice(), + &[ + 4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, + 6, 0 + ] + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_nullable_union() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_nullable_union() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": ["null", "long"] }"#, - )?; + )?; - #[derive(Serialize)] - enum NullableLong { - Null, - Long(i64), - } + #[derive(Serialize)] + enum NullableLong { + Null, + Long(i64), + } - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - Some(10i64).serialize(&mut serializer)?; - None::.serialize(&mut serializer)?; - NullableLong::Long(400).serialize(&mut serializer)?; - NullableLong::Null.serialize(&mut serializer)?; + Some(10i64).serialize(&mut serializer)?; + None::.serialize(&mut serializer)?; + NullableLong::Long(400).serialize(&mut serializer)?; + NullableLong::Null.serialize(&mut serializer)?; - match "invalid" - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "string"); - assert_eq!( + match "invalid" + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, value, - "invalid. Cause: Expected one of the union variants [Null, Long]. Got: String" - ); - assert_eq!(schema, schema); + schema, + }) => { + assert_eq!(value_type, "string"); + assert_eq!( + value, + "invalid. Cause: Expected one of the union variants [Null, Long]. Got: String" + ); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[2, 20, 0, 2, 160, 6, 0]); + assert_eq!(buffer.as_slice(), &[2, 20, 0, 2, 160, 6, 0]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_union() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_union() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": ["null", "long", "string"] }"#, - )?; + )?; - #[derive(Serialize)] - enum LongOrString { - Null, - Long(i64), - Str(String), - } + #[derive(Serialize)] + enum LongOrString { + Null, + Long(i64), + Str(String), + } - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - LongOrString::Null.serialize(&mut serializer)?; - LongOrString::Long(400).serialize(&mut serializer)?; - LongOrString::Str(String::from("test")).serialize(&mut serializer)?; + LongOrString::Null.serialize(&mut serializer)?; + LongOrString::Long(400).serialize(&mut serializer)?; + LongOrString::Str(String::from("test")).serialize(&mut serializer)?; - match 1_f64 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f64"); - assert_eq!( + match 1_f64 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, value, - "1. Cause: Cannot find a Double schema in [Null, Long, String]" - ); - assert_eq!(schema, schema); + schema, + }) => { + assert_eq!(value_type, "f64"); + assert_eq!( + value, + "1. Cause: Cannot find a Double schema in [Null, Long, String]" + ); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!( - buffer.as_slice(), - &[0, 2, 160, 6, 4, 8, b't', b'e', b's', b't'] - ); + assert_eq!( + buffer.as_slice(), + &[0, 2, 160, 6, 4, 8, b't', b'e', b's', b't'] + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_fixed() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_fixed() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "fixed", "size": 8, "name": "LongVal" }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?; + Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?; - // non-8 size - match Bytes::new(&[123]) - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "bytes"); - assert_eq!( + // non-8 size + match Bytes::new(&[123]) + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, value, - "7b. Cause: Fixed schema size (8) does not match the value length (1)" - ); // Bytes represents its values as hexadecimals: '7b' is 123 - assert_eq!(schema, schema); + schema, + }) => { + assert_eq!(value_type, "bytes"); + assert_eq!( + value, + "7b. Cause: Fixed schema size (8) does not match the value length (1)" + ); // Bytes represents its values as hexadecimals: '7b' is 123 + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - // array - match [1; 8] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! - assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); - assert_eq!(schema, schema); + // array + match [1; 8] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! + assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - // slice - match &[1, 2, 3, 4, 5, 6, 7, 8] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(*value_type, "tuple"); // TODO: why is this 'tuple' ?! - assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); - assert_eq!(schema, schema); + // slice + match &[1, 2, 3, 4, 5, 6, 7, 8] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(*value_type, "tuple"); // TODO: why is this 'tuple' ?! + assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[10, 124, 31, 97, 14, 201, 3, 88]); + assert_eq!(buffer.as_slice(), &[10, 124, 31, 97, 14, 201, 3, 88]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_decimal_bytes() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_decimal_bytes() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "bytes", "logicalType": "decimal", "precision": 16, "scale": 2 }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let val = Decimal::from(&[251, 155]); - val.serialize(&mut serializer)?; + let val = Decimal::from(&[251, 155]); + val.serialize(&mut serializer)?; - match ().serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); - assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); - assert_eq!(schema, schema); + match ().serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); + assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[4, 251, 155]); + assert_eq!(buffer.as_slice(), &[4, 251, 155]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_decimal_fixed() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_decimal_fixed() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "fixed", "name": "FixedDecimal", "size": 8, @@ -2390,241 +2483,199 @@ mod tests { "precision": 16, "scale": 2 }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]); - val.serialize(&mut serializer)?; + let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]); + val.serialize(&mut serializer)?; - match ().serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); - assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); - assert_eq!(schema, schema); + match ().serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); + assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[0, 0, 0, 0, 0, 0, 251, 155]); + assert_eq!(buffer.as_slice(), &[0, 0, 0, 0, 0, 0, 251, 155]); - Ok(()) - } + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] - fn test_serialize_bigdecimal() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + #[serial(serde_is_human_readable)] + fn test_serialize_bigdecimal() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "bytes", "logicalType": "big-decimal" }"#, - )?; + )?; - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2); - val.serialize(&mut serializer)?; + let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2); + val.serialize(&mut serializer)?; - assert_eq!(buffer.as_slice(), &[10, 6, 0, 195, 104, 4]); + assert_eq!(buffer.as_slice(), &[10, 6, 0, 195, 104, 4]); - Ok(()) - } + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] - fn test_serialize_uuid() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + #[serial(serde_is_human_readable)] + fn test_serialize_uuid() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "string", "logicalType": "uuid" }"#, - )?; + )?; - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - "8c28da81-238c-4326-bddd-4e3d00cc5099" - .parse::()? - .serialize(&mut serializer)?; + "8c28da81-238c-4326-bddd-4e3d00cc5099" + .parse::()? + .serialize(&mut serializer)?; - match 1_u8.serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "u8"); - assert_eq!(value, "1. Cause: Expected: Uuid. Got: Int"); - assert_eq!(schema, schema); + match 1_u8.serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "u8"); + assert_eq!(value, "1. Cause: Expected: Uuid. Got: Int"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!( - buffer.as_slice(), - &[ - 72, b'8', b'c', b'2', b'8', b'd', b'a', b'8', b'1', b'-', b'2', b'3', b'8', b'c', - b'-', b'4', b'3', b'2', b'6', b'-', b'b', b'd', b'd', b'd', b'-', b'4', b'e', b'3', - b'd', b'0', b'0', b'c', b'c', b'5', b'0', b'9', b'9' - ] - ); + assert_eq!( + buffer.as_slice(), + &[ + 72, b'8', b'c', b'2', b'8', b'd', b'a', b'8', b'1', b'-', b'2', b'3', b'8', + b'c', b'-', b'4', b'3', b'2', b'6', b'-', b'b', b'd', b'd', b'd', b'-', b'4', + b'e', b'3', b'd', b'0', b'0', b'c', b'c', b'5', b'0', b'9', b'9' + ] + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_date() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_date() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "int", "logicalType": "date" }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - 100_u8.serialize(&mut serializer)?; - 1000_u16.serialize(&mut serializer)?; - 10000_u32.serialize(&mut serializer)?; - 1000_i16.serialize(&mut serializer)?; - 10000_i32.serialize(&mut serializer)?; + 100_u8.serialize(&mut serializer)?; + 1000_u16.serialize(&mut serializer)?; + 10000_u32.serialize(&mut serializer)?; + 1000_i16.serialize(&mut serializer)?; + 10000_i32.serialize(&mut serializer)?; - match 10000_f32 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "10000. Cause: Expected: Date. Got: Float"); - assert_eq!(schema, schema); + match 10000_f32 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "10000. Cause: Expected: Date. Got: Float"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!( - buffer.as_slice(), - &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] - ); + assert_eq!( + buffer.as_slice(), + &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_time_millis() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_time_millis() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "int", "logicalType": "time-millis" }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - 100_u8.serialize(&mut serializer)?; - 1000_u16.serialize(&mut serializer)?; - 10000_u32.serialize(&mut serializer)?; - 1000_i16.serialize(&mut serializer)?; - 10000_i32.serialize(&mut serializer)?; + 100_u8.serialize(&mut serializer)?; + 1000_u16.serialize(&mut serializer)?; + 10000_u32.serialize(&mut serializer)?; + 1000_i16.serialize(&mut serializer)?; + 10000_i32.serialize(&mut serializer)?; - match 10000_f32 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "10000. Cause: Expected: TimeMillis. Got: Float"); - assert_eq!(schema, schema); + match 10000_f32 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "10000. Cause: Expected: TimeMillis. Got: Float"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!( - buffer.as_slice(), - &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] - ); + assert_eq!( + buffer.as_slice(), + &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_time_micros() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_time_micros() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "long", "logicalType": "time-micros" }"#, - )?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 100_u8.serialize(&mut serializer)?; - 1000_u16.serialize(&mut serializer)?; - 10000_u32.serialize(&mut serializer)?; - 1000_i16.serialize(&mut serializer)?; - 10000_i32.serialize(&mut serializer)?; - 10000_i64.serialize(&mut serializer)?; - - match 10000_f32 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "10000. Cause: Expected: TimeMicros. Got: Float"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - - assert_eq!( - buffer.as_slice(), - &[ - 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1 - ] - ); - - Ok(()) - } - - #[test] - fn test_serialize_timestamp() -> TestResult { - for precision in ["millis", "micros", "nanos"] { - let schema = Schema::parse_str(&format!( - r#"{{ - "type": "long", - "logicalType": "timestamp-{precision}" - }}"# - ))?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2638,7 +2689,7 @@ mod tests { 10000_i32.serialize(&mut serializer)?; 10000_i64.serialize(&mut serializer)?; - match 10000_f64 + match 10000_f32 .serialize(&mut serializer) .map_err(Error::into_details) { @@ -2647,17 +2698,8 @@ mod tests { value, schema, }) => { - let mut capital_precision = precision.to_string(); - if let Some(c) = capital_precision.chars().next() { - capital_precision.replace_range(..1, &c.to_uppercase().to_string()); - } - assert_eq!(value_type, "f64"); - assert_eq!( - value, - format!( - "10000. Cause: Expected: Timestamp{capital_precision}. Got: Double" - ) - ); + assert_eq!(value_type, "f32"); + assert_eq!(value, "10000. Cause: Expected: TimeMicros. Got: Float"); assert_eq!(schema, schema); } unexpected => panic!("Expected an error. Got: {unexpected:?}"), @@ -2669,59 +2711,118 @@ mod tests { 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1 ] ); + + Ok(()) } - Ok(()) - } + #[test] + fn test_serialize_timestamp() -> TestResult { + for precision in ["millis", "micros", "nanos"] { + let schema = Schema::parse_str(&format!( + r#"{{ + "type": "long", + "logicalType": "timestamp-{precision}" + }}"# + ))?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 100_u8.serialize(&mut serializer)?; + 1000_u16.serialize(&mut serializer)?; + 10000_u32.serialize(&mut serializer)?; + 1000_i16.serialize(&mut serializer)?; + 10000_i32.serialize(&mut serializer)?; + 10000_i64.serialize(&mut serializer)?; + + match 10000_f64 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + let mut capital_precision = precision.to_string(); + if let Some(c) = capital_precision.chars().next() { + capital_precision.replace_range(..1, &c.to_uppercase().to_string()); + } + assert_eq!(value_type, "f64"); + assert_eq!( + value, + format!( + "10000. Cause: Expected: Timestamp{capital_precision}. Got: Double" + ) + ); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), + } + + assert_eq!( + buffer.as_slice(), + &[ + 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1 + ] + ); + } + + Ok(()) + } - #[test] - fn test_serialize_duration() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + fn test_serialize_duration() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "fixed", "size": 12, "name": "duration", "logicalType": "duration" }"#, - )?; + )?; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let duration_bytes = - ByteArray::new(Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into()); - duration_bytes.serialize(&mut serializer)?; + let duration_bytes = ByteArray::new( + Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into(), + ); + duration_bytes.serialize(&mut serializer)?; - match [1; 12] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! - assert_eq!( + match [1; 12] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, value, - "tuple (len=12). Cause: Expected: Duration. Got: Array" - ); - assert_eq!(schema, schema); + schema, + }) => { + assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! + assert_eq!( + value, + "tuple (len=12). Cause: Expected: Duration. Got: Array" + ); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - assert_eq!(buffer.as_slice(), &[3, 0, 0, 0, 2, 0, 0, 0, 176, 4, 0, 0]); + assert_eq!(buffer.as_slice(), &[3, 0, 0, 0, 2, 0, 0, 0, 176, 4, 0, 0]); - Ok(()) - } + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] // for BigDecimal and Uuid - fn test_serialize_recursive_record() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[test] + #[serial(serde_is_human_readable)] // for BigDecimal and Uuid + fn test_serialize_recursive_record() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "record", "name": "TestRecord", "fields": [ @@ -2732,52 +2833,53 @@ mod tests { {"name": "innerRecord", "type": ["null", "TestRecord"]} ] }"#, - )?; - - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct TestRecord { - string_field: String, - int_field: i32, - big_decimal_field: BigDecimal, - uuid_field: Uuid, - // #[serde(skip_serializing_if = "Option::is_none")] => Never ignore None! - inner_record: Option>, - } - - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let mut buffer: Vec = Vec::new(); - let rs = ResolvedSchema::try_from(&schema)?; - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None); - - let good_record = TestRecord { - string_field: String::from("test"), - int_field: 10, - big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2), - uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5098".parse::()?, - inner_record: Some(Box::new(TestRecord { - string_field: String::from("inner_test"), - int_field: 100, - big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![20038]), 2), - uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5099".parse::()?, - inner_record: None, - })), - }; - good_record.serialize(&mut serializer)?; - - assert_eq!( - buffer.as_slice(), - &[ - 8, 116, 101, 115, 116, 20, 10, 6, 0, 195, 104, 4, 72, 56, 99, 50, 56, 100, 97, 56, - 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, 101, 51, - 100, 48, 48, 99, 99, 53, 48, 57, 56, 2, 20, 105, 110, 110, 101, 114, 95, 116, 101, - 115, 116, 200, 1, 8, 4, 78, 70, 4, 72, 56, 99, 50, 56, 100, 97, 56, 49, 45, 50, 51, - 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, 101, 51, 100, 48, 48, - 99, 99, 53, 48, 57, 57, 0 - ] - ); + )?; + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct TestRecord { + string_field: String, + int_field: i32, + big_decimal_field: BigDecimal, + uuid_field: Uuid, + // #[serde(skip_serializing_if = "Option::is_none")] => Never ignore None! + inner_record: Option>, + } + + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let mut buffer: Vec = Vec::new(); + let rs = ResolvedSchema::try_from(&schema)?; + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None); + + let good_record = TestRecord { + string_field: String::from("test"), + int_field: 10, + big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2), + uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5098".parse::()?, + inner_record: Some(Box::new(TestRecord { + string_field: String::from("inner_test"), + int_field: 100, + big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![20038]), 2), + uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5099".parse::()?, + inner_record: None, + })), + }; + good_record.serialize(&mut serializer)?; + + assert_eq!( + buffer.as_slice(), + &[ + 8, 116, 101, 115, 116, 20, 10, 6, 0, 195, 104, 4, 72, 56, 99, 50, 56, 100, 97, + 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, + 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 56, 2, 20, 105, 110, 110, 101, 114, + 95, 116, 101, 115, 116, 200, 1, 8, 4, 78, 70, 4, 72, 56, 99, 50, 56, 100, 97, + 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, + 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 57, 0 + ] + ); - Ok(()) + Ok(()) + } } } diff --git a/avro/src/types.rs b/avro/src/types.rs index cddacaae..6e1c3c02 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -16,118 +16,141 @@ // under the License. //! Logic handling the intermediate representation of Avro values. -use crate::{ - AvroResult, Error, - bigdecimal::{deserialize_big_decimal, serialize_big_decimal}, - decimal::Decimal, - duration::Duration, - error::Details, - schema::{ - DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, - RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, - }, -}; -use bigdecimal::BigDecimal; -use log::{debug, error}; -use serde_json::{Number, Value as JsonValue}; -use std::{ - borrow::Borrow, - collections::{BTreeMap, HashMap}, - fmt::Debug, - hash::BuildHasher, - str::FromStr, -}; -use uuid::Uuid; - -/// Compute the maximum decimal value precision of a byte array of length `len` could hold. -fn max_prec_for_len(len: usize) -> Result { - let len = i32::try_from(len).map_err(|e| Details::ConvertLengthToI32(e, len))?; - Ok((2.0_f64.powi(8 * len - 1) - 1.0).log10().floor() as usize) -} -/// A valid Avro value. -/// -/// More information about Avro values can be found in the [Avro -/// Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) -#[derive(Clone, Debug, PartialEq, strum_macros::EnumDiscriminants)] -#[strum_discriminants(name(ValueKind))] -pub enum Value { - /// A `null` Avro value. - Null, - /// A `boolean` Avro value. - Boolean(bool), - /// A `int` Avro value. - Int(i32), - /// A `long` Avro value. - Long(i64), - /// A `float` Avro value. - Float(f32), - /// A `double` Avro value. - Double(f64), - /// A `bytes` Avro value. - Bytes(Vec), - /// A `string` Avro value. - String(String), - /// A `fixed` Avro value. - /// The size of the fixed value is represented as a `usize`. - Fixed(usize, Vec), - /// An `enum` Avro value. - /// - /// An Enum is represented by a symbol and its position in the symbols list - /// of its corresponding schema. - /// This allows schema-less encoding, as well as schema resolution while - /// reading values. - Enum(u32, String), - /// An `union` Avro value. - /// - /// A Union is represented by the value it holds and its position in the type list - /// of its corresponding schema - /// This allows schema-less encoding, as well as schema resolution while - /// reading values. - Union(u32, Box), - /// An `array` Avro value. - Array(Vec), - /// A `map` Avro value. - Map(HashMap), - /// A `record` Avro value. - /// - /// A Record is represented by a vector of (``, `value`). - /// This allows schema-less encoding. - /// - /// See [Record](types.Record) for a more user-friendly support. - Record(Vec<(String, Value)>), - /// A date value. - /// - /// Serialized and deserialized as `i32` directly. Can only be deserialized properly with a - /// schema. - Date(i32), - /// An Avro Decimal value. Bytes are in big-endian order, per the Avro spec. - Decimal(Decimal), - /// An Avro Decimal value. - BigDecimal(BigDecimal), - /// Time in milliseconds. - TimeMillis(i32), - /// Time in microseconds. - TimeMicros(i64), - /// Timestamp in milliseconds. - TimestampMillis(i64), - /// Timestamp in microseconds. - TimestampMicros(i64), - /// Timestamp in nanoseconds. - TimestampNanos(i64), - /// Local timestamp in milliseconds. - LocalTimestampMillis(i64), - /// Local timestamp in microseconds. - LocalTimestampMicros(i64), - /// Local timestamp in nanoseconds. - LocalTimestampNanos(i64), - /// Avro Duration. An amount of time defined by months, days and milliseconds. - Duration(Duration), - /// Universally unique identifier. - Uuid(Uuid), -} +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + bigdecimal::tokio => bigdecimal::sync, + decimal::tokio => decimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod types { -macro_rules! to_value( + use crate::{ + AvroResult, Error, + bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, + decimal::tokio::Decimal, + duration::Duration, + error::tokio::{Details, Error}, + schema::tokio::{ + DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, + RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, + }, + }; + use bigdecimal::BigDecimal; + #[cfg(feature = "tokio")] + use futures::future::try_join_all; + use log::{debug, error}; + use serde_json::{Number, Value as JsonValue}; + use std::{ + borrow::Borrow, + collections::{BTreeMap, HashMap}, + fmt::Debug, + hash::BuildHasher, + str::FromStr, + }; + use uuid::Uuid; + + /// Compute the maximum decimal value precision of a byte array of length `len` could hold. + fn max_prec_for_len(len: usize) -> Result { + let len = i32::try_from(len).map_err(|e| Details::ConvertLengthToI32(e, len))?; + Ok((2.0_f64.powi(8 * len - 1) - 1.0).log10().floor() as usize) + } + + /// A valid Avro value. + /// + /// More information about Avro values can be found in the [Avro + /// Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) + #[derive(Clone, Debug, PartialEq, strum_macros::EnumDiscriminants)] + #[strum_discriminants(name(ValueKind))] + pub enum Value { + /// A `null` Avro value. + Null, + /// A `boolean` Avro value. + Boolean(bool), + /// A `int` Avro value. + Int(i32), + /// A `long` Avro value. + Long(i64), + /// A `float` Avro value. + Float(f32), + /// A `double` Avro value. + Double(f64), + /// A `bytes` Avro value. + Bytes(Vec), + /// A `string` Avro value. + String(String), + /// A `fixed` Avro value. + /// The size of the fixed value is represented as a `usize`. + Fixed(usize, Vec), + /// An `enum` Avro value. + /// + /// An Enum is represented by a symbol and its position in the symbols list + /// of its corresponding schema. + /// This allows schema-less encoding, as well as schema resolution while + /// reading values. + Enum(u32, String), + /// An `union` Avro value. + /// + /// A Union is represented by the value it holds and its position in the type list + /// of its corresponding schema + /// This allows schema-less encoding, as well as schema resolution while + /// reading values. + Union(u32, Box), + /// An `array` Avro value. + Array(Vec), + /// A `map` Avro value. + Map(HashMap), + /// A `record` Avro value. + /// + /// A Record is represented by a vector of (``, `value`). + /// This allows schema-less encoding. + /// + /// See [Record](types.Record) for a more user-friendly support. + Record(Vec<(String, Value)>), + /// A date value. + /// + /// Serialized and deserialized as `i32` directly. Can only be deserialized properly with a + /// schema. + Date(i32), + /// An Avro Decimal value. Bytes are in big-endian order, per the Avro spec. + Decimal(Decimal), + /// An Avro Decimal value. + BigDecimal(BigDecimal), + /// Time in milliseconds. + TimeMillis(i32), + /// Time in microseconds. + TimeMicros(i64), + /// Timestamp in milliseconds. + TimestampMillis(i64), + /// Timestamp in microseconds. + TimestampMicros(i64), + /// Timestamp in nanoseconds. + TimestampNanos(i64), + /// Local timestamp in milliseconds. + LocalTimestampMillis(i64), + /// Local timestamp in microseconds. + LocalTimestampMicros(i64), + /// Local timestamp in nanoseconds. + LocalTimestampNanos(i64), + /// Avro Duration. An amount of time defined by months, days and milliseconds. + Duration(Duration), + /// Universally unique identifier. + Uuid(Uuid), + } + + macro_rules! to_value( ($type:ty, $variant_constructor:expr) => ( impl From<$type> for Value { fn from(value: $type) -> Self { @@ -137,979 +160,1013 @@ macro_rules! to_value( ); ); -to_value!(bool, Value::Boolean); -to_value!(i32, Value::Int); -to_value!(i64, Value::Long); -to_value!(f32, Value::Float); -to_value!(f64, Value::Double); -to_value!(String, Value::String); -to_value!(Vec, Value::Bytes); -to_value!(uuid::Uuid, Value::Uuid); -to_value!(Decimal, Value::Decimal); -to_value!(BigDecimal, Value::BigDecimal); -to_value!(Duration, Value::Duration); - -impl From<()> for Value { - fn from(_: ()) -> Self { - Self::Null - } -} - -impl From for Value { - fn from(value: usize) -> Self { - i64::try_from(value) - .expect("cannot convert usize to i64") - .into() - } -} - -impl From<&str> for Value { - fn from(value: &str) -> Self { - Self::String(value.to_owned()) + to_value!(bool, Value::Boolean); + to_value!(i32, Value::Int); + to_value!(i64, Value::Long); + to_value!(f32, Value::Float); + to_value!(f64, Value::Double); + to_value!(String, Value::String); + to_value!(Vec, Value::Bytes); + to_value!(uuid::Uuid, Value::Uuid); + to_value!(Decimal, Value::Decimal); + to_value!(BigDecimal, Value::BigDecimal); + to_value!(Duration, Value::Duration); + + impl From<()> for Value { + fn from(_: ()) -> Self { + Self::Null + } } -} -impl From<&[u8]> for Value { - fn from(value: &[u8]) -> Self { - Self::Bytes(value.to_owned()) + impl From for Value { + fn from(value: usize) -> Self { + i64::try_from(value) + .expect("cannot convert usize to i64") + .into() + } } -} -impl From> for Value -where - T: Into, -{ - fn from(value: Option) -> Self { - // FIXME: this is incorrect in case first type in union is not "none" - Self::Union( - value.is_some() as u32, - Box::new(value.map_or_else(|| Self::Null, Into::into)), - ) + impl From<&str> for Value { + fn from(value: &str) -> Self { + Self::String(value.to_owned()) + } } -} -impl From> for Value -where - K: Into, - V: Into, - S: BuildHasher, -{ - fn from(value: HashMap) -> Self { - Self::Map( - value - .into_iter() - .map(|(key, value)| (key.into(), value.into())) - .collect(), - ) + impl From<&[u8]> for Value { + fn from(value: &[u8]) -> Self { + Self::Bytes(value.to_owned()) + } } -} -/// Utility interface to build `Value::Record` objects. -#[derive(Debug, Clone)] -pub struct Record<'a> { - /// List of fields contained in the record. - /// Ordered according to the fields in the schema given to create this - /// `Record` object. Any unset field defaults to `Value::Null`. - pub fields: Vec<(String, Value)>, - schema_lookup: &'a BTreeMap, -} - -impl Record<'_> { - /// Create a `Record` given a `Schema`. - /// - /// If the `Schema` is not a `Schema::Record` variant, `None` will be returned. - pub fn new(schema: &Schema) -> Option> { - match *schema { - Schema::Record(RecordSchema { - fields: ref schema_fields, - lookup: ref schema_lookup, - .. - }) => { - let mut fields = Vec::with_capacity(schema_fields.len()); - for schema_field in schema_fields.iter() { - fields.push((schema_field.name.clone(), Value::Null)); - } - - Some(Record { - fields, - schema_lookup, - }) - } - _ => None, + impl From> for Value + where + T: Into, + { + fn from(value: Option) -> Self { + // FIXME: this is incorrect in case first type in union is not "none" + Self::Union( + value.is_some() as u32, + Box::new(value.map_or_else(|| Self::Null, Into::into)), + ) } } - /// Put a compatible value (implementing the `ToAvro` trait) in the - /// `Record` for a given `field` name. - /// - /// **NOTE** Only ensure that the field name is present in the `Schema` given when creating - /// this `Record`. Does not perform any schema validation. - pub fn put(&mut self, field: &str, value: V) + impl From> for Value where - V: Into, + K: Into, + V: Into, + S: BuildHasher, { - if let Some(&position) = self.schema_lookup.get(field) { - self.fields[position].1 = value.into() + fn from(value: HashMap) -> Self { + Self::Map( + value + .into_iter() + .map(|(key, value)| (key.into(), value.into())) + .collect(), + ) } } - /// Get the value for a given field name. - /// Returns `None` if the field is not present in the schema - pub fn get(&self, field: &str) -> Option<&Value> { - self.schema_lookup - .get(field) - .map(|&position| &self.fields[position].1) + /// Utility interface to build `Value::Record` objects. + #[derive(Debug, Clone)] + pub struct Record<'a> { + /// List of fields contained in the record. + /// Ordered according to the fields in the schema given to create this + /// `Record` object. Any unset field defaults to `Value::Null`. + pub fields: Vec<(String, Value)>, + schema_lookup: &'a BTreeMap, } -} -impl<'a> From> for Value { - fn from(value: Record<'a>) -> Self { - Self::Record(value.fields) - } -} + impl Record<'_> { + /// Create a `Record` given a `Schema`. + /// + /// If the `Schema` is not a `Schema::Record` variant, `None` will be returned. + pub fn new(schema: &Schema) -> Option> { + match *schema { + Schema::Record(RecordSchema { + fields: ref schema_fields, + lookup: ref schema_lookup, + .. + }) => { + let mut fields = Vec::with_capacity(schema_fields.len()); + for schema_field in schema_fields.iter() { + fields.push((schema_field.name.clone(), Value::Null)); + } -impl From for Value { - fn from(value: JsonValue) -> Self { - match value { - JsonValue::Null => Self::Null, - JsonValue::Bool(b) => b.into(), - JsonValue::Number(ref n) if n.is_i64() => { - let n = n.as_i64().unwrap(); - if n >= i32::MIN as i64 && n <= i32::MAX as i64 { - Value::Int(n as i32) - } else { - Value::Long(n) + Some(Record { + fields, + schema_lookup, + }) } + _ => None, } - JsonValue::Number(ref n) if n.is_f64() => Value::Double(n.as_f64().unwrap()), - JsonValue::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great - JsonValue::String(s) => s.into(), - JsonValue::Array(items) => Value::Array(items.into_iter().map(Value::from).collect()), - JsonValue::Object(items) => Value::Map( - items - .into_iter() - .map(|(key, value)| (key, value.into())) - .collect(), - ), } - } -} -/// Convert Avro values to Json values -impl TryFrom for JsonValue { - type Error = crate::error::Error; - fn try_from(value: Value) -> AvroResult { - match value { - Value::Null => Ok(Self::Null), - Value::Boolean(b) => Ok(Self::Bool(b)), - Value::Int(i) => Ok(Self::Number(i.into())), - Value::Long(l) => Ok(Self::Number(l.into())), - Value::Float(f) => Number::from_f64(f.into()) - .map(Self::Number) - .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), - Value::Double(d) => Number::from_f64(d) - .map(Self::Number) - .ok_or_else(|| Details::ConvertF64ToJson(d).into()), - Value::Bytes(bytes) => Ok(Self::Array(bytes.into_iter().map(|b| b.into()).collect())), - Value::String(s) => Ok(Self::String(s)), - Value::Fixed(_size, items) => { - Ok(Self::Array(items.into_iter().map(|v| v.into()).collect())) - } - Value::Enum(_i, s) => Ok(Self::String(s)), - Value::Union(_i, b) => Self::try_from(*b), - Value::Array(items) => items - .into_iter() - .map(Self::try_from) - .collect::, _>>() - .map(Self::Array), - Value::Map(items) => items - .into_iter() - .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) - .collect::, _>>() - .map(|v| Self::Object(v.into_iter().collect())), - Value::Record(items) => items - .into_iter() - .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) - .collect::, _>>() - .map(|v| Self::Object(v.into_iter().collect())), - Value::Date(d) => Ok(Self::Number(d.into())), - Value::Decimal(ref d) => >::try_from(d) - .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), - Value::BigDecimal(ref bg) => { - let vec1: Vec = serialize_big_decimal(bg)?; - Ok(Self::Array(vec1.into_iter().map(|b| b.into()).collect())) + /// Put a compatible value (implementing the `ToAvro` trait) in the + /// `Record` for a given `field` name. + /// + /// **NOTE** Only ensure that the field name is present in the `Schema` given when creating + /// this `Record`. Does not perform any schema validation. + pub fn put(&mut self, field: &str, value: V) + where + V: Into, + { + if let Some(&position) = self.schema_lookup.get(field) { + self.fields[position].1 = value.into() } - Value::TimeMillis(t) => Ok(Self::Number(t.into())), - Value::TimeMicros(t) => Ok(Self::Number(t.into())), - Value::TimestampMillis(t) => Ok(Self::Number(t.into())), - Value::TimestampMicros(t) => Ok(Self::Number(t.into())), - Value::TimestampNanos(t) => Ok(Self::Number(t.into())), - Value::LocalTimestampMillis(t) => Ok(Self::Number(t.into())), - Value::LocalTimestampMicros(t) => Ok(Self::Number(t.into())), - Value::LocalTimestampNanos(t) => Ok(Self::Number(t.into())), - Value::Duration(d) => Ok(Self::Array( - <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), - )), - Value::Uuid(uuid) => Ok(Self::String(uuid.as_hyphenated().to_string())), } - } -} -impl Value { - /// Validate the value against the given [Schema](../schema/enum.Schema.html). - /// - /// See the [Avro specification](https://avro.apache.org/docs/current/specification) - /// for the full set of rules of schema validation. - pub fn validate(&self, schema: &Schema) -> bool { - self.validate_schemata(vec![schema]) + /// Get the value for a given field name. + /// Returns `None` if the field is not present in the schema + pub fn get(&self, field: &str) -> Option<&Value> { + self.schema_lookup + .get(field) + .map(|&position| &self.fields[position].1) + } } - pub fn validate_schemata(&self, schemata: Vec<&Schema>) -> bool { - let rs = ResolvedSchema::try_from(schemata.clone()) - .expect("Schemata didn't successfully resolve"); - let schemata_len = schemata.len(); - schemata.iter().any(|schema| { - let enclosing_namespace = schema.namespace(); + impl<'a> From> for Value { + fn from(value: Record<'a>) -> Self { + Self::Record(value.fields) + } + } - match self.validate_internal(schema, rs.get_names(), &enclosing_namespace) { - Some(reason) => { - let log_message = - format!("Invalid value: {self:?} for schema: {schema:?}. Reason: {reason}"); - if schemata_len == 1 { - error!("{log_message}"); + impl From for Value { + fn from(value: JsonValue) -> Self { + match value { + JsonValue::Null => Self::Null, + JsonValue::Bool(b) => b.into(), + JsonValue::Number(ref n) if n.is_i64() => { + let n = n.as_i64().unwrap(); + if n >= i32::MIN as i64 && n <= i32::MAX as i64 { + Value::Int(n as i32) } else { - debug!("{log_message}"); - }; - false + Value::Long(n) + } } - None => true, + JsonValue::Number(ref n) if n.is_f64() => Value::Double(n.as_f64().unwrap()), + JsonValue::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great + JsonValue::String(s) => s.into(), + JsonValue::Array(items) => { + Value::Array(items.into_iter().map(Value::from).collect()) + } + JsonValue::Object(items) => Value::Map( + items + .into_iter() + .map(|(key, value)| (key, value.into())) + .collect(), + ), } - }) + } } - fn accumulate(accumulator: Option, other: Option) -> Option { - match (accumulator, other) { - (None, None) => None, - (None, s @ Some(_)) => s, - (s @ Some(_), None) => s, - (Some(reason1), Some(reason2)) => Some(format!("{reason1}\n{reason2}")), + /// Convert Avro values to Json values + impl TryFrom for JsonValue { + type Error = Error; + fn try_from(value: Value) -> AvroResult { + match value { + Value::Null => Ok(Self::Null), + Value::Boolean(b) => Ok(Self::Bool(b)), + Value::Int(i) => Ok(Self::Number(i.into())), + Value::Long(l) => Ok(Self::Number(l.into())), + Value::Float(f) => Number::from_f64(f.into()) + .map(Self::Number) + .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), + Value::Double(d) => Number::from_f64(d) + .map(Self::Number) + .ok_or_else(|| Details::ConvertF64ToJson(d).into()), + Value::Bytes(bytes) => { + Ok(Self::Array(bytes.into_iter().map(|b| b.into()).collect())) + } + Value::String(s) => Ok(Self::String(s)), + Value::Fixed(_size, items) => { + Ok(Self::Array(items.into_iter().map(|v| v.into()).collect())) + } + Value::Enum(_i, s) => Ok(Self::String(s)), + Value::Union(_i, b) => Self::try_from(*b), + Value::Array(items) => items + .into_iter() + .map(Self::try_from) + .collect::, _>>() + .map(Self::Array), + Value::Map(items) => items + .into_iter() + .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) + .collect::, _>>() + .map(|v| Self::Object(v.into_iter().collect())), + Value::Record(items) => items + .into_iter() + .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) + .collect::, _>>() + .map(|v| Self::Object(v.into_iter().collect())), + Value::Date(d) => Ok(Self::Number(d.into())), + Value::Decimal(ref d) => >::try_from(d) + .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), + Value::BigDecimal(ref bg) => { + let vec1: Vec = serialize_big_decimal(bg)?; + Ok(Self::Array(vec1.into_iter().map(|b| b.into()).collect())) + } + Value::TimeMillis(t) => Ok(Self::Number(t.into())), + Value::TimeMicros(t) => Ok(Self::Number(t.into())), + Value::TimestampMillis(t) => Ok(Self::Number(t.into())), + Value::TimestampMicros(t) => Ok(Self::Number(t.into())), + Value::TimestampNanos(t) => Ok(Self::Number(t.into())), + Value::LocalTimestampMillis(t) => Ok(Self::Number(t.into())), + Value::LocalTimestampMicros(t) => Ok(Self::Number(t.into())), + Value::LocalTimestampNanos(t) => Ok(Self::Number(t.into())), + Value::Duration(d) => Ok(Self::Array( + <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), + )), + Value::Uuid(uuid) => Ok(Self::String(uuid.as_hyphenated().to_string())), + } } } - /// Validates the value against the provided schema. - pub(crate) fn validate_internal + Debug>( - &self, - schema: &Schema, - names: &HashMap, - enclosing_namespace: &Namespace, - ) -> Option { - match (self, schema) { - (_, Schema::Ref { name }) => { - let name = name.fully_qualified_name(enclosing_namespace); - names.get(&name).map_or_else( - || { - Some(format!( - "Unresolved schema reference: '{:?}'. Parsed names: {:?}", - name, - names.keys() - )) - }, - |s| self.validate_internal(s.borrow(), names, &name.namespace), - ) - } - (&Value::Null, &Schema::Null) => None, - (&Value::Boolean(_), &Schema::Boolean) => None, - (&Value::Int(_), &Schema::Int) => None, - (&Value::Int(_), &Schema::Date) => None, - (&Value::Int(_), &Schema::TimeMillis) => None, - (&Value::Int(_), &Schema::Long) => None, - (&Value::Long(_), &Schema::Long) => None, - (&Value::Long(_), &Schema::TimeMicros) => None, - (&Value::Long(_), &Schema::TimestampMillis) => None, - (&Value::Long(_), &Schema::TimestampMicros) => None, - (&Value::Long(_), &Schema::LocalTimestampMillis) => None, - (&Value::Long(_), &Schema::LocalTimestampMicros) => None, - (&Value::TimestampMicros(_), &Schema::TimestampMicros) => None, - (&Value::TimestampMillis(_), &Schema::TimestampMillis) => None, - (&Value::TimestampNanos(_), &Schema::TimestampNanos) => None, - (&Value::LocalTimestampMicros(_), &Schema::LocalTimestampMicros) => None, - (&Value::LocalTimestampMillis(_), &Schema::LocalTimestampMillis) => None, - (&Value::LocalTimestampNanos(_), &Schema::LocalTimestampNanos) => None, - (&Value::TimeMicros(_), &Schema::TimeMicros) => None, - (&Value::TimeMillis(_), &Schema::TimeMillis) => None, - (&Value::Date(_), &Schema::Date) => None, - (&Value::Decimal(_), &Schema::Decimal { .. }) => None, - (&Value::BigDecimal(_), &Schema::BigDecimal) => None, - (&Value::Duration(_), &Schema::Duration) => None, - (&Value::Uuid(_), &Schema::Uuid) => None, - (&Value::Float(_), &Schema::Float) => None, - (&Value::Float(_), &Schema::Double) => None, - (&Value::Double(_), &Schema::Double) => None, - (&Value::Bytes(_), &Schema::Bytes) => None, - (&Value::Bytes(_), &Schema::Decimal { .. }) => None, - (&Value::String(_), &Schema::String) => None, - (&Value::String(_), &Schema::Uuid) => None, - (&Value::Fixed(n, _), &Schema::Fixed(FixedSchema { size, .. })) => { - if n != size { - Some(format!( - "The value's size ({n}) is different than the schema's size ({size})" - )) - } else { - None + impl Value { + /// Validate the value against the given [Schema](../schema/enum.Schema.html). + /// + /// See the [Avro specification](https://avro.apache.org/docs/current/specification) + /// for the full set of rules of schema validation. + pub fn validate(&self, schema: &Schema) -> bool { + self.validate_schemata(vec![schema]) + } + + pub fn validate_schemata(&self, schemata: Vec<&Schema>) -> bool { + let rs = ResolvedSchema::try_from(schemata.clone()) + .expect("Schemata didn't successfully resolve"); + let schemata_len = schemata.len(); + schemata.iter().any(|schema| { + let enclosing_namespace = schema.namespace(); + + match self.validate_internal(schema, rs.get_names(), &enclosing_namespace) { + Some(reason) => { + let log_message = format!( + "Invalid value: {self:?} for schema: {schema:?}. Reason: {reason}" + ); + if schemata_len == 1 { + error!("{log_message}"); + } else { + debug!("{log_message}"); + }; + false + } + None => true, } + }) + } + + fn accumulate(accumulator: Option, other: Option) -> Option { + match (accumulator, other) { + (None, None) => None, + (None, s @ Some(_)) => s, + (s @ Some(_), None) => s, + (Some(reason1), Some(reason2)) => Some(format!("{reason1}\n{reason2}")), } - (Value::Bytes(b), &Schema::Fixed(FixedSchema { size, .. })) => { - if b.len() != size { - Some(format!( - "The bytes' length ({}) is different than the schema's size ({})", - b.len(), - size - )) - } else { - None + } + + /// Validates the value against the provided schema. + pub(crate) fn validate_internal + Debug>( + &self, + schema: &Schema, + names: &HashMap, + enclosing_namespace: &Namespace, + ) -> Option { + match (self, schema) { + (_, Schema::Ref { name }) => { + let name = name.fully_qualified_name(enclosing_namespace); + names.get(&name).map_or_else( + || { + Some(format!( + "Unresolved schema reference: '{:?}'. Parsed names: {:?}", + name, + names.keys() + )) + }, + |s| self.validate_internal(s.borrow(), names, &name.namespace), + ) } - } - (&Value::Fixed(n, _), &Schema::Duration) => { - if n != 12 { - Some(format!( - "The value's size ('{n}') must be exactly 12 to be a Duration" - )) - } else { - None + (&Value::Null, &Schema::Null) => None, + (&Value::Boolean(_), &Schema::Boolean) => None, + (&Value::Int(_), &Schema::Int) => None, + (&Value::Int(_), &Schema::Date) => None, + (&Value::Int(_), &Schema::TimeMillis) => None, + (&Value::Int(_), &Schema::Long) => None, + (&Value::Long(_), &Schema::Long) => None, + (&Value::Long(_), &Schema::TimeMicros) => None, + (&Value::Long(_), &Schema::TimestampMillis) => None, + (&Value::Long(_), &Schema::TimestampMicros) => None, + (&Value::Long(_), &Schema::LocalTimestampMillis) => None, + (&Value::Long(_), &Schema::LocalTimestampMicros) => None, + (&Value::TimestampMicros(_), &Schema::TimestampMicros) => None, + (&Value::TimestampMillis(_), &Schema::TimestampMillis) => None, + (&Value::TimestampNanos(_), &Schema::TimestampNanos) => None, + (&Value::LocalTimestampMicros(_), &Schema::LocalTimestampMicros) => None, + (&Value::LocalTimestampMillis(_), &Schema::LocalTimestampMillis) => None, + (&Value::LocalTimestampNanos(_), &Schema::LocalTimestampNanos) => None, + (&Value::TimeMicros(_), &Schema::TimeMicros) => None, + (&Value::TimeMillis(_), &Schema::TimeMillis) => None, + (&Value::Date(_), &Schema::Date) => None, + (&Value::Decimal(_), &Schema::Decimal { .. }) => None, + (&Value::BigDecimal(_), &Schema::BigDecimal) => None, + (&Value::Duration(_), &Schema::Duration) => None, + (&Value::Uuid(_), &Schema::Uuid) => None, + (&Value::Float(_), &Schema::Float) => None, + (&Value::Float(_), &Schema::Double) => None, + (&Value::Double(_), &Schema::Double) => None, + (&Value::Bytes(_), &Schema::Bytes) => None, + (&Value::Bytes(_), &Schema::Decimal { .. }) => None, + (&Value::String(_), &Schema::String) => None, + (&Value::String(_), &Schema::Uuid) => None, + (&Value::Fixed(n, _), &Schema::Fixed(FixedSchema { size, .. })) => { + if n != size { + Some(format!( + "The value's size ({n}) is different than the schema's size ({size})" + )) + } else { + None + } } - } - // TODO: check precision against n - (&Value::Fixed(_n, _), &Schema::Decimal { .. }) => None, - (Value::String(s), Schema::Enum(EnumSchema { symbols, .. })) => { - if !symbols.contains(s) { - Some(format!("'{s}' is not a member of the possible symbols")) - } else { - None + (Value::Bytes(b), &Schema::Fixed(FixedSchema { size, .. })) => { + if b.len() != size { + Some(format!( + "The bytes' length ({}) is different than the schema's size ({})", + b.len(), + size + )) + } else { + None + } } - } - ( - &Value::Enum(i, ref s), - Schema::Enum(EnumSchema { - symbols, default, .. - }), - ) => symbols - .get(i as usize) - .map(|ref symbol| { - if symbol != &s { - Some(format!("Symbol '{s}' is not at position '{i}'")) + (&Value::Fixed(n, _), &Schema::Duration) => { + if n != 12 { + Some(format!( + "The value's size ('{n}') must be exactly 12 to be a Duration" + )) } else { None } - }) - .unwrap_or_else(|| match default { - Some(_) => None, - None => Some(format!("No symbol at position '{i}'")), - }), - // (&Value::Union(None), &Schema::Union(_)) => None, - (&Value::Union(i, ref value), Schema::Union(inner)) => inner - .variants() - .get(i as usize) - .map(|schema| value.validate_internal(schema, names, enclosing_namespace)) - .unwrap_or_else(|| Some(format!("No schema in the union at position '{i}'"))), - (v, Schema::Union(inner)) => { - match inner.find_schema_with_known_schemata(v, Some(names), enclosing_namespace) { - Some(_) => None, - None => Some("Could not find matching type in union".to_string()), } - } - (Value::Array(items), Schema::Array(inner)) => items.iter().fold(None, |acc, item| { - Value::accumulate( - acc, - item.validate_internal(&inner.items, names, enclosing_namespace), - ) - }), - (Value::Map(items), Schema::Map(inner)) => { - items.iter().fold(None, |acc, (_, value)| { - Value::accumulate( - acc, - value.validate_internal(&inner.types, names, enclosing_namespace), - ) - }) - } - ( - Value::Record(record_fields), - Schema::Record(RecordSchema { - fields, - lookup, - name, - .. - }), - ) => { - let non_nullable_fields_count = - fields.iter().filter(|&rf| !rf.is_nullable()).count(); - - // If the record contains fewer fields as required fields by the schema, it is invalid. - if record_fields.len() < non_nullable_fields_count { - return Some(format!( - "The value's records length ({}) doesn't match the schema ({} non-nullable fields)", - record_fields.len(), - non_nullable_fields_count - )); - } else if record_fields.len() > fields.len() { - return Some(format!( - "The value's records length ({}) is greater than the schema's ({} fields)", - record_fields.len(), - fields.len(), - )); + // TODO: check precision against n + (&Value::Fixed(_n, _), &Schema::Decimal { .. }) => None, + (Value::String(s), Schema::Enum(EnumSchema { symbols, .. })) => { + if !symbols.contains(s) { + Some(format!("'{s}' is not a member of the possible symbols")) + } else { + None + } } - - record_fields - .iter() - .fold(None, |acc, (field_name, record_field)| { - let record_namespace = if name.namespace.is_none() { - enclosing_namespace + ( + &Value::Enum(i, ref s), + Schema::Enum(EnumSchema { + symbols, default, .. + }), + ) => symbols + .get(i as usize) + .map(|ref symbol| { + if symbol != &s { + Some(format!("Symbol '{s}' is not at position '{i}'")) } else { - &name.namespace - }; - match lookup.get(field_name) { - Some(idx) => { - let field = &fields[*idx]; - Value::accumulate( - acc, - record_field.validate_internal( - &field.schema, - names, - record_namespace, - ), - ) - } - None => Value::accumulate( - acc, - Some(format!("There is no schema field for field '{field_name}'")), - ), + None } }) - } - (Value::Map(items), Schema::Record(RecordSchema { fields, .. })) => { - fields.iter().fold(None, |acc, field| { - if let Some(item) = items.get(&field.name) { - let res = item.validate_internal(&field.schema, names, enclosing_namespace); - Value::accumulate(acc, res) - } else if !field.is_nullable() { + .unwrap_or_else(|| match default { + Some(_) => None, + None => Some(format!("No symbol at position '{i}'")), + }), + // (&Value::Union(None), &Schema::Union(_)) => None, + (&Value::Union(i, ref value), Schema::Union(inner)) => inner + .variants() + .get(i as usize) + .map(|schema| value.validate_internal(schema, names, enclosing_namespace)) + .unwrap_or_else(|| Some(format!("No schema in the union at position '{i}'"))), + (v, Schema::Union(inner)) => { + match inner.find_schema_with_known_schemata(v, Some(names), enclosing_namespace) + { + Some(_) => None, + None => Some("Could not find matching type in union".to_string()), + } + } + (Value::Array(items), Schema::Array(inner)) => { + items.iter().fold(None, |acc, item| { Value::accumulate( acc, - Some(format!( - "Field with name '{:?}' is not a member of the map items", - field.name - )), + item.validate_internal(&inner.items, names, enclosing_namespace), ) - } else { - acc + }) + } + (Value::Map(items), Schema::Map(inner)) => { + items.iter().fold(None, |acc, (_, value)| { + Value::accumulate( + acc, + value.validate_internal(&inner.types, names, enclosing_namespace), + ) + }) + } + ( + Value::Record(record_fields), + Schema::Record(RecordSchema { + fields, + lookup, + name, + .. + }), + ) => { + let non_nullable_fields_count = + fields.iter().filter(|&rf| !rf.is_nullable()).count(); + + // If the record contains fewer fields as required fields by the schema, it is invalid. + if record_fields.len() < non_nullable_fields_count { + return Some(format!( + "The value's records length ({}) doesn't match the schema ({} non-nullable fields)", + record_fields.len(), + non_nullable_fields_count + )); + } else if record_fields.len() > fields.len() { + return Some(format!( + "The value's records length ({}) is greater than the schema's ({} fields)", + record_fields.len(), + fields.len(), + )); } - }) + + record_fields + .iter() + .fold(None, |acc, (field_name, record_field)| { + let record_namespace = if name.namespace.is_none() { + enclosing_namespace + } else { + &name.namespace + }; + match lookup.get(field_name) { + Some(idx) => { + let field = &fields[*idx]; + Value::accumulate( + acc, + record_field.validate_internal( + &field.schema, + names, + record_namespace, + ), + ) + } + None => Value::accumulate( + acc, + Some(format!( + "There is no schema field for field '{field_name}'" + )), + ), + } + }) + } + (Value::Map(items), Schema::Record(RecordSchema { fields, .. })) => { + fields.iter().fold(None, |acc, field| { + if let Some(item) = items.get(&field.name) { + let res = + item.validate_internal(&field.schema, names, enclosing_namespace); + Value::accumulate(acc, res) + } else if !field.is_nullable() { + Value::accumulate( + acc, + Some(format!( + "Field with name '{:?}' is not a member of the map items", + field.name + )), + ) + } else { + acc + } + }) + } + (v, s) => Some(format!( + "Unsupported value-schema combination! Value: {v:?}, schema: {s:?}" + )), } - (v, s) => Some(format!( - "Unsupported value-schema combination! Value: {v:?}, schema: {s:?}" - )), } - } - - /// Attempt to perform schema resolution on the value, with the given - /// [Schema](../schema/enum.Schema.html). - /// - /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) - /// in the Avro specification for the full set of rules of schema - /// resolution. - pub async fn resolve(self, schema: &Schema) -> AvroResult { - self.resolve_schemata(schema, Vec::with_capacity(0)).await - } - /// Attempt to perform schema resolution on the value, with the given - /// [Schema](../schema/enum.Schema.html) and set of schemas to use for Refs resolution. - /// - /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) - /// in the Avro specification for the full set of rules of schema - /// resolution. - pub async fn resolve_schemata(self, schema: &Schema, schemata: Vec<&Schema>) -> AvroResult { - let enclosing_namespace = schema.namespace(); - let rs = if schemata.is_empty() { - ResolvedSchema::try_from(schema)? - } else { - ResolvedSchema::try_from(schemata)? - }; - self.resolve_internal(schema, rs.get_names(), &enclosing_namespace, &None).await - } + /// Attempt to perform schema resolution on the value, with the given + /// [Schema](../schema/enum.Schema.html). + /// + /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) + /// in the Avro specification for the full set of rules of schema + /// resolution. + pub async fn resolve(self, schema: &Schema) -> AvroResult { + self.resolve_schemata(schema, Vec::with_capacity(0)).await + } - pub(crate) async fn resolve_internal + Debug>( - mut self, - schema: &Schema, - names: &HashMap, - enclosing_namespace: &Namespace, - field_default: &Option, - ) -> AvroResult { - // Check if this schema is a union, and if the reader schema is not. - if SchemaKind::from(&self) == SchemaKind::Union - && SchemaKind::from(schema) != SchemaKind::Union - { - // Pull out the Union, and attempt to resolve against it. - let v = match self { - Value::Union(_i, b) => *b, - _ => unreachable!(), + /// Attempt to perform schema resolution on the value, with the given + /// [Schema](../schema/enum.Schema.html) and set of schemas to use for Refs resolution. + /// + /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) + /// in the Avro specification for the full set of rules of schema + /// resolution. + pub async fn resolve_schemata( + self, + schema: &Schema, + schemata: Vec<&Schema>, + ) -> AvroResult { + let enclosing_namespace = schema.namespace(); + let rs = if schemata.is_empty() { + ResolvedSchema::try_from(schema)? + } else { + ResolvedSchema::try_from(schemata)? }; - self = v; + self.resolve_internal(schema, rs.get_names(), &enclosing_namespace, &None) + .await } - match *schema { - Schema::Ref { ref name } => { - let name = name.fully_qualified_name(enclosing_namespace); - if let Some(resolved) = names.get(&name) { - debug!("Resolved {name:?}"); - Box::pin(self.resolve_internal(resolved.borrow(), names, &name.namespace, field_default)).await - } else { - error!("Failed to resolve schema {name:?}"); - Err(Details::SchemaResolutionError(name.clone()).into()) - } - } - Schema::Null => self.resolve_null(), - Schema::Boolean => self.resolve_boolean(), - Schema::Int => self.resolve_int(), - Schema::Long => self.resolve_long(), - Schema::Float => self.resolve_float(), - Schema::Double => self.resolve_double(), - Schema::Bytes => self.resolve_bytes(), - Schema::String => self.resolve_string(), - Schema::Fixed(FixedSchema { size, .. }) => self.resolve_fixed(size), - Schema::Union(ref inner) => { - Box::pin(self.resolve_union(inner, names, enclosing_namespace, field_default)).await - } - Schema::Enum(EnumSchema { - ref symbols, - ref default, - .. - }) => self.resolve_enum(symbols, default, field_default), - Schema::Array(ref inner) => { - self.resolve_array(&inner.items, names, enclosing_namespace).await + pub(crate) async fn resolve_internal + Debug>( + mut self, + schema: &Schema, + names: &HashMap, + enclosing_namespace: &Namespace, + field_default: &Option, + ) -> AvroResult { + // Check if this schema is a union, and if the reader schema is not. + if SchemaKind::from(&self) == SchemaKind::Union + && SchemaKind::from(schema) != SchemaKind::Union + { + // Pull out the Union, and attempt to resolve against it. + let v = match self { + Value::Union(_i, b) => *b, + _ => unreachable!(), + }; + self = v; } - Schema::Map(ref inner) => self.resolve_map(&inner.types, names, enclosing_namespace), - Schema::Record(RecordSchema { ref fields, .. }) => { - self.resolve_record(fields, names, enclosing_namespace).await + match *schema { + Schema::Ref { ref name } => { + let name = name.fully_qualified_name(enclosing_namespace); + + if let Some(resolved) = names.get(&name) { + debug!("Resolved {name:?}"); + Box::pin(self.resolve_internal( + resolved.borrow(), + names, + &name.namespace, + field_default, + )) + .await + } else { + error!("Failed to resolve schema {name:?}"); + Err(Details::SchemaResolutionError(name.clone()).into()) + } + } + Schema::Null => self.resolve_null(), + Schema::Boolean => self.resolve_boolean(), + Schema::Int => self.resolve_int(), + Schema::Long => self.resolve_long(), + Schema::Float => self.resolve_float(), + Schema::Double => self.resolve_double(), + Schema::Bytes => self.resolve_bytes().await, + Schema::String => self.resolve_string(), + Schema::Fixed(FixedSchema { size, .. }) => self.resolve_fixed(size), + Schema::Union(ref inner) => { + Box::pin(self.resolve_union(inner, names, enclosing_namespace, field_default)) + .await + } + Schema::Enum(EnumSchema { + ref symbols, + ref default, + .. + }) => self.resolve_enum(symbols, default, field_default), + Schema::Array(ref inner) => { + self.resolve_array(&inner.items, names, enclosing_namespace) + .await + } + Schema::Map(ref inner) => { + self.resolve_map(&inner.types, names, enclosing_namespace) + } + Schema::Record(RecordSchema { ref fields, .. }) => { + self.resolve_record(fields, names, enclosing_namespace) + .await + } + Schema::Decimal(DecimalSchema { + scale, + precision, + ref inner, + }) => self.resolve_decimal(precision, scale, inner), + Schema::BigDecimal => self.resolve_bigdecimal().await, + Schema::Date => self.resolve_date(), + Schema::TimeMillis => self.resolve_time_millis(), + Schema::TimeMicros => self.resolve_time_micros(), + Schema::TimestampMillis => self.resolve_timestamp_millis(), + Schema::TimestampMicros => self.resolve_timestamp_micros(), + Schema::TimestampNanos => self.resolve_timestamp_nanos(), + Schema::LocalTimestampMillis => self.resolve_local_timestamp_millis(), + Schema::LocalTimestampMicros => self.resolve_local_timestamp_micros(), + Schema::LocalTimestampNanos => self.resolve_local_timestamp_nanos(), + Schema::Duration => self.resolve_duration(), + Schema::Uuid => self.resolve_uuid(), } - Schema::Decimal(DecimalSchema { - scale, - precision, - ref inner, - }) => self.resolve_decimal(precision, scale, inner), - Schema::BigDecimal => self.resolve_bigdecimal().await, - Schema::Date => self.resolve_date(), - Schema::TimeMillis => self.resolve_time_millis(), - Schema::TimeMicros => self.resolve_time_micros(), - Schema::TimestampMillis => self.resolve_timestamp_millis(), - Schema::TimestampMicros => self.resolve_timestamp_micros(), - Schema::TimestampNanos => self.resolve_timestamp_nanos(), - Schema::LocalTimestampMillis => self.resolve_local_timestamp_millis(), - Schema::LocalTimestampMicros => self.resolve_local_timestamp_micros(), - Schema::LocalTimestampNanos => self.resolve_local_timestamp_nanos(), - Schema::Duration => self.resolve_duration(), - Schema::Uuid => self.resolve_uuid(), } - } - fn resolve_uuid(self) -> Result { - Ok(match self { - uuid @ Value::Uuid(_) => uuid, - Value::String(ref string) => { - Value::Uuid(Uuid::from_str(string).map_err(Details::ConvertStrToUuid)?) - } - other => return Err(Details::GetUuid(other).into()), - }) - } + fn resolve_uuid(self) -> Result { + Ok(match self { + uuid @ Value::Uuid(_) => uuid, + Value::String(ref string) => { + Value::Uuid(Uuid::from_str(string).map_err(Details::ConvertStrToUuid)?) + } + other => return Err(Details::GetUuid(other).into()), + }) + } - async fn resolve_bigdecimal(self) -> Result { - Ok(match self { - bg @ Value::BigDecimal(_) => bg, - Value::Bytes(b) => Value::BigDecimal(deserialize_big_decimal(&b).await.unwrap()), - other => return Err(Details::GetBigDecimal(other).into()), - }) - } + async fn resolve_bigdecimal(self) -> Result { + Ok(match self { + bg @ Value::BigDecimal(_) => bg, + Value::Bytes(b) => Value::BigDecimal(deserialize_big_decimal(&b).await.unwrap()), + other => return Err(Details::GetBigDecimal(other).into()), + }) + } - fn resolve_duration(self) -> Result { - Ok(match self { - duration @ Value::Duration { .. } => duration, - Value::Fixed(size, bytes) => { - if size != 12 { - return Err(Details::GetDecimalFixedBytes(size).into()); + fn resolve_duration(self) -> Result { + Ok(match self { + duration @ Value::Duration { .. } => duration, + Value::Fixed(size, bytes) => { + if size != 12 { + return Err(Details::GetDecimalFixedBytes(size).into()); + } + Value::Duration(Duration::from([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], + bytes[7], bytes[8], bytes[9], bytes[10], bytes[11], + ])) } - Value::Duration(Duration::from([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - bytes[8], bytes[9], bytes[10], bytes[11], - ])) - } - other => return Err(Details::ResolveDuration(other).into()), - }) - } + other => return Err(Details::ResolveDuration(other).into()), + }) + } - fn resolve_decimal( - self, - precision: Precision, - scale: Scale, - inner: &Schema, - ) -> Result { - if scale > precision { - return Err(Details::GetScaleAndPrecision { scale, precision }.into()); - } - match inner { - &Schema::Fixed(FixedSchema { size, .. }) => { - if max_prec_for_len(size)? < precision { - return Err(Details::GetScaleWithFixedSize { size, precision }.into()); - } + fn resolve_decimal( + self, + precision: Precision, + scale: Scale, + inner: &Schema, + ) -> Result { + if scale > precision { + return Err(Details::GetScaleAndPrecision { scale, precision }.into()); } - Schema::Bytes => (), - _ => return Err(Details::ResolveDecimalSchema(inner.into()).into()), - }; - match self { - Value::Decimal(num) => { - let num_bytes = num.len(); - if max_prec_for_len(num_bytes)? < precision { - Err(Details::ComparePrecisionAndSize { - precision, - num_bytes, + match inner { + &Schema::Fixed(FixedSchema { size, .. }) => { + if max_prec_for_len(size)? < precision { + return Err(Details::GetScaleWithFixedSize { size, precision }.into()); } - .into()) - } else { - Ok(Value::Decimal(num)) } - // check num.bits() here - } - Value::Fixed(_, bytes) | Value::Bytes(bytes) => { - if max_prec_for_len(bytes.len())? < precision { - Err(Details::ComparePrecisionAndSize { - precision, - num_bytes: bytes.len(), + Schema::Bytes => (), + _ => return Err(Details::ResolveDecimalSchema(inner.into()).into()), + }; + match self { + Value::Decimal(num) => { + let num_bytes = num.len(); + if max_prec_for_len(num_bytes)? < precision { + Err(Details::ComparePrecisionAndSize { + precision, + num_bytes, + } + .into()) + } else { + Ok(Value::Decimal(num)) + } + // check num.bits() here + } + Value::Fixed(_, bytes) | Value::Bytes(bytes) => { + if max_prec_for_len(bytes.len())? < precision { + Err(Details::ComparePrecisionAndSize { + precision, + num_bytes: bytes.len(), + } + .into()) + } else { + // precision and scale match, can we assume the underlying type can hold the data? + Ok(Value::Decimal(Decimal::from(bytes))) } - .into()) - } else { - // precision and scale match, can we assume the underlying type can hold the data? - Ok(Value::Decimal(Decimal::from(bytes))) } + other => Err(Details::ResolveDecimal(other).into()), } - other => Err(Details::ResolveDecimal(other).into()), } - } - fn resolve_date(self) -> Result { - match self { - Value::Date(d) | Value::Int(d) => Ok(Value::Date(d)), - other => Err(Details::GetDate(other).into()), + fn resolve_date(self) -> Result { + match self { + Value::Date(d) | Value::Int(d) => Ok(Value::Date(d)), + other => Err(Details::GetDate(other).into()), + } } - } - fn resolve_time_millis(self) -> Result { - match self { - Value::TimeMillis(t) | Value::Int(t) => Ok(Value::TimeMillis(t)), - other => Err(Details::GetTimeMillis(other).into()), + fn resolve_time_millis(self) -> Result { + match self { + Value::TimeMillis(t) | Value::Int(t) => Ok(Value::TimeMillis(t)), + other => Err(Details::GetTimeMillis(other).into()), + } } - } - fn resolve_time_micros(self) -> Result { - match self { - Value::TimeMicros(t) | Value::Long(t) => Ok(Value::TimeMicros(t)), - Value::Int(t) => Ok(Value::TimeMicros(i64::from(t))), - other => Err(Details::GetTimeMicros(other).into()), + fn resolve_time_micros(self) -> Result { + match self { + Value::TimeMicros(t) | Value::Long(t) => Ok(Value::TimeMicros(t)), + Value::Int(t) => Ok(Value::TimeMicros(i64::from(t))), + other => Err(Details::GetTimeMicros(other).into()), + } } - } - fn resolve_timestamp_millis(self) -> Result { - match self { - Value::TimestampMillis(ts) | Value::Long(ts) => Ok(Value::TimestampMillis(ts)), - Value::Int(ts) => Ok(Value::TimestampMillis(i64::from(ts))), - other => Err(Details::GetTimestampMillis(other).into()), + fn resolve_timestamp_millis(self) -> Result { + match self { + Value::TimestampMillis(ts) | Value::Long(ts) => Ok(Value::TimestampMillis(ts)), + Value::Int(ts) => Ok(Value::TimestampMillis(i64::from(ts))), + other => Err(Details::GetTimestampMillis(other).into()), + } } - } - fn resolve_timestamp_micros(self) -> Result { - match self { - Value::TimestampMicros(ts) | Value::Long(ts) => Ok(Value::TimestampMicros(ts)), - Value::Int(ts) => Ok(Value::TimestampMicros(i64::from(ts))), - other => Err(Details::GetTimestampMicros(other).into()), + fn resolve_timestamp_micros(self) -> Result { + match self { + Value::TimestampMicros(ts) | Value::Long(ts) => Ok(Value::TimestampMicros(ts)), + Value::Int(ts) => Ok(Value::TimestampMicros(i64::from(ts))), + other => Err(Details::GetTimestampMicros(other).into()), + } } - } - fn resolve_timestamp_nanos(self) -> Result { - match self { - Value::TimestampNanos(ts) | Value::Long(ts) => Ok(Value::TimestampNanos(ts)), - Value::Int(ts) => Ok(Value::TimestampNanos(i64::from(ts))), - other => Err(Details::GetTimestampNanos(other).into()), + fn resolve_timestamp_nanos(self) -> Result { + match self { + Value::TimestampNanos(ts) | Value::Long(ts) => Ok(Value::TimestampNanos(ts)), + Value::Int(ts) => Ok(Value::TimestampNanos(i64::from(ts))), + other => Err(Details::GetTimestampNanos(other).into()), + } } - } - fn resolve_local_timestamp_millis(self) -> Result { - match self { - Value::LocalTimestampMillis(ts) | Value::Long(ts) => { - Ok(Value::LocalTimestampMillis(ts)) + fn resolve_local_timestamp_millis(self) -> Result { + match self { + Value::LocalTimestampMillis(ts) | Value::Long(ts) => { + Ok(Value::LocalTimestampMillis(ts)) + } + Value::Int(ts) => Ok(Value::LocalTimestampMillis(i64::from(ts))), + other => Err(Details::GetLocalTimestampMillis(other).into()), } - Value::Int(ts) => Ok(Value::LocalTimestampMillis(i64::from(ts))), - other => Err(Details::GetLocalTimestampMillis(other).into()), } - } - fn resolve_local_timestamp_micros(self) -> Result { - match self { - Value::LocalTimestampMicros(ts) | Value::Long(ts) => { - Ok(Value::LocalTimestampMicros(ts)) + fn resolve_local_timestamp_micros(self) -> Result { + match self { + Value::LocalTimestampMicros(ts) | Value::Long(ts) => { + Ok(Value::LocalTimestampMicros(ts)) + } + Value::Int(ts) => Ok(Value::LocalTimestampMicros(i64::from(ts))), + other => Err(Details::GetLocalTimestampMicros(other).into()), } - Value::Int(ts) => Ok(Value::LocalTimestampMicros(i64::from(ts))), - other => Err(Details::GetLocalTimestampMicros(other).into()), } - } - fn resolve_local_timestamp_nanos(self) -> Result { - match self { - Value::LocalTimestampNanos(ts) | Value::Long(ts) => Ok(Value::LocalTimestampNanos(ts)), - Value::Int(ts) => Ok(Value::LocalTimestampNanos(i64::from(ts))), - other => Err(Details::GetLocalTimestampNanos(other).into()), + fn resolve_local_timestamp_nanos(self) -> Result { + match self { + Value::LocalTimestampNanos(ts) | Value::Long(ts) => { + Ok(Value::LocalTimestampNanos(ts)) + } + Value::Int(ts) => Ok(Value::LocalTimestampNanos(i64::from(ts))), + other => Err(Details::GetLocalTimestampNanos(other).into()), + } } - } - fn resolve_null(self) -> Result { - match self { - Value::Null => Ok(Value::Null), - other => Err(Details::GetNull(other).into()), + fn resolve_null(self) -> Result { + match self { + Value::Null => Ok(Value::Null), + other => Err(Details::GetNull(other).into()), + } } - } - fn resolve_boolean(self) -> Result { - match self { - Value::Boolean(b) => Ok(Value::Boolean(b)), - other => Err(Details::GetBoolean(other).into()), + fn resolve_boolean(self) -> Result { + match self { + Value::Boolean(b) => Ok(Value::Boolean(b)), + other => Err(Details::GetBoolean(other).into()), + } } - } - fn resolve_int(self) -> Result { - match self { - Value::Int(n) => Ok(Value::Int(n)), - Value::Long(n) => Ok(Value::Int(n as i32)), - other => Err(Details::GetInt(other).into()), + fn resolve_int(self) -> Result { + match self { + Value::Int(n) => Ok(Value::Int(n)), + Value::Long(n) => Ok(Value::Int(n as i32)), + other => Err(Details::GetInt(other).into()), + } } - } - fn resolve_long(self) -> Result { - match self { - Value::Int(n) => Ok(Value::Long(i64::from(n))), - Value::Long(n) => Ok(Value::Long(n)), - other => Err(Details::GetLong(other).into()), + fn resolve_long(self) -> Result { + match self { + Value::Int(n) => Ok(Value::Long(i64::from(n))), + Value::Long(n) => Ok(Value::Long(n)), + other => Err(Details::GetLong(other).into()), + } } - } - fn resolve_float(self) -> Result { - match self { - Value::Int(n) => Ok(Value::Float(n as f32)), - Value::Long(n) => Ok(Value::Float(n as f32)), - Value::Float(x) => Ok(Value::Float(x)), - Value::Double(x) => Ok(Value::Float(x as f32)), - Value::String(ref x) => match Self::parse_special_float(x) { - Some(f) => Ok(Value::Float(f)), - None => Err(Details::GetFloat(self).into()), - }, - other => Err(Details::GetFloat(other).into()), + fn resolve_float(self) -> Result { + match self { + Value::Int(n) => Ok(Value::Float(n as f32)), + Value::Long(n) => Ok(Value::Float(n as f32)), + Value::Float(x) => Ok(Value::Float(x)), + Value::Double(x) => Ok(Value::Float(x as f32)), + Value::String(ref x) => match Self::parse_special_float(x) { + Some(f) => Ok(Value::Float(f)), + None => Err(Details::GetFloat(self).into()), + }, + other => Err(Details::GetFloat(other).into()), + } } - } - fn resolve_double(self) -> Result { - match self { - Value::Int(n) => Ok(Value::Double(f64::from(n))), - Value::Long(n) => Ok(Value::Double(n as f64)), - Value::Float(x) => Ok(Value::Double(f64::from(x))), - Value::Double(x) => Ok(Value::Double(x)), - Value::String(ref x) => match Self::parse_special_float(x) { - Some(f) => Ok(Value::Double(f64::from(f))), - None => Err(Details::GetDouble(self).into()), - }, - other => Err(Details::GetDouble(other).into()), + fn resolve_double(self) -> Result { + match self { + Value::Int(n) => Ok(Value::Double(f64::from(n))), + Value::Long(n) => Ok(Value::Double(n as f64)), + Value::Float(x) => Ok(Value::Double(f64::from(x))), + Value::Double(x) => Ok(Value::Double(x)), + Value::String(ref x) => match Self::parse_special_float(x) { + Some(f) => Ok(Value::Double(f64::from(f))), + None => Err(Details::GetDouble(self).into()), + }, + other => Err(Details::GetDouble(other).into()), + } } - } - /// IEEE 754 NaN and infinities are not valid JSON numbers. - /// So they are represented in JSON as strings. - fn parse_special_float(value: &str) -> Option { - match value { - "NaN" => Some(f32::NAN), - "INF" | "Infinity" => Some(f32::INFINITY), - "-INF" | "-Infinity" => Some(f32::NEG_INFINITY), - _ => None, + /// IEEE 754 NaN and infinities are not valid JSON numbers. + /// So they are represented in JSON as strings. + fn parse_special_float(value: &str) -> Option { + match value { + "NaN" => Some(f32::NAN), + "INF" | "Infinity" => Some(f32::INFINITY), + "-INF" | "-Infinity" => Some(f32::NEG_INFINITY), + _ => None, + } } - } - fn resolve_bytes(self) -> Result { - match self { - Value::Bytes(bytes) => Ok(Value::Bytes(bytes)), - Value::String(s) => Ok(Value::Bytes(s.into_bytes())), - Value::Array(items) => Ok(Value::Bytes( - items - .into_iter() - .map(Value::try_u8) - .collect::, _>>()?, - )), - other => Err(Details::GetBytes(other).into()), + async fn resolve_bytes(self) -> Result { + match self { + Value::Bytes(bytes) => Ok(Value::Bytes(bytes)), + Value::String(s) => Ok(Value::Bytes(s.into_bytes())), + Value::Array(items) => { + let mut us = Vec::with_capacity(items.len()); + for i in items.into_iter() { + let u = Box::pin(Value::try_u8(i)).await?; + us.push(u); + } + Ok(Value::Bytes(us)) + } + other => Err(Details::GetBytes(other).into()), + } } - } - fn resolve_string(self) -> Result { - match self { - Value::String(s) => Ok(Value::String(s)), - Value::Bytes(bytes) | Value::Fixed(_, bytes) => Ok(Value::String( - String::from_utf8(bytes).map_err(Details::ConvertToUtf8)?, - )), - other => Err(Details::GetString(other).into()), + fn resolve_string(self) -> Result { + match self { + Value::String(s) => Ok(Value::String(s)), + Value::Bytes(bytes) | Value::Fixed(_, bytes) => Ok(Value::String( + String::from_utf8(bytes).map_err(Details::ConvertToUtf8)?, + )), + other => Err(Details::GetString(other).into()), + } } - } - fn resolve_fixed(self, size: usize) -> Result { - match self { - Value::Fixed(n, bytes) => { - if n == size { - Ok(Value::Fixed(n, bytes)) - } else { - Err(Details::CompareFixedSizes { size, n }.into()) + fn resolve_fixed(self, size: usize) -> Result { + match self { + Value::Fixed(n, bytes) => { + if n == size { + Ok(Value::Fixed(n, bytes)) + } else { + Err(Details::CompareFixedSizes { size, n }.into()) + } } - } - Value::String(s) => Ok(Value::Fixed(s.len(), s.into_bytes())), - Value::Bytes(s) => { - if s.len() == size { - Ok(Value::Fixed(size, s)) - } else { - Err(Details::CompareFixedSizes { size, n: s.len() }.into()) + Value::String(s) => Ok(Value::Fixed(s.len(), s.into_bytes())), + Value::Bytes(s) => { + if s.len() == size { + Ok(Value::Fixed(size, s)) + } else { + Err(Details::CompareFixedSizes { size, n: s.len() }.into()) + } } + other => Err(Details::GetStringForFixed(other).into()), } - other => Err(Details::GetStringForFixed(other).into()), } - } - pub(crate) fn resolve_enum( - self, - symbols: &[String], - enum_default: &Option, - _field_default: &Option, - ) -> Result { - let validate_symbol = |symbol: String, symbols: &[String]| { - if let Some(index) = symbols.iter().position(|item| item == &symbol) { - Ok(Value::Enum(index as u32, symbol)) - } else { - match enum_default { - Some(default) => { - if let Some(index) = symbols.iter().position(|item| item == default) { - Ok(Value::Enum(index as u32, default.clone())) - } else { - Err(Details::GetEnumDefault { - symbol, - symbols: symbols.into(), + pub(crate) fn resolve_enum( + self, + symbols: &[String], + enum_default: &Option, + _field_default: &Option, + ) -> Result { + let validate_symbol = |symbol: String, symbols: &[String]| { + if let Some(index) = symbols.iter().position(|item| item == &symbol) { + Ok(Value::Enum(index as u32, symbol)) + } else { + match enum_default { + Some(default) => { + if let Some(index) = symbols.iter().position(|item| item == default) { + Ok(Value::Enum(index as u32, default.clone())) + } else { + Err(Details::GetEnumDefault { + symbol, + symbols: symbols.into(), + } + .into()) } - .into()) } + _ => Err(Details::GetEnumDefault { + symbol, + symbols: symbols.into(), + } + .into()), } - _ => Err(Details::GetEnumDefault { - symbol, - symbols: symbols.into(), - } - .into()), } - } - }; + }; - match self { - Value::Enum(_raw_index, s) => validate_symbol(s, symbols), - Value::String(s) => validate_symbol(s, symbols), - other => Err(Details::GetEnum(other).into()), + match self { + Value::Enum(_raw_index, s) => validate_symbol(s, symbols), + Value::String(s) => validate_symbol(s, symbols), + other => Err(Details::GetEnum(other).into()), + } } - } - async fn resolve_union + Debug>( - self, - schema: &UnionSchema, - names: &HashMap, - enclosing_namespace: &Namespace, - field_default: &Option, - ) -> Result { - let v = match self { - // Both are unions case. - Value::Union(_i, v) => *v, - // Reader is a union, but writer is not. - v => v, - }; - let (i, inner) = schema - .find_schema_with_known_schemata(&v, Some(names), enclosing_namespace) - .ok_or_else(|| Details::FindUnionVariant { - schema: schema.clone(), - value: v.clone(), - })?; - - Ok(Value::Union( - i as u32, - Box::new(v.resolve_internal(inner, names, enclosing_namespace, field_default).await?), - )) - } + async fn resolve_union + Debug>( + self, + schema: &UnionSchema, + names: &HashMap, + enclosing_namespace: &Namespace, + field_default: &Option, + ) -> Result { + let v = match self { + // Both are unions case. + Value::Union(_i, v) => *v, + // Reader is a union, but writer is not. + v => v, + }; + let (i, inner) = schema + .find_schema_with_known_schemata(&v, Some(names), enclosing_namespace) + .await + .ok_or_else(|| Details::FindUnionVariant { + schema: schema.clone(), + value: v.clone(), + })?; + + Ok(Value::Union( + i as u32, + Box::new( + v.resolve_internal(inner, names, enclosing_namespace, field_default) + .await?, + ), + )) + } - async fn resolve_array + Debug>( - self, - schema: &Schema, - names: &HashMap, - enclosing_namespace: &Namespace, - ) -> Result { - match self { - Value::Array(items) => Ok(Value::Array( - items - .into_iter() - .map(async |item| item.resolve_internal(schema, names, enclosing_namespace, &None).await) - .collect::>()? - )), - other => Err(Details::GetArray { - expected: schema.into(), - other, + async fn resolve_array + Debug>( + self, + schema: &Schema, + names: &HashMap, + enclosing_namespace: &Namespace, + ) -> Result { + match self { + Value::Array(items) => Ok(Value::Array( + try_join_all(items.into_iter().map(|item| { + item.resolve_internal(schema, names, enclosing_namespace, &None) + })) + .await?, + )), + other => Err(Details::GetArray { + expected: schema.into(), + other, + } + .into()), } - .into()), } - } - fn resolve_map + Debug>( - self, - schema: &Schema, - names: &HashMap, - enclosing_namespace: &Namespace, - ) -> Result { - match self { - Value::Map(items) => Ok(Value::Map( - items - .into_iter() - .map(async |(key, value)| { - value - .resolve_internal(schema, names, enclosing_namespace, &None).await - .map(|value| (key, value)) - }) - .collect::>()?, - )), - other => Err(Details::GetMap { - expected: schema.into(), - other, + fn resolve_map + Debug>( + self, + schema: &Schema, + names: &HashMap, + enclosing_namespace: &Namespace, + ) -> Result { + match self { + Value::Map(items) => Ok(Value::Map( + items + .into_iter() + .map(async |(key, value)| { + value + .resolve_internal(schema, names, enclosing_namespace, &None) + .await + .map(|value| (key, value)) + }) + .collect::>()?, + )), + other => Err(Details::GetMap { + expected: schema.into(), + other, + } + .into()), } - .into()), } - } - - async fn resolve_record + Debug>( - self, - fields: &[RecordField], - names: &HashMap, - enclosing_namespace: &Namespace, - ) -> Result { - let mut items = match self { - Value::Map(items) => Ok(items), - Value::Record(fields) => Ok(fields.into_iter().collect::>()), - other => Err(Error::new(Details::GetRecord { - expected: fields - .iter() - .map(|field| (field.name.clone(), field.schema.clone().into())) - .collect(), - other, - })), - }?; - let new_fields = fields - .iter() - .map(async |field| { + async fn resolve_record + Debug>( + self, + fields: &[RecordField], + names: &HashMap, + enclosing_namespace: &Namespace, + ) -> Result { + let mut items = match self { + Value::Map(items) => Ok(items), + Value::Record(fields) => Ok(fields.into_iter().collect::>()), + other => Err(Error::new(Details::GetRecord { + expected: fields + .iter() + .map(|field| (field.name.clone(), field.schema.clone().into())) + .collect(), + other, + })), + }?; + + let new_fields = try_join_all(fields.iter().map(|field| async { let value = match items.remove(&field.name) { Some(value) => value, None => match field.default { @@ -1131,12 +1188,16 @@ impl Value { Schema::Null => Value::Union(0, Box::new(Value::Null)), _ => Value::Union( 0, - Box::new(Value::from(value.clone()).resolve_internal( - first, - names, - enclosing_namespace, - &field.default, - ).await?), + Box::new( + Value::from(value.clone()) + .resolve_internal( + first, + names, + enclosing_namespace, + &field.default, + ) + .await?, + ), ), } } @@ -1148,46 +1209,47 @@ impl Value { }, }; value - .resolve_internal(&field.schema, names, enclosing_namespace, &field.default).await + .resolve_internal(&field.schema, names, enclosing_namespace, &field.default) + .await .map(|value| (field.name.clone(), value)) - }) - .collect::, _>>()?; + })) + .await?; - Ok(Value::Record(new_fields)) - } + Ok(Value::Record(new_fields)) + } - async fn try_u8(self) -> AvroResult { - let int = self.resolve(&Schema::Int).await?; - if let Value::Int(n) = int { - if n >= 0 && n <= i32::from(u8::MAX) { - return Ok(n as u8); + async fn try_u8(self) -> AvroResult { + let int = self.resolve(&Schema::Int).await?; + if let Value::Int(n) = int { + if n >= 0 && n <= i32::from(u8::MAX) { + return Ok(n as u8); + } } - } - Err(Details::GetU8(int).into()) + Err(Details::GetU8(int).into()) + } } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - duration::{Days, Millis, Months}, - error::Details, - schema::RecordFieldOrder, - }; - use apache_avro_test_helper::{ - TestResult, - logger::{assert_logged, assert_not_logged}, - }; - use num_bigint::BigInt; - use pretty_assertions::assert_eq; - use serde_json::json; - - #[tokio::test] - async fn avro_3809_validate_nested_records_with_implicit_namespace() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[cfg(test)] + mod tests { + use super::*; + use crate::{ + duration::{Days, Millis, Months}, + error::Details, + schema::RecordFieldOrder, + }; + use apache_avro_test_helper::{ + TestResult, + logger::{assert_logged, assert_not_logged}, + }; + use num_bigint::BigInt; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[tokio::test] + async fn avro_3809_validate_nested_records_with_implicit_namespace() -> TestResult { + let schema = Schema::parse_str( + r#"{ "name": "record_name", "namespace": "space", "type": "record", @@ -1215,677 +1277,713 @@ mod tests { } ] }"#, - )?; - let value = Value::Record(vec![( - "outer_field_1".into(), - Value::Record(vec![ + )?; + let value = Value::Record(vec![( + "outer_field_1".into(), + Value::Record(vec![ + ( + "middle_field_1".into(), + Value::Record(vec![("inner_field_1".into(), Value::Double(1.2f64))]), + ), + ( + "middle_field_2".into(), + Value::Record(vec![("inner_field_1".into(), Value::Double(1.6f64))]), + ), + ]), + )]); + + assert!(value.validate(&schema)); + Ok(()) + } + + #[tokio::test] + async fn validate() -> TestResult { + let value_schema_valid = vec![ + (Value::Int(42), Schema::Int, true, ""), + (Value::Int(43), Schema::Long, true, ""), + (Value::Float(43.2), Schema::Float, true, ""), + (Value::Float(45.9), Schema::Double, true, ""), ( - "middle_field_1".into(), - Value::Record(vec![("inner_field_1".into(), Value::Double(1.2f64))]), + Value::Int(42), + Schema::Boolean, + false, + "Invalid value: Int(42) for schema: Boolean. Reason: Unsupported value-schema combination! Value: Int(42), schema: Boolean", ), ( - "middle_field_2".into(), - Value::Record(vec![("inner_field_1".into(), Value::Double(1.6f64))]), + Value::Union(0, Box::new(Value::Null)), + Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), + true, + "", ), - ]), - )]); - - assert!(value.validate(&schema)); - Ok(()) - } - - #[tokio::test] - async fn validate() -> TestResult { - let value_schema_valid = vec![ - (Value::Int(42), Schema::Int, true, ""), - (Value::Int(43), Schema::Long, true, ""), - (Value::Float(43.2), Schema::Float, true, ""), - (Value::Float(45.9), Schema::Double, true, ""), - ( - Value::Int(42), - Schema::Boolean, - false, - "Invalid value: Int(42) for schema: Boolean. Reason: Unsupported value-schema combination! Value: Int(42), schema: Boolean", - ), - ( - Value::Union(0, Box::new(Value::Null)), - Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), - true, - "", - ), - ( - Value::Union(1, Box::new(Value::Int(42))), - Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), - true, - "", - ), - ( - Value::Union(0, Box::new(Value::Null)), - Schema::Union(UnionSchema::new(vec![Schema::Double, Schema::Int])?), - false, - "Invalid value: Union(0, Null) for schema: Union(UnionSchema { schemas: [Double, Int], variant_index: {Int: 1, Double: 0} }). Reason: Unsupported value-schema combination! Value: Null, schema: Double", - ), - ( - Value::Union(3, Box::new(Value::Int(42))), - Schema::Union(UnionSchema::new(vec![ - Schema::Null, - Schema::Double, - Schema::String, - Schema::Int, - ])?), - true, - "", - ), - ( - Value::Union(1, Box::new(Value::Long(42i64))), - Schema::Union(UnionSchema::new(vec![ + ( + Value::Union(1, Box::new(Value::Int(42))), + Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), + true, + "", + ), + ( + Value::Union(0, Box::new(Value::Null)), + Schema::Union(UnionSchema::new(vec![Schema::Double, Schema::Int])?), + false, + "Invalid value: Union(0, Null) for schema: Union(UnionSchema { schemas: [Double, Int], variant_index: {Int: 1, Double: 0} }). Reason: Unsupported value-schema combination! Value: Null, schema: Double", + ), + ( + Value::Union(3, Box::new(Value::Int(42))), + Schema::Union(UnionSchema::new(vec![ + Schema::Null, + Schema::Double, + Schema::String, + Schema::Int, + ])?), + true, + "", + ), + ( + Value::Union(1, Box::new(Value::Long(42i64))), + Schema::Union(UnionSchema::new(vec![ + Schema::Null, + Schema::TimestampMillis, + ])?), + true, + "", + ), + ( + Value::Union(2, Box::new(Value::Long(1_i64))), + Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), + false, + "Invalid value: Union(2, Long(1)) for schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }). Reason: No schema in the union at position '2'", + ), + ( + Value::Array(vec![Value::Long(42i64)]), + Schema::array(Schema::Long), + true, + "", + ), + ( + Value::Array(vec![Value::Boolean(true)]), + Schema::array(Schema::Long), + false, + "Invalid value: Array([Boolean(true)]) for schema: Array(ArraySchema { items: Long, attributes: {} }). Reason: Unsupported value-schema combination! Value: Boolean(true), schema: Long", + ), + ( + Value::Record(vec![]), Schema::Null, - Schema::TimestampMillis, - ])?), - true, - "", - ), - ( - Value::Union(2, Box::new(Value::Long(1_i64))), - Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), - false, - "Invalid value: Union(2, Long(1)) for schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }). Reason: No schema in the union at position '2'", - ), - ( - Value::Array(vec![Value::Long(42i64)]), - Schema::array(Schema::Long), - true, - "", - ), - ( - Value::Array(vec![Value::Boolean(true)]), - Schema::array(Schema::Long), - false, - "Invalid value: Array([Boolean(true)]) for schema: Array(ArraySchema { items: Long, attributes: {} }). Reason: Unsupported value-schema combination! Value: Boolean(true), schema: Long", - ), - ( - Value::Record(vec![]), - Schema::Null, - false, - "Invalid value: Record([]) for schema: Null. Reason: Unsupported value-schema combination! Value: Record([]), schema: Null", - ), - ( - Value::Fixed(12, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), - Schema::Duration, - true, - "", - ), - ( - Value::Fixed(11, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), - Schema::Duration, - false, - "Invalid value: Fixed(11, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) for schema: Duration. Reason: The value's size ('11') must be exactly 12 to be a Duration", - ), - ( - Value::Record(vec![("unknown_field_name".to_string(), Value::Null)]), - Schema::Record(RecordSchema { - name: Name::new("record_name").unwrap(), - aliases: None, - doc: None, - fields: vec![RecordField { - name: "field_name".to_string(), - doc: None, - default: None, + false, + "Invalid value: Record([]) for schema: Null. Reason: Unsupported value-schema combination! Value: Record([]), schema: Null", + ), + ( + Value::Fixed(12, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), + Schema::Duration, + true, + "", + ), + ( + Value::Fixed(11, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + Schema::Duration, + false, + "Invalid value: Fixed(11, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) for schema: Duration. Reason: The value's size ('11') must be exactly 12 to be a Duration", + ), + ( + Value::Record(vec![("unknown_field_name".to_string(), Value::Null)]), + Schema::Record(RecordSchema { + name: Name::new("record_name").unwrap(), aliases: None, - schema: Schema::Int, - order: RecordFieldOrder::Ignore, - position: 0, - custom_attributes: Default::default(), - }], - lookup: Default::default(), - attributes: Default::default(), - }), - false, - r#"Invalid value: Record([("unknown_field_name", Null)]) for schema: Record(RecordSchema { name: Name { name: "record_name", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "field_name", doc: None, aliases: None, default: None, schema: Int, order: Ignore, position: 0, custom_attributes: {} }], lookup: {}, attributes: {} }). Reason: There is no schema field for field 'unknown_field_name'"#, - ), - ( - Value::Record(vec![("field_name".to_string(), Value::Null)]), - Schema::Record(RecordSchema { - name: Name::new("record_name").unwrap(), - aliases: None, - doc: None, - fields: vec![RecordField { - name: "field_name".to_string(), doc: None, - default: None, + fields: vec![RecordField { + name: "field_name".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Int, + order: RecordFieldOrder::Ignore, + position: 0, + custom_attributes: Default::default(), + }], + lookup: Default::default(), + attributes: Default::default(), + }), + false, + r#"Invalid value: Record([("unknown_field_name", Null)]) for schema: Record(RecordSchema { name: Name { name: "record_name", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "field_name", doc: None, aliases: None, default: None, schema: Int, order: Ignore, position: 0, custom_attributes: {} }], lookup: {}, attributes: {} }). Reason: There is no schema field for field 'unknown_field_name'"#, + ), + ( + Value::Record(vec![("field_name".to_string(), Value::Null)]), + Schema::Record(RecordSchema { + name: Name::new("record_name").unwrap(), aliases: None, - schema: Schema::Ref { - name: Name::new("missing").unwrap(), - }, - order: RecordFieldOrder::Ignore, - position: 0, - custom_attributes: Default::default(), - }], - lookup: [("field_name".to_string(), 0)].iter().cloned().collect(), - attributes: Default::default(), - }), - false, - r#"Invalid value: Record([("field_name", Null)]) for schema: Record(RecordSchema { name: Name { name: "record_name", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "field_name", doc: None, aliases: None, default: None, schema: Ref { name: Name { name: "missing", namespace: None } }, order: Ignore, position: 0, custom_attributes: {} }], lookup: {"field_name": 0}, attributes: {} }). Reason: Unresolved schema reference: 'Name { name: "missing", namespace: None }'. Parsed names: []"#, - ), - ]; - - for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { - let err_message = - value.validate_internal::(&schema, &HashMap::default(), &None); - assert_eq!(valid, err_message.is_none()); - if !valid { - let full_err_message = format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, - schema, - err_message.unwrap() - ); - assert_eq!(expected_err_message, full_err_message); + doc: None, + fields: vec![RecordField { + name: "field_name".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Ref { + name: Name::new("missing").unwrap(), + }, + order: RecordFieldOrder::Ignore, + position: 0, + custom_attributes: Default::default(), + }], + lookup: [("field_name".to_string(), 0)].iter().cloned().collect(), + attributes: Default::default(), + }), + false, + r#"Invalid value: Record([("field_name", Null)]) for schema: Record(RecordSchema { name: Name { name: "record_name", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "field_name", doc: None, aliases: None, default: None, schema: Ref { name: Name { name: "missing", namespace: None } }, order: Ignore, position: 0, custom_attributes: {} }], lookup: {"field_name": 0}, attributes: {} }). Reason: Unresolved schema reference: 'Name { name: "missing", namespace: None }'. Parsed names: []"#, + ), + ]; + + for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { + let err_message = + value.validate_internal::(&schema, &HashMap::default(), &None); + assert_eq!(valid, err_message.is_none()); + if !valid { + let full_err_message = format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, + schema, + err_message.unwrap() + ); + assert_eq!(expected_err_message, full_err_message); + } } + + Ok(()) } - Ok(()) - } + #[tokio::test] + async fn validate_fixed() -> TestResult { + let schema = Schema::Fixed(FixedSchema { + size: 4, + name: Name::new("some_fixed").unwrap(), + aliases: None, + doc: None, + default: None, + attributes: Default::default(), + }); + + assert!(Value::Fixed(4, vec![0, 0, 0, 0]).validate(&schema)); + let value = Value::Fixed(5, vec![0, 0, 0, 0, 0]); + assert!(!value.validate(&schema)); + assert_logged( + format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, schema, "The value's size (5) is different than the schema's size (4)" + ) + .as_str(), + ); + + assert!(Value::Bytes(vec![0, 0, 0, 0]).validate(&schema)); + let value = Value::Bytes(vec![0, 0, 0, 0, 0]); + assert!(!value.validate(&schema)); + assert_logged( + format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, schema, "The bytes' length (5) is different than the schema's size (4)" + ) + .as_str(), + ); - #[tokio::test] - async fn validate_fixed() -> TestResult { - let schema = Schema::Fixed(FixedSchema { - size: 4, - name: Name::new("some_fixed").unwrap(), - aliases: None, - doc: None, - default: None, - attributes: Default::default(), - }); - - assert!(Value::Fixed(4, vec![0, 0, 0, 0]).validate(&schema)); - let value = Value::Fixed(5, vec![0, 0, 0, 0, 0]); - assert!(!value.validate(&schema)); - assert_logged( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, schema, "The value's size (5) is different than the schema's size (4)" - ) - .as_str(), - ); - - assert!(Value::Bytes(vec![0, 0, 0, 0]).validate(&schema)); - let value = Value::Bytes(vec![0, 0, 0, 0, 0]); - assert!(!value.validate(&schema)); - assert_logged( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, schema, "The bytes' length (5) is different than the schema's size (4)" - ) - .as_str(), - ); + Ok(()) + } - Ok(()) - } + #[tokio::test] + async fn validate_enum() -> TestResult { + let schema = Schema::Enum(EnumSchema { + name: Name::new("some_enum").unwrap(), + aliases: None, + doc: None, + symbols: vec![ + "spades".to_string(), + "hearts".to_string(), + "diamonds".to_string(), + "clubs".to_string(), + ], + default: None, + attributes: Default::default(), + }); - #[tokio::test] - async fn validate_enum() -> TestResult { - let schema = Schema::Enum(EnumSchema { - name: Name::new("some_enum").unwrap(), - aliases: None, - doc: None, - symbols: vec![ - "spades".to_string(), - "hearts".to_string(), - "diamonds".to_string(), - "clubs".to_string(), - ], - default: None, - attributes: Default::default(), - }); - - assert!(Value::Enum(0, "spades".to_string()).validate(&schema)); - assert!(Value::String("spades".to_string()).validate(&schema)); - - let value = Value::Enum(1, "spades".to_string()); - assert!(!value.validate(&schema)); - assert_logged( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, schema, "Symbol 'spades' is not at position '1'" - ) - .as_str(), - ); - - let value = Value::Enum(1000, "spades".to_string()); - assert!(!value.validate(&schema)); - assert_logged( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, schema, "No symbol at position '1000'" - ) - .as_str(), - ); - - let value = Value::String("lorem".to_string()); - assert!(!value.validate(&schema)); - assert_logged( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, schema, "'lorem' is not a member of the possible symbols" - ) - .as_str(), - ); - - let other_schema = Schema::Enum(EnumSchema { - name: Name::new("some_other_enum").unwrap(), - aliases: None, - doc: None, - symbols: vec![ - "hearts".to_string(), - "diamonds".to_string(), - "clubs".to_string(), - "spades".to_string(), - ], - default: None, - attributes: Default::default(), - }); - - let value = Value::Enum(0, "spades".to_string()); - assert!(!value.validate(&other_schema)); - assert_logged( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, other_schema, "Symbol 'spades' is not at position '0'" - ) - .as_str(), - ); + assert!(Value::Enum(0, "spades".to_string()).validate(&schema)); + assert!(Value::String("spades".to_string()).validate(&schema)); - Ok(()) - } + let value = Value::Enum(1, "spades".to_string()); + assert!(!value.validate(&schema)); + assert_logged( + format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, schema, "Symbol 'spades' is not at position '1'" + ) + .as_str(), + ); - #[tokio::test] - async fn validate_record() -> TestResult { - // { - // "type": "record", - // "fields": [ - // {"type": "long", "name": "a"}, - // {"type": "string", "name": "b"}, - // { - // "type": ["null", "int"] - // "name": "c", - // "default": null - // } - // ] - // } - let schema = Schema::Record(RecordSchema { - name: Name::new("some_record").unwrap(), - aliases: None, - doc: None, - fields: vec![ - RecordField { - name: "a".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::Long, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), - }, - RecordField { - name: "b".to_string(), - doc: None, - default: None, - aliases: None, - schema: Schema::String, - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - RecordField { - name: "c".to_string(), - doc: None, - default: Some(JsonValue::Null), - aliases: None, - schema: Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), - order: RecordFieldOrder::Ascending, - position: 2, - custom_attributes: Default::default(), - }, - ], - lookup: [ - ("a".to_string(), 0), - ("b".to_string(), 1), - ("c".to_string(), 2), - ] - .iter() - .cloned() - .collect(), - attributes: Default::default(), - }); - - assert!( - Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ]) - .validate(&schema) - ); - - let value = Value::Record(vec![ - ("b".to_string(), Value::String("foo".to_string())), - ("a".to_string(), Value::Long(42i64)), - ]); - assert!(value.validate(&schema)); - - let value = Value::Record(vec![ - ("a".to_string(), Value::Boolean(false)), - ("b".to_string(), Value::String("foo".to_string())), - ]); - assert!(!value.validate(&schema)); - assert_logged( - r#"Invalid value: Record([("a", Boolean(false)), ("b", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Unsupported value-schema combination! Value: Boolean(false), schema: Long"#, - ); - - let value = Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("c".to_string(), Value::String("foo".to_string())), - ]); - assert!(!value.validate(&schema)); - assert_logged( - r#"Invalid value: Record([("a", Long(42)), ("c", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Could not find matching type in union"#, - ); - assert_not_logged( - r#"Invalid value: String("foo") for schema: Int. Reason: Unsupported value-schema combination"#, - ); - - let value = Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("d".to_string(), Value::String("foo".to_string())), - ]); - assert!(!value.validate(&schema)); - assert_logged( - r#"Invalid value: Record([("a", Long(42)), ("d", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: There is no schema field for field 'd'"#, - ); - - let value = Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ("c".to_string(), Value::Null), - ("d".to_string(), Value::Null), - ]); - assert!(!value.validate(&schema)); - assert_logged( - r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")), ("c", Null), ("d", Null)]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: The value's records length (4) is greater than the schema's (3 fields)"#, - ); - - assert!( - Value::Map( - vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ] - .into_iter() - .collect() - ) - .validate(&schema) - ); + let value = Value::Enum(1000, "spades".to_string()); + assert!(!value.validate(&schema)); + assert_logged( + format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, schema, "No symbol at position '1000'" + ) + .as_str(), + ); - assert!( - !Value::Map( - vec![("d".to_string(), Value::Long(123_i64)),] - .into_iter() - .collect() - ) - .validate(&schema) - ); - assert_logged( - r#"Invalid value: Map({"d": Long(123)}) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Field with name '"a"' is not a member of the map items -Field with name '"b"' is not a member of the map items"#, - ); + let value = Value::String("lorem".to_string()); + assert!(!value.validate(&schema)); + assert_logged( + format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, schema, "'lorem' is not a member of the possible symbols" + ) + .as_str(), + ); + + let other_schema = Schema::Enum(EnumSchema { + name: Name::new("some_other_enum").unwrap(), + aliases: None, + doc: None, + symbols: vec![ + "hearts".to_string(), + "diamonds".to_string(), + "clubs".to_string(), + "spades".to_string(), + ], + default: None, + attributes: Default::default(), + }); + + let value = Value::Enum(0, "spades".to_string()); + assert!(!value.validate(&other_schema)); + assert_logged( + format!( + "Invalid value: {:?} for schema: {:?}. Reason: {}", + value, other_schema, "Symbol 'spades' is not at position '0'" + ) + .as_str(), + ); + + Ok(()) + } - let union_schema = Schema::Union(UnionSchema::new(vec![Schema::Null, schema])?); + #[tokio::test] + async fn validate_record() -> TestResult { + // { + // "type": "record", + // "fields": [ + // {"type": "long", "name": "a"}, + // {"type": "string", "name": "b"}, + // { + // "type": ["null", "int"] + // "name": "c", + // "default": null + // } + // ] + // } + let schema = Schema::Record(RecordSchema { + name: Name::new("some_record").unwrap(), + aliases: None, + doc: None, + fields: vec![ + RecordField { + name: "a".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Long, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: Default::default(), + }, + RecordField { + name: "b".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::String, + order: RecordFieldOrder::Ascending, + position: 1, + custom_attributes: Default::default(), + }, + RecordField { + name: "c".to_string(), + doc: None, + default: Some(JsonValue::Null), + aliases: None, + schema: Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), + order: RecordFieldOrder::Ascending, + position: 2, + custom_attributes: Default::default(), + }, + ], + lookup: [ + ("a".to_string(), 0), + ("b".to_string(), 1), + ("c".to_string(), 2), + ] + .iter() + .cloned() + .collect(), + attributes: Default::default(), + }); - assert!( - Value::Union( - 1, - Box::new(Value::Record(vec![ + assert!( + Value::Record(vec![ ("a".to_string(), Value::Long(42i64)), ("b".to_string(), Value::String("foo".to_string())), - ])) - ) - .validate(&union_schema) - ); + ]) + .validate(&schema) + ); + + let value = Value::Record(vec![ + ("b".to_string(), Value::String("foo".to_string())), + ("a".to_string(), Value::Long(42i64)), + ]); + assert!(value.validate(&schema)); + + let value = Value::Record(vec![ + ("a".to_string(), Value::Boolean(false)), + ("b".to_string(), Value::String("foo".to_string())), + ]); + assert!(!value.validate(&schema)); + assert_logged( + r#"Invalid value: Record([("a", Boolean(false)), ("b", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Unsupported value-schema combination! Value: Boolean(false), schema: Long"#, + ); - assert!( - Value::Union( - 1, - Box::new(Value::Map( + let value = Value::Record(vec![ + ("a".to_string(), Value::Long(42i64)), + ("c".to_string(), Value::String("foo".to_string())), + ]); + assert!(!value.validate(&schema)); + assert_logged( + r#"Invalid value: Record([("a", Long(42)), ("c", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Could not find matching type in union"#, + ); + assert_not_logged( + r#"Invalid value: String("foo") for schema: Int. Reason: Unsupported value-schema combination"#, + ); + + let value = Value::Record(vec![ + ("a".to_string(), Value::Long(42i64)), + ("d".to_string(), Value::String("foo".to_string())), + ]); + assert!(!value.validate(&schema)); + assert_logged( + r#"Invalid value: Record([("a", Long(42)), ("d", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: There is no schema field for field 'd'"#, + ); + + let value = Value::Record(vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ("c".to_string(), Value::Null), + ("d".to_string(), Value::Null), + ]); + assert!(!value.validate(&schema)); + assert_logged( + r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")), ("c", Null), ("d", Null)]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: The value's records length (4) is greater than the schema's (3 fields)"#, + ); + + assert!( + Value::Map( vec![ ("a".to_string(), Value::Long(42i64)), ("b".to_string(), Value::String("foo".to_string())), ] .into_iter() .collect() - )) - ) - .validate(&union_schema) - ); - - Ok(()) - } + ) + .validate(&schema) + ); + + assert!( + !Value::Map( + vec![("d".to_string(), Value::Long(123_i64)),] + .into_iter() + .collect() + ) + .validate(&schema) + ); + assert_logged( + r#"Invalid value: Map({"d": Long(123)}) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Field with name '"a"' is not a member of the map items +Field with name '"b"' is not a member of the map items"#, + ); - #[tokio::test] - async fn resolve_bytes_ok() -> TestResult { - let value = Value::Array(vec![Value::Int(0), Value::Int(42)]); - assert_eq!( - value.resolve(&Schema::Bytes).await?, - Value::Bytes(vec![0u8, 42u8]) - ); + let union_schema = Schema::Union(UnionSchema::new(vec![Schema::Null, schema])?); - Ok(()) - } + assert!( + Value::Union( + 1, + Box::new(Value::Record(vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ])) + ) + .validate(&union_schema) + ); + + assert!( + Value::Union( + 1, + Box::new(Value::Map( + vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ] + .into_iter() + .collect() + )) + ) + .validate(&union_schema) + ); - #[tokio::test] - async fn resolve_string_from_bytes() -> TestResult { - let value = Value::Bytes(vec![97, 98, 99]); - assert_eq!( - value.resolve(&Schema::String).await?, - Value::String("abc".to_string()) - ); + Ok(()) + } - Ok(()) - } + #[tokio::test] + async fn resolve_bytes_ok() -> TestResult { + let value = Value::Array(vec![Value::Int(0), Value::Int(42)]); + assert_eq!( + value.resolve(&Schema::Bytes).await?, + Value::Bytes(vec![0u8, 42u8]) + ); - #[tokio::test] - async fn resolve_string_from_fixed() -> TestResult { - let value = Value::Fixed(3, vec![97, 98, 99]); - assert_eq!( - value.resolve(&Schema::String).await?, - Value::String("abc".to_string()) - ); + Ok(()) + } - Ok(()) - } + #[tokio::test] + async fn resolve_string_from_bytes() -> TestResult { + let value = Value::Bytes(vec![97, 98, 99]); + assert_eq!( + value.resolve(&Schema::String).await?, + Value::String("abc".to_string()) + ); - #[tokio::test] - async fn resolve_bytes_failure() { - let value = Value::Array(vec![Value::Int(2000), Value::Int(-42)]); - assert!(value.resolve(&Schema::Bytes).await.is_err()); - } + Ok(()) + } - #[tokio::test] - async fn resolve_decimal_bytes() -> TestResult { - let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); - value.clone().resolve(&Schema::Decimal(DecimalSchema { - precision: 10, - scale: 4, - inner: Box::new(Schema::Bytes), - })).await?; - assert!(value.resolve(&Schema::String).await.is_err()); - - Ok(()) - } + #[tokio::test] + async fn resolve_string_from_fixed() -> TestResult { + let value = Value::Fixed(3, vec![97, 98, 99]); + assert_eq!( + value.resolve(&Schema::String).await?, + Value::String("abc".to_string()) + ); - #[tokio::test] - async fn resolve_decimal_invalid_scale() { - let value = Value::Decimal(Decimal::from(vec![1, 2])); - assert!( - value - .resolve(&Schema::Decimal(DecimalSchema { - precision: 2, - scale: 3, - inner: Box::new(Schema::Bytes), - })).await - .is_err() - ); - } + Ok(()) + } - #[tokio::test] - async fn resolve_decimal_invalid_precision_for_length() { - let value = Value::Decimal(Decimal::from((1u8..=8u8).rev().collect::>())); - assert!( - value - .resolve(&Schema::Decimal(DecimalSchema { - precision: 1, - scale: 0, - inner: Box::new(Schema::Bytes), - })).await - .is_ok() - ); - } + #[tokio::test] + async fn resolve_bytes_failure() { + let value = Value::Array(vec![Value::Int(2000), Value::Int(-42)]); + assert!(value.resolve(&Schema::Bytes).await.is_err()); + } - #[tokio::test] - async fn resolve_decimal_fixed() { - let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); - assert!( + #[tokio::test] + async fn resolve_decimal_bytes() -> TestResult { + let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); value .clone() .resolve(&Schema::Decimal(DecimalSchema { precision: 10, - scale: 1, - inner: Box::new(Schema::Fixed(FixedSchema { - name: Name::new("decimal").unwrap(), - aliases: None, - size: 20, - doc: None, - default: None, - attributes: Default::default(), - })) - })).await - .is_ok() - ); - assert!(value.resolve(&Schema::String).await.is_err()); - } - - #[tokio::test] - async fn resolve_date() { - let value = Value::Date(2345); - assert!(value.clone().resolve(&Schema::Date).await.is_ok()); - assert!(value.resolve(&Schema::String).await.is_err()); - } + scale: 4, + inner: Box::new(Schema::Bytes), + })) + .await?; + assert!(value.resolve(&Schema::String).await.is_err()); - #[tokio::test] - async fn resolve_time_millis() { - let value = Value::TimeMillis(10); - assert!(value.clone().resolve(&Schema::TimeMillis).await.is_ok()); - assert!(value.resolve(&Schema::TimeMicros).await.is_err()); - } + Ok(()) + } - #[tokio::test] - async fn resolve_time_micros() { - let value = Value::TimeMicros(10); - assert!(value.clone().resolve(&Schema::TimeMicros).await.is_ok()); - assert!(value.resolve(&Schema::TimeMillis).await.is_err()); - } + #[tokio::test] + async fn resolve_decimal_invalid_scale() { + let value = Value::Decimal(Decimal::from(vec![1, 2])); + assert!( + value + .resolve(&Schema::Decimal(DecimalSchema { + precision: 2, + scale: 3, + inner: Box::new(Schema::Bytes), + })) + .await + .is_err() + ); + } - #[tokio::test] - async fn resolve_timestamp_millis() { - let value = Value::TimestampMillis(10); - assert!(value.clone().resolve(&Schema::TimestampMillis).await.is_ok()); - assert!(value.resolve(&Schema::Float).await.is_err()); + #[tokio::test] + async fn resolve_decimal_invalid_precision_for_length() { + let value = Value::Decimal(Decimal::from((1u8..=8u8).rev().collect::>())); + assert!( + value + .resolve(&Schema::Decimal(DecimalSchema { + precision: 1, + scale: 0, + inner: Box::new(Schema::Bytes), + })) + .await + .is_ok() + ); + } - let value = Value::Float(10.0f32); - assert!(value.resolve(&Schema::TimestampMillis).await.is_err()); - } + #[tokio::test] + async fn resolve_decimal_fixed() { + let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); + assert!( + value + .clone() + .resolve(&Schema::Decimal(DecimalSchema { + precision: 10, + scale: 1, + inner: Box::new(Schema::Fixed(FixedSchema { + name: Name::new("decimal").unwrap(), + aliases: None, + size: 20, + doc: None, + default: None, + attributes: Default::default(), + })) + })) + .await + .is_ok() + ); + assert!(value.resolve(&Schema::String).await.is_err()); + } - #[tokio::test] - async fn resolve_timestamp_micros() { - let value = Value::TimestampMicros(10); - assert!(value.clone().resolve(&Schema::TimestampMicros).await.is_ok()); - assert!(value.resolve(&Schema::Int).await.is_err()); + #[tokio::test] + async fn resolve_date() { + let value = Value::Date(2345); + assert!(value.clone().resolve(&Schema::Date).await.is_ok()); + assert!(value.resolve(&Schema::String).await.is_err()); + } - let value = Value::Double(10.0); - assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); - } + #[tokio::test] + async fn resolve_time_millis() { + let value = Value::TimeMillis(10); + assert!(value.clone().resolve(&Schema::TimeMillis).await.is_ok()); + assert!(value.resolve(&Schema::TimeMicros).await.is_err()); + } - #[tokio::test] - async fn test_avro_3914_resolve_timestamp_nanos() { - let value = Value::TimestampNanos(10); - assert!(value.clone().resolve(&Schema::TimestampNanos).await.is_ok()); - assert!(value.resolve(&Schema::Int).await.is_err()); + #[tokio::test] + async fn resolve_time_micros() { + let value = Value::TimeMicros(10); + assert!(value.clone().resolve(&Schema::TimeMicros).await.is_ok()); + assert!(value.resolve(&Schema::TimeMillis).await.is_err()); + } - let value = Value::Double(10.0); - assert!(value.resolve(&Schema::TimestampNanos).await.is_err()); - } + #[tokio::test] + async fn resolve_timestamp_millis() { + let value = Value::TimestampMillis(10); + assert!( + value + .clone() + .resolve(&Schema::TimestampMillis) + .await + .is_ok() + ); + assert!(value.resolve(&Schema::Float).await.is_err()); + + let value = Value::Float(10.0f32); + assert!(value.resolve(&Schema::TimestampMillis).await.is_err()); + } - #[tokio::test] - async fn test_avro_3853_resolve_timestamp_millis() { - let value = Value::LocalTimestampMillis(10); - assert!(value.clone().resolve(&Schema::LocalTimestampMillis).await.is_ok()); - assert!(value.resolve(&Schema::Float).await.is_err()); + #[tokio::test] + async fn resolve_timestamp_micros() { + let value = Value::TimestampMicros(10); + assert!( + value + .clone() + .resolve(&Schema::TimestampMicros) + .await + .is_ok() + ); + assert!(value.resolve(&Schema::Int).await.is_err()); + + let value = Value::Double(10.0); + assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); + } - let value = Value::Float(10.0f32); - assert!(value.resolve(&Schema::LocalTimestampMillis).await.is_err()); - } + #[tokio::test] + async fn test_avro_3914_resolve_timestamp_nanos() { + let value = Value::TimestampNanos(10); + assert!(value.clone().resolve(&Schema::TimestampNanos).await.is_ok()); + assert!(value.resolve(&Schema::Int).await.is_err()); - #[tokio::test] - async fn test_avro_3853_resolve_timestamp_micros() { - let value = Value::LocalTimestampMicros(10); - assert!(value.clone().resolve(&Schema::LocalTimestampMicros).await.is_ok()); - assert!(value.resolve(&Schema::Int).await.is_err()); + let value = Value::Double(10.0); + assert!(value.resolve(&Schema::TimestampNanos).await.is_err()); + } - let value = Value::Double(10.0); - assert!(value.resolve(&Schema::LocalTimestampMicros).await.is_err()); - } + #[tokio::test] + async fn test_avro_3853_resolve_timestamp_millis() { + let value = Value::LocalTimestampMillis(10); + assert!( + value + .clone() + .resolve(&Schema::LocalTimestampMillis) + .await + .is_ok() + ); + assert!(value.resolve(&Schema::Float).await.is_err()); + + let value = Value::Float(10.0f32); + assert!(value.resolve(&Schema::LocalTimestampMillis).await.is_err()); + } - #[tokio::test] - async fn test_avro_3916_resolve_timestamp_nanos() { - let value = Value::LocalTimestampNanos(10); - assert!(value.clone().resolve(&Schema::LocalTimestampNanos).await.is_ok()); - assert!(value.resolve(&Schema::Int).await.is_err()); + #[tokio::test] + async fn test_avro_3853_resolve_timestamp_micros() { + let value = Value::LocalTimestampMicros(10); + assert!( + value + .clone() + .resolve(&Schema::LocalTimestampMicros) + .await + .is_ok() + ); + assert!(value.resolve(&Schema::Int).await.is_err()); + + let value = Value::Double(10.0); + assert!(value.resolve(&Schema::LocalTimestampMicros).await.is_err()); + } - let value = Value::Double(10.0); - assert!(value.resolve(&Schema::LocalTimestampNanos).await.is_err()); - } + #[tokio::test] + async fn test_avro_3916_resolve_timestamp_nanos() { + let value = Value::LocalTimestampNanos(10); + assert!( + value + .clone() + .resolve(&Schema::LocalTimestampNanos) + .await + .is_ok() + ); + assert!(value.resolve(&Schema::Int).await.is_err()); + + let value = Value::Double(10.0); + assert!(value.resolve(&Schema::LocalTimestampNanos).await.is_err()); + } - #[tokio::test] - async fn resolve_duration() { - let value = Value::Duration(Duration::new( - Months::new(10), - Days::new(5), - Millis::new(3000), - )); - assert!(value.clone().resolve(&Schema::Duration).await.is_ok()); - assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); - assert!(Value::Long(1i64).resolve(&Schema::Duration).await.is_err()); - } + #[tokio::test] + async fn resolve_duration() { + let value = Value::Duration(Duration::new( + Months::new(10), + Days::new(5), + Millis::new(3000), + )); + assert!(value.clone().resolve(&Schema::Duration).await.is_ok()); + assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); + assert!(Value::Long(1i64).resolve(&Schema::Duration).await.is_err()); + } - #[tokio::test] - async fn resolve_uuid() -> TestResult { - let value = Value::Uuid(Uuid::parse_str("1481531d-ccc9-46d9-a56f-5b67459c0537")?); - assert!(value.clone().resolve(&Schema::Uuid).await.is_ok()); - assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); + #[tokio::test] + async fn resolve_uuid() -> TestResult { + let value = Value::Uuid(Uuid::parse_str("1481531d-ccc9-46d9-a56f-5b67459c0537")?); + assert!(value.clone().resolve(&Schema::Uuid).await.is_ok()); + assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn avro_3678_resolve_float_to_double() { - let value = Value::Float(2345.1); - assert!(value.resolve(&Schema::Double).await.is_ok()); - } + #[tokio::test] + async fn avro_3678_resolve_float_to_double() { + let value = Value::Float(2345.1); + assert!(value.resolve(&Schema::Double).await.is_ok()); + } - #[tokio::test] - async fn test_avro_3621_resolve_to_nullable_union() -> TestResult { - let schema = Schema::parse_str( - r#"{ + #[tokio::test] + async fn test_avro_3621_resolve_to_nullable_union() -> TestResult { + let schema = Schema::parse_str( + r#"{ "type": "record", "name": "root", "fields": [ @@ -1916,202 +2014,202 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + )?; - let value = Value::Record(vec![( - "event".to_string(), - Value::Record(vec![("amount".to_string(), Value::Int(200))]), - )]); - assert!(value.resolve(&schema).await.is_ok()); + let value = Value::Record(vec![( + "event".to_string(), + Value::Record(vec![("amount".to_string(), Value::Int(200))]), + )]); + assert!(value.resolve(&schema).await.is_ok()); - let value = Value::Record(vec![( - "event".to_string(), - Value::Record(vec![("size".to_string(), Value::Int(1))]), - )]); - assert!(value.resolve(&schema).await.is_err()); + let value = Value::Record(vec![( + "event".to_string(), + Value::Record(vec![("size".to_string(), Value::Int(1))]), + )]); + assert!(value.resolve(&schema).await.is_err()); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn json_from_avro() -> TestResult { - assert_eq!(JsonValue::try_from(Value::Null)?, JsonValue::Null); - assert_eq!( - JsonValue::try_from(Value::Boolean(true))?, - JsonValue::Bool(true) - ); - assert_eq!( - JsonValue::try_from(Value::Int(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::Long(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::Float(1.0))?, - JsonValue::Number(Number::from_f64(1.0).unwrap()) - ); - assert_eq!( - JsonValue::try_from(Value::Double(1.0))?, - JsonValue::Number(Number::from_f64(1.0).unwrap()) - ); - assert_eq!( - JsonValue::try_from(Value::Bytes(vec![1, 2, 3]))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) - ]) - ); - assert_eq!( - JsonValue::try_from(Value::String("test".into()))?, - JsonValue::String("test".into()) - ); - assert_eq!( - JsonValue::try_from(Value::Fixed(3, vec![1, 2, 3]))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) - ]) - ); - assert_eq!( - JsonValue::try_from(Value::Enum(1, "test_enum".into()))?, - JsonValue::String("test_enum".into()) - ); - assert_eq!( - JsonValue::try_from(Value::Union(1, Box::new(Value::String("test_enum".into()))))?, - JsonValue::String("test_enum".into()) - ); - assert_eq!( - JsonValue::try_from(Value::Array(vec![ - Value::Int(1), - Value::Int(2), - Value::Int(3) - ]))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) - ]) - ); - assert_eq!( - JsonValue::try_from(Value::Map( - vec![ + #[tokio::test] + async fn json_from_avro() -> TestResult { + assert_eq!(JsonValue::try_from(Value::Null)?, JsonValue::Null); + assert_eq!( + JsonValue::try_from(Value::Boolean(true))?, + JsonValue::Bool(true) + ); + assert_eq!( + JsonValue::try_from(Value::Int(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::Long(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::Float(1.0))?, + JsonValue::Number(Number::from_f64(1.0).unwrap()) + ); + assert_eq!( + JsonValue::try_from(Value::Double(1.0))?, + JsonValue::Number(Number::from_f64(1.0).unwrap()) + ); + assert_eq!( + JsonValue::try_from(Value::Bytes(vec![1, 2, 3]))?, + JsonValue::Array(vec![ + JsonValue::Number(1.into()), + JsonValue::Number(2.into()), + JsonValue::Number(3.into()) + ]) + ); + assert_eq!( + JsonValue::try_from(Value::String("test".into()))?, + JsonValue::String("test".into()) + ); + assert_eq!( + JsonValue::try_from(Value::Fixed(3, vec![1, 2, 3]))?, + JsonValue::Array(vec![ + JsonValue::Number(1.into()), + JsonValue::Number(2.into()), + JsonValue::Number(3.into()) + ]) + ); + assert_eq!( + JsonValue::try_from(Value::Enum(1, "test_enum".into()))?, + JsonValue::String("test_enum".into()) + ); + assert_eq!( + JsonValue::try_from(Value::Union(1, Box::new(Value::String("test_enum".into()))))?, + JsonValue::String("test_enum".into()) + ); + assert_eq!( + JsonValue::try_from(Value::Array(vec![ + Value::Int(1), + Value::Int(2), + Value::Int(3) + ]))?, + JsonValue::Array(vec![ + JsonValue::Number(1.into()), + JsonValue::Number(2.into()), + JsonValue::Number(3.into()) + ]) + ); + assert_eq!( + JsonValue::try_from(Value::Map( + vec![ + ("v1".to_string(), Value::Int(1)), + ("v2".to_string(), Value::Int(2)), + ("v3".to_string(), Value::Int(3)) + ] + .into_iter() + .collect() + ))?, + JsonValue::Object( + vec![ + ("v1".to_string(), JsonValue::Number(1.into())), + ("v2".to_string(), JsonValue::Number(2.into())), + ("v3".to_string(), JsonValue::Number(3.into())) + ] + .into_iter() + .collect() + ) + ); + assert_eq!( + JsonValue::try_from(Value::Record(vec![ ("v1".to_string(), Value::Int(1)), ("v2".to_string(), Value::Int(2)), ("v3".to_string(), Value::Int(3)) - ] - .into_iter() - .collect() - ))?, - JsonValue::Object( - vec![ - ("v1".to_string(), JsonValue::Number(1.into())), - ("v2".to_string(), JsonValue::Number(2.into())), - ("v3".to_string(), JsonValue::Number(3.into())) - ] - .into_iter() - .collect() - ) - ); - assert_eq!( - JsonValue::try_from(Value::Record(vec![ - ("v1".to_string(), Value::Int(1)), - ("v2".to_string(), Value::Int(2)), - ("v3".to_string(), Value::Int(3)) - ]))?, - JsonValue::Object( - vec![ - ("v1".to_string(), JsonValue::Number(1.into())), - ("v2".to_string(), JsonValue::Number(2.into())), - ("v3".to_string(), JsonValue::Number(3.into())) - ] - .into_iter() - .collect() - ) - ); - assert_eq!( - JsonValue::try_from(Value::Date(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::Decimal(vec![1, 2, 3].into()))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) - ]) - ); - assert_eq!( - JsonValue::try_from(Value::TimeMillis(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::TimeMicros(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::TimestampMillis(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::TimestampMicros(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::TimestampNanos(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::LocalTimestampMillis(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::LocalTimestampMicros(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::LocalTimestampNanos(1))?, - JsonValue::Number(1.into()) - ); - assert_eq!( - JsonValue::try_from(Value::Duration( - [ - 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8, 10u8, 11u8, 12u8 - ] - .into() - ))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()), - JsonValue::Number(4.into()), - JsonValue::Number(5.into()), - JsonValue::Number(6.into()), - JsonValue::Number(7.into()), - JsonValue::Number(8.into()), - JsonValue::Number(9.into()), - JsonValue::Number(10.into()), - JsonValue::Number(11.into()), - JsonValue::Number(12.into()), - ]) - ); - assert_eq!( - JsonValue::try_from(Value::Uuid(Uuid::parse_str( - "936DA01F-9ABD-4D9D-80C7-02AF85C822A8" - )?))?, - JsonValue::String("936da01f-9abd-4d9d-80c7-02af85c822a8".into()) - ); - - Ok(()) - } + ]))?, + JsonValue::Object( + vec![ + ("v1".to_string(), JsonValue::Number(1.into())), + ("v2".to_string(), JsonValue::Number(2.into())), + ("v3".to_string(), JsonValue::Number(3.into())) + ] + .into_iter() + .collect() + ) + ); + assert_eq!( + JsonValue::try_from(Value::Date(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::Decimal(vec![1, 2, 3].into()))?, + JsonValue::Array(vec![ + JsonValue::Number(1.into()), + JsonValue::Number(2.into()), + JsonValue::Number(3.into()) + ]) + ); + assert_eq!( + JsonValue::try_from(Value::TimeMillis(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::TimeMicros(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::TimestampMillis(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::TimestampMicros(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::TimestampNanos(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::LocalTimestampMillis(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::LocalTimestampMicros(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::LocalTimestampNanos(1))?, + JsonValue::Number(1.into()) + ); + assert_eq!( + JsonValue::try_from(Value::Duration( + [ + 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8, 10u8, 11u8, 12u8 + ] + .into() + ))?, + JsonValue::Array(vec![ + JsonValue::Number(1.into()), + JsonValue::Number(2.into()), + JsonValue::Number(3.into()), + JsonValue::Number(4.into()), + JsonValue::Number(5.into()), + JsonValue::Number(6.into()), + JsonValue::Number(7.into()), + JsonValue::Number(8.into()), + JsonValue::Number(9.into()), + JsonValue::Number(10.into()), + JsonValue::Number(11.into()), + JsonValue::Number(12.into()), + ]) + ); + assert_eq!( + JsonValue::try_from(Value::Uuid(Uuid::parse_str( + "936DA01F-9ABD-4D9D-80C7-02AF85C822A8" + )?))?, + JsonValue::String("936da01f-9abd-4d9d-80c7-02af85c822a8".into()) + ); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3433_recursive_resolves_record() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3433_recursive_resolves_record() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2133,22 +2231,23 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + )?; - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - outer - .resolve(&schema).await - .expect("Record definition defined in one field must be available in other field"); + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); + outer + .resolve(&schema) + .await + .expect("Record definition defined in one field must be available in other field"); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3433_recursive_resolves_array() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3433_recursive_resolves_array() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2176,28 +2275,29 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ("a".into(), Value::Array(vec![inner_value1])), - ( - "b".into(), - Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), - ), - ]); - outer_value - .resolve(&schema).await - .expect("Record defined in array definition must be resolvable from map"); - - Ok(()) - } + )?; + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ("a".into(), Value::Array(vec![inner_value1])), + ( + "b".into(), + Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), + ), + ]); + outer_value + .resolve(&schema) + .await + .expect("Record defined in array definition must be resolvable from map"); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3433_recursive_resolves_map() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3433_recursive_resolves_map() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2222,28 +2322,29 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ("a".into(), inner_value1), - ( - "b".into(), - Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), - ), - ]); - outer_value - .resolve(&schema).await - .expect("Record defined in record field must be resolvable from map field"); - - Ok(()) - } + )?; - #[tokio::test] - async fn test_avro_3433_recursive_resolves_record_wrapper() -> TestResult { - let schema = Schema::parse_str( - r#" + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ("a".into(), inner_value1), + ( + "b".into(), + Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), + ), + ]); + outer_value + .resolve(&schema) + .await + .expect("Record defined in record field must be resolvable from map field"); + + Ok(()) + } + + #[tokio::test] + async fn test_avro_3433_recursive_resolves_record_wrapper() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2272,24 +2373,24 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![( - "j".into(), - Value::Record(vec![("z".into(), Value::Int(6))]), - )]); - let outer_value = - Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - outer_value.resolve(&schema).await.expect("Record schema defined in field must be resolvable in Record schema defined in other field"); - - Ok(()) - } + )?; + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![( + "j".into(), + Value::Record(vec![("z".into(), Value::Int(6))]), + )]); + let outer_value = + Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); + outer_value.resolve(&schema).await.expect("Record schema defined in field must be resolvable in Record schema defined in other field"); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3433_recursive_resolves_map_and_array() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3433_recursive_resolves_map_and_array() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2317,28 +2418,29 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer_value = Value::Record(vec![ - ( - "a".into(), - Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), - ), - ("b".into(), Value::Array(vec![inner_value1])), - ]); - outer_value - .resolve(&schema).await - .expect("Record defined in map definition must be resolvable from array"); - - Ok(()) - } + )?; - #[tokio::test] - async fn test_avro_3433_recursive_resolves_union() -> TestResult { - let schema = Schema::parse_str( - r#" + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer_value = Value::Record(vec![ + ( + "a".into(), + Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), + ), + ("b".into(), Value::Array(vec![inner_value1])), + ]); + outer_value + .resolve(&schema) + .await + .expect("Record defined in map definition must be resolvable from array"); + + Ok(()) + } + + #[tokio::test] + async fn test_avro_3433_recursive_resolves_union() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2360,28 +2462,30 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; - - let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); - let outer1 = Value::Record(vec![ - ("a".into(), inner_value1), - ("b".into(), inner_value2.clone()), - ]); - outer1 - .resolve(&schema).await - .expect("Record definition defined in union must be resolved in other field"); - let outer2 = Value::Record(vec![("a".into(), Value::Null), ("b".into(), inner_value2)]); - outer2 - .resolve(&schema).await - .expect("Record definition defined in union must be resolved in other field"); - - Ok(()) - } + )?; + + let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); + let outer1 = Value::Record(vec![ + ("a".into(), inner_value1), + ("b".into(), inner_value2.clone()), + ]); + outer1 + .resolve(&schema) + .await + .expect("Record definition defined in union must be resolved in other field"); + let outer2 = Value::Record(vec![("a".into(), Value::Null), ("b".into(), inner_value2)]); + outer2 + .resolve(&schema) + .await + .expect("Record definition defined in union must be resolved in other field"); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3461_test_multi_level_resolve_outer_namespace() -> TestResult { - let schema = r#" + #[tokio::test] + async fn test_avro_3461_test_multi_level_resolve_outer_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -2422,54 +2526,57 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema)?; - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema)?; + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - outer_record_variation_1 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_2 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_3 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - - Ok(()) - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + outer_record_variation_1 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + outer_record_variation_2 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + outer_record_variation_3 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3461_test_multi_level_resolve_middle_namespace() -> TestResult { - let schema = r#" + #[tokio::test] + async fn test_avro_3461_test_multi_level_resolve_middle_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -2511,54 +2618,57 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema)?; - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema)?; + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - outer_record_variation_1 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_2 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_3 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - - Ok(()) - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + outer_record_variation_1 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + outer_record_variation_2 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + outer_record_variation_3 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3461_test_multi_level_resolve_inner_namespace() -> TestResult { - let schema = r#" + #[tokio::test] + async fn test_avro_3461_test_multi_level_resolve_inner_namespace() -> TestResult { + let schema = r#" { "name": "record_name", "namespace": "space", @@ -2601,56 +2711,59 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema)?; - - let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); - let middle_record_variation_1 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(0, Box::new(Value::Null)), - )]); - let middle_record_variation_2 = Value::Record(vec![( - "middle_field_1".into(), - Value::Union(1, Box::new(inner_record.clone())), - )]); - let outer_record_variation_1 = Value::Record(vec![ - ( - "outer_field_1".into(), + let schema = Schema::parse_str(schema)?; + + let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); + let middle_record_variation_1 = Value::Record(vec![( + "middle_field_1".into(), Value::Union(0, Box::new(Value::Null)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_2 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_1)), - ), - ("outer_field_2".into(), inner_record.clone()), - ]); - let outer_record_variation_3 = Value::Record(vec![ - ( - "outer_field_1".into(), - Value::Union(1, Box::new(middle_record_variation_2)), - ), - ("outer_field_2".into(), inner_record), - ]); - - outer_record_variation_1 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_2 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_3 - .resolve(&schema).await - .expect("Should be able to resolve value to the schema that is it's definition"); - - Ok(()) - } + )]); + let middle_record_variation_2 = Value::Record(vec![( + "middle_field_1".into(), + Value::Union(1, Box::new(inner_record.clone())), + )]); + let outer_record_variation_1 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(0, Box::new(Value::Null)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_2 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_1)), + ), + ("outer_field_2".into(), inner_record.clone()), + ]); + let outer_record_variation_3 = Value::Record(vec![ + ( + "outer_field_1".into(), + Value::Union(1, Box::new(middle_record_variation_2)), + ), + ("outer_field_2".into(), inner_record), + ]); + + outer_record_variation_1 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + outer_record_variation_2 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + outer_record_variation_3 + .resolve(&schema) + .await + .expect("Should be able to resolve value to the schema that is it's definition"); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3460_validation_with_refs() -> TestResult { - let schema = Schema::parse_str( - r#" + #[tokio::test] + async fn test_avro_3460_validation_with_refs() -> TestResult { + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2672,63 +2785,64 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; - - let inner_value_right = Value::Record(vec![("z".into(), Value::Int(3))]); - let inner_value_wrong1 = Value::Record(vec![("z".into(), Value::Null)]); - let inner_value_wrong2 = Value::Record(vec![("a".into(), Value::String("testing".into()))]); - let outer1 = Value::Record(vec![ - ("a".into(), inner_value_right.clone()), - ("b".into(), inner_value_wrong1), - ]); - - let outer2 = Value::Record(vec![ - ("a".into(), inner_value_right), - ("b".into(), inner_value_wrong2), - ]); - - assert!( - !outer1.validate(&schema), - "field b record is invalid against the schema" - ); // this should pass, but doesn't - assert!( - !outer2.validate(&schema), - "field b record is invalid against the schema" - ); // this should pass, but doesn't - - Ok(()) - } + )?; + + let inner_value_right = Value::Record(vec![("z".into(), Value::Int(3))]); + let inner_value_wrong1 = Value::Record(vec![("z".into(), Value::Null)]); + let inner_value_wrong2 = + Value::Record(vec![("a".into(), Value::String("testing".into()))]); + let outer1 = Value::Record(vec![ + ("a".into(), inner_value_right.clone()), + ("b".into(), inner_value_wrong1), + ]); + + let outer2 = Value::Record(vec![ + ("a".into(), inner_value_right), + ("b".into(), inner_value_wrong2), + ]); + + assert!( + !outer1.validate(&schema), + "field b record is invalid against the schema" + ); // this should pass, but doesn't + assert!( + !outer2.validate(&schema), + "field b record is invalid against the schema" + ); // this should pass, but doesn't + + Ok(()) + } - #[tokio::test] - async fn test_avro_3460_validation_with_refs_real_struct() -> TestResult { - use crate::ser::Serializer; - use serde::Serialize; + #[tokio::test] + async fn test_avro_3460_validation_with_refs_real_struct() -> TestResult { + use crate::ser::Serializer; + use serde::Serialize; - #[derive(Serialize, Clone)] - struct TestInner { - z: i32, - } + #[derive(Serialize, Clone)] + struct TestInner { + z: i32, + } - #[derive(Serialize)] - struct TestRefSchemaStruct1 { - a: TestInner, - b: String, // could be literally anything - } + #[derive(Serialize)] + struct TestRefSchemaStruct1 { + a: TestInner, + b: String, // could be literally anything + } - #[derive(Serialize)] - struct TestRefSchemaStruct2 { - a: TestInner, - b: i32, // could be literally anything - } + #[derive(Serialize)] + struct TestRefSchemaStruct2 { + a: TestInner, + b: i32, // could be literally anything + } - #[derive(Serialize)] - struct TestRefSchemaStruct3 { - a: TestInner, - b: Option, // could be literally anything - } + #[derive(Serialize)] + struct TestRefSchemaStruct3 { + a: TestInner, + b: Option, // could be literally anything + } - let schema = Schema::parse_str( - r#" + let schema = Schema::parse_str( + r#" { "type":"record", "name":"TestStruct", @@ -2750,50 +2864,50 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + )?; - let test_inner = TestInner { z: 3 }; - let test_outer1 = TestRefSchemaStruct1 { - a: test_inner.clone(), - b: "testing".into(), - }; - let test_outer2 = TestRefSchemaStruct2 { - a: test_inner.clone(), - b: 24, - }; - let test_outer3 = TestRefSchemaStruct3 { - a: test_inner, - b: None, - }; + let test_inner = TestInner { z: 3 }; + let test_outer1 = TestRefSchemaStruct1 { + a: test_inner.clone(), + b: "testing".into(), + }; + let test_outer2 = TestRefSchemaStruct2 { + a: test_inner.clone(), + b: 24, + }; + let test_outer3 = TestRefSchemaStruct3 { + a: test_inner, + b: None, + }; - let mut ser = Serializer::default(); - let test_outer1: Value = test_outer1.serialize(&mut ser)?; - let mut ser = Serializer::default(); - let test_outer2: Value = test_outer2.serialize(&mut ser)?; - let mut ser = Serializer::default(); - let test_outer3: Value = test_outer3.serialize(&mut ser)?; - - assert!( - !test_outer1.validate(&schema), - "field b record is invalid against the schema" - ); - assert!( - !test_outer2.validate(&schema), - "field b record is invalid against the schema" - ); - assert!( - !test_outer3.validate(&schema), - "field b record is invalid against the schema" - ); - - Ok(()) - } + let mut ser = Serializer::default(); + let test_outer1: Value = test_outer1.serialize(&mut ser)?; + let mut ser = Serializer::default(); + let test_outer2: Value = test_outer2.serialize(&mut ser)?; + let mut ser = Serializer::default(); + let test_outer3: Value = test_outer3.serialize(&mut ser)?; + + assert!( + !test_outer1.validate(&schema), + "field b record is invalid against the schema" + ); + assert!( + !test_outer2.validate(&schema), + "field b record is invalid against the schema" + ); + assert!( + !test_outer3.validate(&schema), + "field b record is invalid against the schema" + ); + + Ok(()) + } - async fn avro_3674_with_or_without_namespace(with_namespace: bool) -> TestResult { - use crate::ser::Serializer; - use serde::Serialize; + async fn avro_3674_with_or_without_namespace(with_namespace: bool) -> TestResult { + use crate::ser::Serializer; + use serde::Serialize; - let schema_str = r#" + let schema_str = r#" { "type": "record", "name": "NamespacedMessage", @@ -2824,69 +2938,69 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema_str = schema_str.replace( - "[NAMESPACE]", - if with_namespace { - r#""namespace": "com.domain","# - } else { - "" - }, - ); + let schema_str = schema_str.replace( + "[NAMESPACE]", + if with_namespace { + r#""namespace": "com.domain","# + } else { + "" + }, + ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str)?; - #[derive(Serialize)] - enum EnumType { - #[serde(rename = "SYMBOL_1")] - Symbol1, - #[serde(rename = "SYMBOL_2")] - Symbol2, - } + #[derive(Serialize)] + enum EnumType { + #[serde(rename = "SYMBOL_1")] + Symbol1, + #[serde(rename = "SYMBOL_2")] + Symbol2, + } - #[derive(Serialize)] - struct FieldA { - enum_a: EnumType, - enum_b: EnumType, - } + #[derive(Serialize)] + struct FieldA { + enum_a: EnumType, + enum_b: EnumType, + } - #[derive(Serialize)] - struct NamespacedMessage { - field_a: FieldA, - } + #[derive(Serialize)] + struct NamespacedMessage { + field_a: FieldA, + } - let msg = NamespacedMessage { - field_a: FieldA { - enum_a: EnumType::Symbol2, - enum_b: EnumType::Symbol1, - }, - }; + let msg = NamespacedMessage { + field_a: FieldA { + enum_a: EnumType::Symbol2, + enum_b: EnumType::Symbol1, + }, + }; - let mut ser = Serializer::default(); - let test_value: Value = msg.serialize(&mut ser)?; - assert!(test_value.validate(&schema), "test_value should validate"); - assert!( - test_value.resolve(&schema).await.is_ok(), - "test_value should resolve" - ); + let mut ser = Serializer::default(); + let test_value: Value = msg.serialize(&mut ser)?; + assert!(test_value.validate(&schema), "test_value should validate"); + assert!( + test_value.resolve(&schema).await.is_ok(), + "test_value should resolve" + ); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3674_validate_no_namespace_resolution() -> TestResult { - avro_3674_with_or_without_namespace(false).await - } + #[tokio::test] + async fn test_avro_3674_validate_no_namespace_resolution() -> TestResult { + avro_3674_with_or_without_namespace(false).await + } - #[tokio::test] - async fn test_avro_3674_validate_with_namespace_resolution() -> TestResult { - avro_3674_with_or_without_namespace(true).await - } + #[tokio::test] + async fn test_avro_3674_validate_with_namespace_resolution() -> TestResult { + avro_3674_with_or_without_namespace(true).await + } - async fn avro_3688_schema_resolution_panic(set_field_b: bool) -> TestResult { - use crate::ser::Serializer; - use serde::{Deserialize, Serialize}; + async fn avro_3688_schema_resolution_panic(set_field_b: bool) -> TestResult { + use crate::ser::Serializer; + use serde::{Deserialize, Serialize}; - let schema_str = r#"{ + let schema_str = r#"{ "type": "record", "name": "Message", "fields": [ @@ -2918,441 +3032,450 @@ Field with name '"b"' is not a member of the map items"#, ] }"#; - #[derive(Serialize, Deserialize)] - struct Inner { - inner_a: String, - } + #[derive(Serialize, Deserialize)] + struct Inner { + inner_a: String, + } - #[derive(Serialize, Deserialize)] - struct Message { - field_a: Option, - field_b: Option, - } + #[derive(Serialize, Deserialize)] + struct Message { + field_a: Option, + field_b: Option, + } - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str)?; - let msg = Message { - field_a: Some(Inner { - inner_a: "foo".to_string(), - }), - field_b: if set_field_b { - Some(Inner { - inner_a: "bar".to_string(), - }) - } else { - None - }, - }; + let msg = Message { + field_a: Some(Inner { + inner_a: "foo".to_string(), + }), + field_b: if set_field_b { + Some(Inner { + inner_a: "bar".to_string(), + }) + } else { + None + }, + }; - let mut ser = Serializer::default(); - let test_value: Value = msg.serialize(&mut ser)?; - assert!(test_value.validate(&schema), "test_value should validate"); - assert!( - test_value.resolve(&schema).await.is_ok(), - "test_value should resolve" - ); + let mut ser = Serializer::default(); + let test_value: Value = msg.serialize(&mut ser)?; + assert!(test_value.validate(&schema), "test_value should validate"); + assert!( + test_value.resolve(&schema).await.is_ok(), + "test_value should resolve" + ); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3688_field_b_not_set() -> TestResult { - avro_3688_schema_resolution_panic(false).await - } + #[tokio::test] + async fn test_avro_3688_field_b_not_set() -> TestResult { + avro_3688_schema_resolution_panic(false).await + } - #[tokio::test] - async fn test_avro_3688_field_b_set() -> TestResult { - avro_3688_schema_resolution_panic(true).await - } + #[tokio::test] + async fn test_avro_3688_field_b_set() -> TestResult { + avro_3688_schema_resolution_panic(true).await + } - #[tokio::test] - async fn test_avro_3764_use_resolve_schemata() -> TestResult { - let referenced_schema = - r#"{"name": "enumForReference", "type": "enum", "symbols": ["A", "B"]}"#; - let main_schema = r#"{"name": "recordWithReference", "type": "record", "fields": [{"name": "reference", "type": "enumForReference"}]}"#; + #[tokio::test] + async fn test_avro_3764_use_resolve_schemata() -> TestResult { + let referenced_schema = + r#"{"name": "enumForReference", "type": "enum", "symbols": ["A", "B"]}"#; + let main_schema = r#"{"name": "recordWithReference", "type": "record", "fields": [{"name": "reference", "type": "enumForReference"}]}"#; - let value: serde_json::Value = serde_json::from_str( - r#" + let value: serde_json::Value = serde_json::from_str( + r#" { "reference": "A" } "#, - )?; + )?; - let avro_value = Value::from(value); + let avro_value = Value::from(value); - let schemas = Schema::parse_list([main_schema, referenced_schema])?; + let schemas = Schema::parse_list([main_schema, referenced_schema])?; - let main_schema = schemas.first().unwrap(); - let schemata: Vec<_> = schemas.iter().skip(1).collect(); + let main_schema = schemas.first().unwrap(); + let schemata: Vec<_> = schemas.iter().skip(1).collect(); - let resolve_result = avro_value.clone().resolve_schemata(main_schema, schemata).await; + let resolve_result = avro_value + .clone() + .resolve_schemata(main_schema, schemata) + .await; - assert!( - resolve_result.is_ok(), - "result of resolving with schemata should be ok, got: {resolve_result:?}" - ); + assert!( + resolve_result.is_ok(), + "result of resolving with schemata should be ok, got: {resolve_result:?}" + ); - let resolve_result = avro_value.resolve(main_schema).await; - assert!( - resolve_result.is_err(), - "result of resolving without schemata should be err, got: {resolve_result:?}" - ); + let resolve_result = avro_value.resolve(main_schema).await; + assert!( + resolve_result.is_err(), + "result of resolving without schemata should be err, got: {resolve_result:?}" + ); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3767_union_resolve_complex_refs() -> TestResult { - let referenced_enum = - r#"{"name": "enumForReference", "type": "enum", "symbols": ["A", "B"]}"#; - let referenced_record = r#"{"name": "recordForReference", "type": "record", "fields": [{"name": "refInRecord", "type": "enumForReference"}]}"#; - let main_schema = r#"{"name": "recordWithReference", "type": "record", "fields": [{"name": "reference", "type": ["null", "recordForReference"]}]}"#; + #[tokio::test] + async fn test_avro_3767_union_resolve_complex_refs() -> TestResult { + let referenced_enum = + r#"{"name": "enumForReference", "type": "enum", "symbols": ["A", "B"]}"#; + let referenced_record = r#"{"name": "recordForReference", "type": "record", "fields": [{"name": "refInRecord", "type": "enumForReference"}]}"#; + let main_schema = r#"{"name": "recordWithReference", "type": "record", "fields": [{"name": "reference", "type": ["null", "recordForReference"]}]}"#; - let value: serde_json::Value = serde_json::from_str( - r#" + let value: serde_json::Value = serde_json::from_str( + r#" { "reference": { "refInRecord": "A" } } "#, - )?; + )?; - let avro_value = Value::from(value); + let avro_value = Value::from(value); - let schemata = Schema::parse_list([referenced_enum, referenced_record, main_schema])?; + let schemata = Schema::parse_list([referenced_enum, referenced_record, main_schema])?; - let main_schema = schemata.last().unwrap(); - let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); + let main_schema = schemata.last().unwrap(); + let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); - let resolve_result = avro_value.resolve_schemata(main_schema, other_schemata).await; + let resolve_result = avro_value + .resolve_schemata(main_schema, other_schemata) + .await; - assert!( - resolve_result.is_ok(), - "result of resolving with schemata should be ok, got: {resolve_result:?}" - ); + assert!( + resolve_result.is_ok(), + "result of resolving with schemata should be ok, got: {resolve_result:?}" + ); - assert!( - resolve_result?.validate_schemata(schemata.iter().collect()), - "result of validation with schemata should be true" - ); + assert!( + resolve_result?.validate_schemata(schemata.iter().collect()), + "result of validation with schemata should be true" + ); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3782_incorrect_decimal_resolving() -> TestResult { - let schema = r#"{"name": "decimalSchema", "logicalType": "decimal", "type": "fixed", "precision": 8, "scale": 0, "size": 8}"#; - - let avro_value = Value::Decimal(Decimal::from( - BigInt::from(12345678u32).to_signed_bytes_be(), - )); - let schema = Schema::parse_str(schema)?; - let resolve_result = avro_value.resolve(&schema).await; - assert!( - resolve_result.is_ok(), - "resolve result must be ok, got: {resolve_result:?}" - ); - - Ok(()) - } + #[tokio::test] + async fn test_avro_3782_incorrect_decimal_resolving() -> TestResult { + let schema = r#"{"name": "decimalSchema", "logicalType": "decimal", "type": "fixed", "precision": 8, "scale": 0, "size": 8}"#; + + let avro_value = Value::Decimal(Decimal::from( + BigInt::from(12345678u32).to_signed_bytes_be(), + )); + let schema = Schema::parse_str(schema)?; + let resolve_result = avro_value.resolve(&schema).await; + assert!( + resolve_result.is_ok(), + "resolve result must be ok, got: {resolve_result:?}" + ); + + Ok(()) + } - #[tokio::test] - async fn test_avro_3779_bigdecimal_resolving() -> TestResult { - let schema = - r#"{"name": "bigDecimalSchema", "logicalType": "big-decimal", "type": "bytes" }"#; + #[tokio::test] + async fn test_avro_3779_bigdecimal_resolving() -> TestResult { + let schema = + r#"{"name": "bigDecimalSchema", "logicalType": "big-decimal", "type": "bytes" }"#; - let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); - let schema = Schema::parse_str(schema)?; - let resolve_result: AvroResult = avro_value.resolve(&schema).await; - assert!( - resolve_result.is_ok(), - "resolve result must be ok, got: {resolve_result:?}" - ); + let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); + let schema = Schema::parse_str(schema)?; + let resolve_result: AvroResult = avro_value.resolve(&schema).await; + assert!( + resolve_result.is_ok(), + "resolve result must be ok, got: {resolve_result:?}" + ); - Ok(()) - } + Ok(()) + } - #[tokio::test] - async fn test_avro_3892_resolve_fixed_from_bytes() -> TestResult { - let value = Value::Bytes(vec![97, 98, 99]); - assert_eq!( - value.resolve(&Schema::Fixed(FixedSchema { - name: "test".into(), - aliases: None, - doc: None, - size: 3, - default: None, - attributes: Default::default() - })).await?, - Value::Fixed(3, vec![97, 98, 99]) - ); + #[tokio::test] + async fn test_avro_3892_resolve_fixed_from_bytes() -> TestResult { + let value = Value::Bytes(vec![97, 98, 99]); + assert_eq!( + value + .resolve(&Schema::Fixed(FixedSchema { + name: "test".into(), + aliases: None, + doc: None, + size: 3, + default: None, + attributes: Default::default() + })) + .await?, + Value::Fixed(3, vec![97, 98, 99]) + ); - let value = Value::Bytes(vec![97, 99]); - assert!( - value - .resolve(&Schema::Fixed(FixedSchema { - name: "test".into(), - aliases: None, - doc: None, - size: 3, - default: None, - attributes: Default::default() - })).await - .is_err(), - ); - - let value = Value::Bytes(vec![97, 98, 99, 100]); - assert!( - value - .resolve(&Schema::Fixed(FixedSchema { - name: "test".into(), - aliases: None, - doc: None, - size: 3, - default: None, - attributes: Default::default() - })).await - .is_err(), - ); - - Ok(()) - } + let value = Value::Bytes(vec![97, 99]); + assert!( + value + .resolve(&Schema::Fixed(FixedSchema { + name: "test".into(), + aliases: None, + doc: None, + size: 3, + default: None, + attributes: Default::default() + })) + .await + .is_err(), + ); - #[tokio::test] - async fn avro_3928_from_serde_value_to_types_value() { - assert_eq!(Value::from(serde_json::Value::Null), Value::Null); - assert_eq!(Value::from(json!(true)), Value::Boolean(true)); - assert_eq!(Value::from(json!(false)), Value::Boolean(false)); - assert_eq!(Value::from(json!(0)), Value::Int(0)); - assert_eq!(Value::from(json!(i32::MIN)), Value::Int(i32::MIN)); - assert_eq!(Value::from(json!(i32::MAX)), Value::Int(i32::MAX)); - assert_eq!( - Value::from(json!(i32::MIN as i64 - 1)), - Value::Long(i32::MIN as i64 - 1) - ); - assert_eq!( - Value::from(json!(i32::MAX as i64 + 1)), - Value::Long(i32::MAX as i64 + 1) - ); - assert_eq!(Value::from(json!(1.23)), Value::Double(1.23)); - assert_eq!(Value::from(json!(-1.23)), Value::Double(-1.23)); - assert_eq!(Value::from(json!(u64::MIN)), Value::Int(u64::MIN as i32)); - assert_eq!(Value::from(json!(u64::MAX)), Value::Long(u64::MAX as i64)); - assert_eq!( - Value::from(json!("some text")), - Value::String("some text".into()) - ); - assert_eq!( - Value::from(json!(["text1", "text2", "text3"])), - Value::Array(vec![ - Value::String("text1".into()), - Value::String("text2".into()), - Value::String("text3".into()) - ]) - ); - assert_eq!( - Value::from(json!({"key1": "value1", "key2": "value2"})), - Value::Map( - vec![ - ("key1".into(), Value::String("value1".into())), - ("key2".into(), Value::String("value2".into())) - ] - .into_iter() - .collect() - ) - ); - } + let value = Value::Bytes(vec![97, 98, 99, 100]); + assert!( + value + .resolve(&Schema::Fixed(FixedSchema { + name: "test".into(), + aliases: None, + doc: None, + size: 3, + default: None, + attributes: Default::default() + })) + .await + .is_err(), + ); - #[tokio::test] - async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "double"}"#)?; - let value = Value::String("unknown".to_owned()); - match value.resolve(&schema).await.map_err(Error::into_details) { - Err(err @ Details::GetDouble(_)) => { - assert_eq!( - format!("{err:?}"), - r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: String("unknown")"# - ); - } - other => { - panic!("Expected Details::GetDouble, got {other:?}"); - } + Ok(()) } - Ok(()) - } - #[tokio::test] - async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "float"}"#)?; - let value = Value::String("unknown".to_owned()); - match value.resolve(&schema).await.map_err(Error::into_details) { - Err(err @ Details::GetFloat(_)) => { - assert_eq!( - format!("{err:?}"), - r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: String("unknown")"# - ); - } - other => { - panic!("Expected Details::GetFloat, got {other:?}"); + #[tokio::test] + async fn avro_3928_from_serde_value_to_types_value() { + assert_eq!(Value::from(serde_json::Value::Null), Value::Null); + assert_eq!(Value::from(json!(true)), Value::Boolean(true)); + assert_eq!(Value::from(json!(false)), Value::Boolean(false)); + assert_eq!(Value::from(json!(0)), Value::Int(0)); + assert_eq!(Value::from(json!(i32::MIN)), Value::Int(i32::MIN)); + assert_eq!(Value::from(json!(i32::MAX)), Value::Int(i32::MAX)); + assert_eq!( + Value::from(json!(i32::MIN as i64 - 1)), + Value::Long(i32::MIN as i64 - 1) + ); + assert_eq!( + Value::from(json!(i32::MAX as i64 + 1)), + Value::Long(i32::MAX as i64 + 1) + ); + assert_eq!(Value::from(json!(1.23)), Value::Double(1.23)); + assert_eq!(Value::from(json!(-1.23)), Value::Double(-1.23)); + assert_eq!(Value::from(json!(u64::MIN)), Value::Int(u64::MIN as i32)); + assert_eq!(Value::from(json!(u64::MAX)), Value::Long(u64::MAX as i64)); + assert_eq!( + Value::from(json!("some text")), + Value::String("some text".into()) + ); + assert_eq!( + Value::from(json!(["text1", "text2", "text3"])), + Value::Array(vec![ + Value::String("text1".into()), + Value::String("text2".into()), + Value::String("text3".into()) + ]) + ); + assert_eq!( + Value::from(json!({"key1": "value1", "key2": "value2"})), + Value::Map( + vec![ + ("key1".into(), Value::String("value1".into())), + ("key2".into(), Value::String("value2".into())) + ] + .into_iter() + .collect() + ) + ); + } + + #[tokio::test] + async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "double"}"#)?; + let value = Value::String("unknown".to_owned()); + match value.resolve(&schema).await.map_err(Error::into_details) { + Err(err @ Details::GetDouble(_)) => { + assert_eq!( + format!("{err:?}"), + r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: String("unknown")"# + ); + } + other => { + panic!("Expected Details::GetDouble, got {other:?}"); + } } + Ok(()) } - Ok(()) - } - #[tokio::test] - async fn avro_4029_resolve_from_unsupported_err() -> TestResult { - let data: Vec<(&str, Value, &str)> = vec![ - ( - r#"{ "name": "NAME", "type": "int" }"#, - Value::Float(123_f32), - "Expected Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "fixed", "size": 3 }"#, - Value::Float(123_f32), - "String expected for fixed, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "bytes" }"#, - Value::Float(123_f32), - "Expected Value::Bytes, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "string", "logicalType": "uuid" }"#, - Value::String("abc-1234".into()), - "Failed to convert &str to UUID: invalid group count: expected 5, found 2", - ), - ( - r#"{ "name": "NAME", "type": "string", "logicalType": "uuid" }"#, - Value::Float(123_f32), - "Expected Value::Uuid, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "bytes", "logicalType": "big-decimal" }"#, - Value::Float(123_f32), - "Expected Value::BigDecimal, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "fixed", "size": 12, "logicalType": "duration" }"#, - Value::Float(123_f32), - "Expected Value::Duration or Value::Fixed(12), got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3 }"#, - Value::Float(123_f32), - "Expected Value::Decimal, Value::Bytes or Value::Fixed, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "bytes" }"#, - Value::Array(vec![Value::Long(256_i64)]), - "Unable to convert to u8, got Int(256)", - ), - ( - r#"{ "name": "NAME", "type": "int", "logicalType": "date" }"#, - Value::Float(123_f32), - "Expected Value::Date or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "int", "logicalType": "time-millis" }"#, - Value::Float(123_f32), - "Expected Value::TimeMillis or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "time-micros" }"#, - Value::Float(123_f32), - "Expected Value::TimeMicros, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "timestamp-millis" }"#, - Value::Float(123_f32), - "Expected Value::TimestampMillis, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "timestamp-micros" }"#, - Value::Float(123_f32), - "Expected Value::TimestampMicros, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "timestamp-nanos" }"#, - Value::Float(123_f32), - "Expected Value::TimestampNanos, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "local-timestamp-millis" }"#, - Value::Float(123_f32), - "Expected Value::LocalTimestampMillis, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "local-timestamp-micros" }"#, - Value::Float(123_f32), - "Expected Value::LocalTimestampMicros, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long", "logicalType": "local-timestamp-nanos" }"#, - Value::Float(123_f32), - "Expected Value::LocalTimestampNanos, Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "null" }"#, - Value::Float(123_f32), - "Expected Value::Null, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "boolean" }"#, - Value::Float(123_f32), - "Expected Value::Boolean, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "int" }"#, - Value::Float(123_f32), - "Expected Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "long" }"#, - Value::Float(123_f32), - "Expected Value::Long or Value::Int, got: Float(123.0)", - ), - ( - r#"{ "name": "NAME", "type": "float" }"#, - Value::Boolean(false), - r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: Boolean(false)"#, - ), - ( - r#"{ "name": "NAME", "type": "double" }"#, - Value::Boolean(false), - r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: Boolean(false)"#, - ), - ( - r#"{ "name": "NAME", "type": "string" }"#, - Value::Boolean(false), - "Expected Value::String, Value::Bytes or Value::Fixed, got: Boolean(false)", - ), - ( - r#"{ "name": "NAME", "type": "enum", "symbols": ["one", "two"] }"#, - Value::Boolean(false), - "Expected Value::Enum, got: Boolean(false)", - ), - ]; - - for (schema_str, value, expected_error) in data { - let schema = Schema::parse_str(schema_str)?; - match value.resolve(&schema).await { - Err(error) => { - assert_eq!(format!("{error}"), expected_error); + #[tokio::test] + async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "float"}"#)?; + let value = Value::String("unknown".to_owned()); + match value.resolve(&schema).await.map_err(Error::into_details) { + Err(err @ Details::GetFloat(_)) => { + assert_eq!( + format!("{err:?}"), + r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: String("unknown")"# + ); } other => { - panic!("Expected '{expected_error}', got {other:?}"); + panic!("Expected Details::GetFloat, got {other:?}"); } } + Ok(()) + } + + #[tokio::test] + async fn avro_4029_resolve_from_unsupported_err() -> TestResult { + let data: Vec<(&str, Value, &str)> = vec![ + ( + r#"{ "name": "NAME", "type": "int" }"#, + Value::Float(123_f32), + "Expected Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "fixed", "size": 3 }"#, + Value::Float(123_f32), + "String expected for fixed, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "bytes" }"#, + Value::Float(123_f32), + "Expected Value::Bytes, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "string", "logicalType": "uuid" }"#, + Value::String("abc-1234".into()), + "Failed to convert &str to UUID: invalid group count: expected 5, found 2", + ), + ( + r#"{ "name": "NAME", "type": "string", "logicalType": "uuid" }"#, + Value::Float(123_f32), + "Expected Value::Uuid, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "bytes", "logicalType": "big-decimal" }"#, + Value::Float(123_f32), + "Expected Value::BigDecimal, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "fixed", "size": 12, "logicalType": "duration" }"#, + Value::Float(123_f32), + "Expected Value::Duration or Value::Fixed(12), got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3 }"#, + Value::Float(123_f32), + "Expected Value::Decimal, Value::Bytes or Value::Fixed, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "bytes" }"#, + Value::Array(vec![Value::Long(256_i64)]), + "Unable to convert to u8, got Int(256)", + ), + ( + r#"{ "name": "NAME", "type": "int", "logicalType": "date" }"#, + Value::Float(123_f32), + "Expected Value::Date or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "int", "logicalType": "time-millis" }"#, + Value::Float(123_f32), + "Expected Value::TimeMillis or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "time-micros" }"#, + Value::Float(123_f32), + "Expected Value::TimeMicros, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "timestamp-millis" }"#, + Value::Float(123_f32), + "Expected Value::TimestampMillis, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "timestamp-micros" }"#, + Value::Float(123_f32), + "Expected Value::TimestampMicros, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "timestamp-nanos" }"#, + Value::Float(123_f32), + "Expected Value::TimestampNanos, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "local-timestamp-millis" }"#, + Value::Float(123_f32), + "Expected Value::LocalTimestampMillis, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "local-timestamp-micros" }"#, + Value::Float(123_f32), + "Expected Value::LocalTimestampMicros, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long", "logicalType": "local-timestamp-nanos" }"#, + Value::Float(123_f32), + "Expected Value::LocalTimestampNanos, Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "null" }"#, + Value::Float(123_f32), + "Expected Value::Null, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "boolean" }"#, + Value::Float(123_f32), + "Expected Value::Boolean, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "int" }"#, + Value::Float(123_f32), + "Expected Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "long" }"#, + Value::Float(123_f32), + "Expected Value::Long or Value::Int, got: Float(123.0)", + ), + ( + r#"{ "name": "NAME", "type": "float" }"#, + Value::Boolean(false), + r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: Boolean(false)"#, + ), + ( + r#"{ "name": "NAME", "type": "double" }"#, + Value::Boolean(false), + r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: Boolean(false)"#, + ), + ( + r#"{ "name": "NAME", "type": "string" }"#, + Value::Boolean(false), + "Expected Value::String, Value::Bytes or Value::Fixed, got: Boolean(false)", + ), + ( + r#"{ "name": "NAME", "type": "enum", "symbols": ["one", "two"] }"#, + Value::Boolean(false), + "Expected Value::Enum, got: Boolean(false)", + ), + ]; + + for (schema_str, value, expected_error) in data { + let schema = Schema::parse_str(schema_str)?; + match value.resolve(&schema).await { + Err(error) => { + assert_eq!(format!("{error}"), expected_error); + } + other => { + panic!("Expected '{expected_error}', got {other:?}"); + } + } + } + Ok(()) } - Ok(()) - } - #[tokio::test] - async fn avro_rs_130_get_from_record() -> TestResult { - let schema = r#" + #[tokio::test] + async fn avro_rs_130_get_from_record() -> TestResult { + let schema = r#" { "type": "record", "name": "NamespacedMessage", @@ -3370,20 +3493,21 @@ Field with name '"b"' is not a member of the map items"#, } "#; - let schema = Schema::parse_str(schema)?; - let mut record = Record::new(&schema).unwrap(); - record.put("foo", "hello"); - record.put("bar", 123_i64); + let schema = Schema::parse_str(schema)?; + let mut record = Record::new(&schema).unwrap(); + record.put("foo", "hello"); + record.put("bar", 123_i64); - assert_eq!( - record.get("foo").unwrap(), - &Value::String("hello".to_string()) - ); - assert_eq!(record.get("bar").unwrap(), &Value::Long(123)); + assert_eq!( + record.get("foo").unwrap(), + &Value::String("hello".to_string()) + ); + assert_eq!(record.get("bar").unwrap(), &Value::Long(123)); - // also make sure it doesn't fail but return None for non-existing field - assert_eq!(record.get("baz"), None); + // also make sure it doesn't fail but return None for non-existing field + assert_eq!(record.get("baz"), None); - Ok(()) + Ok(()) + } } } diff --git a/avro/src/util.rs b/avro/src/util.rs index 020adf54..379b4645 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -15,15 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::{ - sync::{ - Once, - atomic::{AtomicBool, AtomicUsize}, - }, -}; -use std::sync::atomic::Ordering; use crate::AvroResult; -use crate::error::Details; +use crate::error::sync::Details; +use std::sync::atomic::Ordering; +use std::sync::{ + Once, + atomic::{AtomicBool, AtomicUsize}, +}; /// Maximum number of bytes that can be allocated when decoding /// Avro-encoded values. This is a protection against ill-formed @@ -67,7 +65,7 @@ pub fn safe_len(len: usize) -> AvroResult { desired: len, maximum: max_bytes, } - .into()) + .into()) } } @@ -92,22 +90,31 @@ pub fn set_serde_human_readable(human_readable: bool) { pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, #[tokio::test] => #[test] ); } )] mod util { - #[synca::cfg(tokio)] + #[cfg(feature = "tokio")] + use futures::future::TryFutureExt; + #[cfg(feature = "sync")] + use std::io::Read as AvroRead; + #[cfg(feature = "tokio")] + use tokio::io::AsyncRead as AvroRead; + #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; - use std::{ - io::{Write}, - }; - use crate::schema::Documentation; - use serde_json::{Map, Value}; use crate::AvroResult; - use crate::error::Details; + use crate::error::tokio::Details; + use crate::schema::tokio::Documentation; + use serde_json::{Map, Value}; + use std::io::Write; pub trait MapHelper { fn string(&self, key: &str) -> Option; @@ -144,7 +151,7 @@ mod util { } } - pub async fn read_long(reader: &mut R) -> AvroResult { + pub async fn read_long(reader: &mut R) -> AvroResult { zag_i64(reader).await } @@ -156,12 +163,12 @@ mod util { encode_variable(((n << 1) ^ (n >> 63)) as u64, writer) } - pub async fn zag_i32(reader: &mut R) -> AvroResult { + pub async fn zag_i32(reader: &mut R) -> AvroResult { let i = zag_i64(reader).await?; i32::try_from(i).map_err(|e| Details::ZagI32(e, i).into()) } - pub async fn zag_i64(reader: &mut R) -> AvroResult { + pub async fn zag_i64(reader: &mut R) -> AvroResult { let z = decode_variable(reader).await?; Ok(if z & 0x1 == 0 { (z >> 1) as i64 @@ -189,7 +196,7 @@ mod util { .map_err(|e| Details::WriteBytes(e).into()) } - async fn decode_variable(reader: &mut R) -> AvroResult { + async fn decode_variable(reader: &mut R) -> AvroResult { let mut i = 0u64; let mut buf = [0u8; 1]; @@ -200,8 +207,9 @@ mod util { return Err(Details::IntegerOverflow.into()); } reader - .read_exact(&mut buf[..]).await - .map_err(Details::ReadVariableIntegerBytes)?; + .read_exact(&mut buf[..]) + .map_err(Details::ReadVariableIntegerBytes) + .await?; i |= (u64::from(buf[0] & 0x7F)) << (j * 7); if (buf[0] >> 7) == 0 { break; @@ -213,13 +221,12 @@ mod util { Ok(i) } - #[cfg(test)] mod tests { use super::*; + use crate::util::safe_len; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; - use crate::util::safe_len; #[test] fn test_zigzag() { diff --git a/avro/src/validator.rs b/avro/src/validator.rs index 971870cb..94c6d359 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -15,305 +15,327 @@ // specific language governing permissions and limitations // under the License. -use crate::{AvroResult, error::Details, schema::Namespace}; -use log::debug; -use regex_lite::Regex; -use std::sync::OnceLock; - -/// A validator that validates names and namespaces according to the Avro specification. -struct SpecificationValidator; - -/// A trait that validates schema names. -/// To register a custom one use [set_schema_name_validator]. -pub trait SchemaNameValidator: Send + Sync { - /// Returns the regex used to validate the schema name - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static SCHEMA_NAME_ONCE: OnceLock = OnceLock::new(); - SCHEMA_NAME_ONCE.get_or_init(|| { +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + schema::tokio => schema::sync, + #[tokio::test] => #[test] + ); + } +)] +mod validator { + + use crate::{AvroResult, error::tokio::Details, schema::tokio::Namespace}; + use log::debug; + use regex_lite::Regex; + use std::sync::OnceLock; + + /// A validator that validates names and namespaces according to the Avro specification. + struct SpecificationValidator; + + /// A trait that validates schema names. + /// To register a custom one use [set_schema_name_validator]. + pub trait SchemaNameValidator: Send + Sync { + /// Returns the regex used to validate the schema name + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static SCHEMA_NAME_ONCE: OnceLock = OnceLock::new(); + SCHEMA_NAME_ONCE.get_or_init(|| { Regex::new( // An optional namespace (with optional dots) followed by a name without any dots in it. r"^((?P([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?)\.)?(?P[A-Za-z_][A-Za-z0-9_]*)$", ) .unwrap() }) + } + + /// Validates the schema name and returns the name and the optional namespace, + /// or [Details::InvalidSchemaName] if it is invalid. + fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)>; } - /// Validates the schema name and returns the name and the optional namespace, - /// or [Details::InvalidSchemaName] if it is invalid. - fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)>; -} + impl SchemaNameValidator for SpecificationValidator { + fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)> { + let regex = SchemaNameValidator::regex(self); + let caps = regex.captures(schema_name).ok_or_else(|| { + Details::InvalidSchemaName(schema_name.to_string(), regex.as_str()) + })?; + Ok(( + caps["name"].to_string(), + caps.name("namespace").map(|s| s.as_str().to_string()), + )) + } + } -impl SchemaNameValidator for SpecificationValidator { - fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)> { - let regex = SchemaNameValidator::regex(self); - let caps = regex - .captures(schema_name) - .ok_or_else(|| Details::InvalidSchemaName(schema_name.to_string(), regex.as_str()))?; - Ok(( - caps["name"].to_string(), - caps.name("namespace").map(|s| s.as_str().to_string()), - )) + static NAME_VALIDATOR_ONCE: OnceLock> = + OnceLock::new(); + + /// Sets a custom schema name validator. + /// + /// Returns a unit if the registration was successful or the already + /// registered validator if the registration failed. + /// + /// **Note**: This function must be called before parsing any schema because this will + /// register the default validator and the registration is one time only! + pub fn set_schema_name_validator( + validator: Box, + ) -> Result<(), Box> { + debug!("Setting a custom schema name validator."); + NAME_VALIDATOR_ONCE.set(validator) } -} -static NAME_VALIDATOR_ONCE: OnceLock> = OnceLock::new(); - -/// Sets a custom schema name validator. -/// -/// Returns a unit if the registration was successful or the already -/// registered validator if the registration failed. -/// -/// **Note**: This function must be called before parsing any schema because this will -/// register the default validator and the registration is one time only! -pub fn set_schema_name_validator( - validator: Box, -) -> Result<(), Box> { - debug!("Setting a custom schema name validator."); - NAME_VALIDATOR_ONCE.set(validator) -} + pub(crate) fn validate_schema_name(schema_name: &str) -> AvroResult<(String, Namespace)> { + NAME_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default name validator."); + Box::new(SpecificationValidator) + }) + .validate(schema_name) + } -pub(crate) fn validate_schema_name(schema_name: &str) -> AvroResult<(String, Namespace)> { - NAME_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default name validator."); - Box::new(SpecificationValidator) - }) - .validate(schema_name) -} + /// A trait that validates schema namespaces. + /// To register a custom one use [set_schema_namespace_validator]. + pub trait SchemaNamespaceValidator: Send + Sync { + /// Returns the regex used to validate the schema namespace + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static NAMESPACE_ONCE: OnceLock = OnceLock::new(); + NAMESPACE_ONCE.get_or_init(|| { + Regex::new(r"^([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?$").unwrap() + }) + } -/// A trait that validates schema namespaces. -/// To register a custom one use [set_schema_namespace_validator]. -pub trait SchemaNamespaceValidator: Send + Sync { - /// Returns the regex used to validate the schema namespace - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static NAMESPACE_ONCE: OnceLock = OnceLock::new(); - NAMESPACE_ONCE.get_or_init(|| { - Regex::new(r"^([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?$").unwrap() - }) + /// Validates the schema namespace or [Details::InvalidNamespace] if it is invalid. + fn validate(&self, namespace: &str) -> AvroResult<()>; } - /// Validates the schema namespace or [Details::InvalidNamespace] if it is invalid. - fn validate(&self, namespace: &str) -> AvroResult<()>; -} - -impl SchemaNamespaceValidator for SpecificationValidator { - fn validate(&self, ns: &str) -> AvroResult<()> { - let regex = SchemaNamespaceValidator::regex(self); - if !regex.is_match(ns) { - Err(Details::InvalidNamespace(ns.to_string(), regex.as_str()).into()) - } else { - Ok(()) + impl SchemaNamespaceValidator for SpecificationValidator { + fn validate(&self, ns: &str) -> AvroResult<()> { + let regex = SchemaNamespaceValidator::regex(self); + if !regex.is_match(ns) { + Err(Details::InvalidNamespace(ns.to_string(), regex.as_str()).into()) + } else { + Ok(()) + } } } -} - -static NAMESPACE_VALIDATOR_ONCE: OnceLock> = - OnceLock::new(); - -/// Sets a custom schema namespace validator. -/// -/// Returns a unit if the registration was successful or the already -/// registered validator if the registration failed. -/// -/// **Note**: This function must be called before parsing any schema because this will -/// register the default validator and the registration is one time only! -pub fn set_schema_namespace_validator( - validator: Box, -) -> Result<(), Box> { - NAMESPACE_VALIDATOR_ONCE.set(validator) -} -pub(crate) fn validate_namespace(ns: &str) -> AvroResult<()> { - NAMESPACE_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default namespace validator."); - Box::new(SpecificationValidator) - }) - .validate(ns) -} - -/// A trait that validates enum symbol names. -/// To register a custom one use [set_enum_symbol_name_validator]. -pub trait EnumSymbolNameValidator: Send + Sync { - /// Returns the regex used to validate the symbols of enum schema - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static ENUM_SYMBOL_NAME_ONCE: OnceLock = OnceLock::new(); - ENUM_SYMBOL_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) + static NAMESPACE_VALIDATOR_ONCE: OnceLock> = + OnceLock::new(); + + /// Sets a custom schema namespace validator. + /// + /// Returns a unit if the registration was successful or the already + /// registered validator if the registration failed. + /// + /// **Note**: This function must be called before parsing any schema because this will + /// register the default validator and the registration is one time only! + pub fn set_schema_namespace_validator( + validator: Box, + ) -> Result<(), Box> { + NAMESPACE_VALIDATOR_ONCE.set(validator) } - /// Validates the symbols of an Enum schema name and returns nothing (unit), - /// or [Details::EnumSymbolName] if it is invalid. - fn validate(&self, name: &str) -> AvroResult<()>; -} + pub(crate) fn validate_namespace(ns: &str) -> AvroResult<()> { + NAMESPACE_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default namespace validator."); + Box::new(SpecificationValidator) + }) + .validate(ns) + } -impl EnumSymbolNameValidator for SpecificationValidator { - fn validate(&self, symbol: &str) -> AvroResult<()> { - let regex = EnumSymbolNameValidator::regex(self); - if !regex.is_match(symbol) { - return Err(Details::EnumSymbolName(symbol.to_string()).into()); + /// A trait that validates enum symbol names. + /// To register a custom one use [set_enum_symbol_name_validator]. + pub trait EnumSymbolNameValidator: Send + Sync { + /// Returns the regex used to validate the symbols of enum schema + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static ENUM_SYMBOL_NAME_ONCE: OnceLock = OnceLock::new(); + ENUM_SYMBOL_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) } - Ok(()) + /// Validates the symbols of an Enum schema name and returns nothing (unit), + /// or [Details::EnumSymbolName] if it is invalid. + fn validate(&self, name: &str) -> AvroResult<()>; } -} -static ENUM_SYMBOL_NAME_VALIDATOR_ONCE: OnceLock> = - OnceLock::new(); - -/// Sets a custom enum symbol name validator. -/// -/// Returns a unit if the registration was successful or the already -/// registered validator if the registration failed. -/// -/// **Note**: This function must be called before parsing any schema because this will -/// register the default validator and the registration is one time only! -pub fn set_enum_symbol_name_validator( - validator: Box, -) -> Result<(), Box> { - ENUM_SYMBOL_NAME_VALIDATOR_ONCE.set(validator) -} + impl EnumSymbolNameValidator for SpecificationValidator { + fn validate(&self, symbol: &str) -> AvroResult<()> { + let regex = EnumSymbolNameValidator::regex(self); + if !regex.is_match(symbol) { + return Err(Details::EnumSymbolName(symbol.to_string()).into()); + } -pub(crate) fn validate_enum_symbol_name(symbol: &str) -> AvroResult<()> { - ENUM_SYMBOL_NAME_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default enum symbol name validator."); - Box::new(SpecificationValidator) - }) - .validate(symbol) -} + Ok(()) + } + } -/// A trait that validates record field names. -/// To register a custom one use [set_record_field_name_validator]. -pub trait RecordFieldNameValidator: Send + Sync { - /// Returns the regex used to validate the record field names - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static FIELD_NAME_ONCE: OnceLock = OnceLock::new(); - FIELD_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) + static ENUM_SYMBOL_NAME_VALIDATOR_ONCE: OnceLock< + Box, + > = OnceLock::new(); + + /// Sets a custom enum symbol name validator. + /// + /// Returns a unit if the registration was successful or the already + /// registered validator if the registration failed. + /// + /// **Note**: This function must be called before parsing any schema because this will + /// register the default validator and the registration is one time only! + pub fn set_enum_symbol_name_validator( + validator: Box, + ) -> Result<(), Box> { + ENUM_SYMBOL_NAME_VALIDATOR_ONCE.set(validator) } - /// Validates the record field's names and returns nothing (unit), - /// or [Details::FieldName] if it is invalid. - fn validate(&self, name: &str) -> AvroResult<()>; -} + pub(crate) fn validate_enum_symbol_name(symbol: &str) -> AvroResult<()> { + ENUM_SYMBOL_NAME_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default enum symbol name validator."); + Box::new(SpecificationValidator) + }) + .validate(symbol) + } -impl RecordFieldNameValidator for SpecificationValidator { - fn validate(&self, field_name: &str) -> AvroResult<()> { - let regex = RecordFieldNameValidator::regex(self); - if !regex.is_match(field_name) { - return Err(Details::FieldName(field_name.to_string()).into()); + /// A trait that validates record field names. + /// To register a custom one use [set_record_field_name_validator]. + pub trait RecordFieldNameValidator: Send + Sync { + /// Returns the regex used to validate the record field names + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static FIELD_NAME_ONCE: OnceLock = OnceLock::new(); + FIELD_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) } - Ok(()) + /// Validates the record field's names and returns nothing (unit), + /// or [Details::FieldName] if it is invalid. + fn validate(&self, name: &str) -> AvroResult<()>; } -} -static RECORD_FIELD_NAME_VALIDATOR_ONCE: OnceLock> = - OnceLock::new(); - -/// Sets a custom record field name validator. -/// -/// Returns a unit if the registration was successful or the already -/// registered validator if the registration failed. -/// -/// **Note**: This function must be called before parsing any schema because this will -/// register the default validator and the registration is one time only! -pub fn set_record_field_name_validator( - validator: Box, -) -> Result<(), Box> { - RECORD_FIELD_NAME_VALIDATOR_ONCE.set(validator) -} + impl RecordFieldNameValidator for SpecificationValidator { + fn validate(&self, field_name: &str) -> AvroResult<()> { + let regex = RecordFieldNameValidator::regex(self); + if !regex.is_match(field_name) { + return Err(Details::FieldName(field_name.to_string()).into()); + } -pub(crate) fn validate_record_field_name(field_name: &str) -> AvroResult<()> { - RECORD_FIELD_NAME_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default record field name validator."); - Box::new(SpecificationValidator) - }) - .validate(field_name) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::schema::Name; - use apache_avro_test_helper::TestResult; - - #[test] - fn avro_3900_default_name_validator_with_valid_ns() -> TestResult { - validate_schema_name("example")?; - Ok(()) + Ok(()) + } } - #[test] - fn avro_3900_default_name_validator_with_invalid_ns() -> TestResult { - assert!(validate_schema_name("com-example").is_err()); - Ok(()) + static RECORD_FIELD_NAME_VALIDATOR_ONCE: OnceLock< + Box, + > = OnceLock::new(); + + /// Sets a custom record field name validator. + /// + /// Returns a unit if the registration was successful or the already + /// registered validator if the registration failed. + /// + /// **Note**: This function must be called before parsing any schema because this will + /// register the default validator and the registration is one time only! + pub fn set_record_field_name_validator( + validator: Box, + ) -> Result<(), Box> { + RECORD_FIELD_NAME_VALIDATOR_ONCE.set(validator) } - #[test] - fn test_avro_3897_disallow_invalid_namespaces_in_fully_qualified_name() -> TestResult { - let full_name = "ns.0.record1"; - let name = Name::new(full_name); - assert!(name.is_err()); - let validator = SpecificationValidator; - let expected = Details::InvalidSchemaName( - full_name.to_string(), - SchemaNameValidator::regex(&validator).as_str(), - ) - .to_string(); - let err = name.map_err(|e| e.to_string()).err().unwrap(); - pretty_assertions::assert_eq!(expected, err); - - let full_name = "ns..record1"; - let name = Name::new(full_name); - assert!(name.is_err()); - let expected = Details::InvalidSchemaName( - full_name.to_string(), - SchemaNameValidator::regex(&validator).as_str(), - ) - .to_string(); - let err = name.map_err(|e| e.to_string()).err().unwrap(); - pretty_assertions::assert_eq!(expected, err); - Ok(()) + pub(crate) fn validate_record_field_name(field_name: &str) -> AvroResult<()> { + RECORD_FIELD_NAME_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default record field name validator."); + Box::new(SpecificationValidator) + }) + .validate(field_name) } - #[test] - fn avro_3900_default_namespace_validator_with_valid_ns() -> TestResult { - validate_namespace("com.example")?; - Ok(()) - } + #[cfg(test)] + mod tests { + use super::*; + use crate::schema::Name; + use apache_avro_test_helper::TestResult; - #[test] - fn avro_3900_default_namespace_validator_with_invalid_ns() -> TestResult { - assert!(validate_namespace("com-example").is_err()); - Ok(()) - } + #[test] + fn avro_3900_default_name_validator_with_valid_ns() -> TestResult { + validate_schema_name("example")?; + Ok(()) + } - #[test] - fn avro_3900_default_enum_symbol_validator_with_valid_symbol_name() -> TestResult { - validate_enum_symbol_name("spades")?; - Ok(()) - } + #[test] + fn avro_3900_default_name_validator_with_invalid_ns() -> TestResult { + assert!(validate_schema_name("com-example").is_err()); + Ok(()) + } - #[test] - fn avro_3900_default_enum_symbol_validator_with_invalid_symbol_name() -> TestResult { - assert!(validate_enum_symbol_name("com-example").is_err()); - Ok(()) - } + #[test] + fn test_avro_3897_disallow_invalid_namespaces_in_fully_qualified_name() -> TestResult { + let full_name = "ns.0.record1"; + let name = Name::new(full_name); + assert!(name.is_err()); + let validator = SpecificationValidator; + let expected = Details::InvalidSchemaName( + full_name.to_string(), + SchemaNameValidator::regex(&validator).as_str(), + ) + .to_string(); + let err = name.map_err(|e| e.to_string()).err().unwrap(); + pretty_assertions::assert_eq!(expected, err); + + let full_name = "ns..record1"; + let name = Name::new(full_name); + assert!(name.is_err()); + let expected = Details::InvalidSchemaName( + full_name.to_string(), + SchemaNameValidator::regex(&validator).as_str(), + ) + .to_string(); + let err = name.map_err(|e| e.to_string()).err().unwrap(); + pretty_assertions::assert_eq!(expected, err); + Ok(()) + } - #[test] - fn avro_3900_default_record_field_validator_with_valid_name() -> TestResult { - validate_record_field_name("test")?; - Ok(()) - } + #[test] + fn avro_3900_default_namespace_validator_with_valid_ns() -> TestResult { + validate_namespace("com.example")?; + Ok(()) + } + + #[test] + fn avro_3900_default_namespace_validator_with_invalid_ns() -> TestResult { + assert!(validate_namespace("com-example").is_err()); + Ok(()) + } + + #[test] + fn avro_3900_default_enum_symbol_validator_with_valid_symbol_name() -> TestResult { + validate_enum_symbol_name("spades")?; + Ok(()) + } - #[test] - fn avro_3900_default_record_field_validator_with_invalid_name() -> TestResult { - assert!(validate_record_field_name("com-example").is_err()); - Ok(()) + #[test] + fn avro_3900_default_enum_symbol_validator_with_invalid_symbol_name() -> TestResult { + assert!(validate_enum_symbol_name("com-example").is_err()); + Ok(()) + } + + #[test] + fn avro_3900_default_record_field_validator_with_valid_name() -> TestResult { + validate_record_field_name("test")?; + Ok(()) + } + + #[test] + fn avro_3900_default_record_field_validator_with_invalid_name() -> TestResult { + assert!(validate_record_field_name("com-example").is_err()); + Ok(()) + } } } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 9c879918..e9977f59 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -16,782 +16,813 @@ // under the License. //! Logic handling writing in Avro format at user level. -use crate::{ - AvroResult, Codec, Error, - encode::{encode, encode_internal, encode_to_vec}, - error::Details, - headers::{HeaderBuilder, RabinFingerprintHeader}, - schema::{AvroSchema, Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, - ser_schema::SchemaAwareWriteSerializer, - types::Value, -}; -use serde::Serialize; -use std::{ - collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive, -}; - -const DEFAULT_BLOCK_SIZE: usize = 16000; -const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; - -/// Main interface for writing Avro formatted values. -/// -/// It is critical to call flush before `Writer` is dropped. Though dropping will attempt to flush -/// the contents of the buffer, any errors that happen in the process of dropping will be ignored. -/// Calling flush ensures that the buffer is empty and thus dropping will not even attempt file operations. -#[derive(bon::Builder)] -pub struct Writer<'a, W: Write> { - schema: &'a Schema, - writer: W, - #[builder(skip)] - resolved_schema: Option>, - #[builder(default = Codec::Null)] - codec: Codec, - #[builder(default = DEFAULT_BLOCK_SIZE)] - block_size: usize, - #[builder(skip = Vec::with_capacity(block_size))] - buffer: Vec, - #[builder(skip)] - num_values: usize, - #[builder(default = generate_sync_marker())] - marker: [u8; 16], - /// Has the header already been written. - /// - /// To disable writing the header, this can be set to `true`. - #[builder(default = false)] - has_header: bool, - #[builder(default)] - user_metadata: HashMap, -} -impl<'a, W: Write> Writer<'a, W> { - /// Creates a `Writer` given a `Schema` and something implementing the `io::Write` trait to write - /// to. - /// No compression `Codec` will be used. - pub fn new(schema: &'a Schema, writer: W) -> Self { - Writer::with_codec(schema, writer, Codec::Null) - } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + tokio::io::AsyncRead => std::io::Read, + bigdecimal::tokio => bigdecimal::sync, + decode::tokio => decode::sync, + encode::tokio => encode::sync, + error::tokio => error::sync, + headers::tokio => headers::sync, + schema::tokio => schema::sync, + util::tokio => util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod writer { - /// Creates a `Writer` with a specific `Codec` given a `Schema` and something implementing the - /// `io::Write` trait to write to. - pub fn with_codec(schema: &'a Schema, writer: W, codec: Codec) -> Self { - let mut w = Self::builder() - .schema(schema) - .writer(writer) - .codec(codec) - .build(); - w.resolved_schema = ResolvedSchema::try_from(schema).ok(); - w - } + use crate::{ + AvroResult, Codec, Error, + encode::tokio::{encode, encode_internal, encode_to_vec}, + error::tokio::Details, + headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, + schema::tokio::{AvroSchema, Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, + ser_schema::tokio::SchemaAwareWriteSerializer, + types::tokio::Value, + }; + use serde::Serialize; + use std::{ + collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, + ops::RangeInclusive, + }; - /// Creates a `Writer` with a specific `Codec` given a `Schema` and something implementing the - /// `io::Write` trait to write to. - /// If the `schema` is incomplete, i.e. contains `Schema::Ref`s then all dependencies must - /// be provided in `schemata`. - pub fn with_schemata( - schema: &'a Schema, - schemata: Vec<&'a Schema>, - writer: W, - codec: Codec, - ) -> Self { - let mut w = Self::builder() - .schema(schema) - .writer(writer) - .codec(codec) - .build(); - w.resolved_schema = ResolvedSchema::try_from(schemata).ok(); - w - } + const DEFAULT_BLOCK_SIZE: usize = 16000; + const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; - /// Creates a `Writer` that will append values to already populated - /// `std::io::Write` using the provided `marker` - /// No compression `Codec` will be used. - pub fn append_to(schema: &'a Schema, writer: W, marker: [u8; 16]) -> Self { - Writer::append_to_with_codec(schema, writer, Codec::Null, marker) - } - - /// Creates a `Writer` that will append values to already populated - /// `std::io::Write` using the provided `marker` - pub fn append_to_with_codec( + /// Main interface for writing Avro formatted values. + /// + /// It is critical to call flush before `Writer` is dropped. Though dropping will attempt to flush + /// the contents of the buffer, any errors that happen in the process of dropping will be ignored. + /// Calling flush ensures that the buffer is empty and thus dropping will not even attempt file operations. + #[derive(bon::Builder)] + pub struct Writer<'a, W: Write> { schema: &'a Schema, writer: W, + #[builder(skip)] + resolved_schema: Option>, + #[builder(default = Codec::Null)] codec: Codec, + #[builder(default = DEFAULT_BLOCK_SIZE)] + block_size: usize, + #[builder(skip = Vec::with_capacity(block_size))] + buffer: Vec, + #[builder(skip)] + num_values: usize, + #[builder(default = generate_sync_marker())] marker: [u8; 16], - ) -> Self { - let mut w = Self::builder() - .schema(schema) - .writer(writer) - .codec(codec) - .marker(marker) - .has_header(true) - .build(); - w.resolved_schema = ResolvedSchema::try_from(schema).ok(); - w - } + /// Has the header already been written. + /// + /// To disable writing the header, this can be set to `true`. + #[builder(default = false)] + has_header: bool, + #[builder(default)] + user_metadata: HashMap, + } + + impl<'a, W: Write> Writer<'a, W> { + /// Creates a `Writer` given a `Schema` and something implementing the `io::Write` trait to write + /// to. + /// No compression `Codec` will be used. + pub fn new(schema: &'a Schema, writer: W) -> Self { + Writer::with_codec(schema, writer, Codec::Null) + } - /// Creates a `Writer` that will append values to already populated - /// `std::io::Write` using the provided `marker` - pub fn append_to_with_codec_schemata( - schema: &'a Schema, - schemata: Vec<&'a Schema>, - writer: W, - codec: Codec, - marker: [u8; 16], - ) -> Self { - let mut w = Self::builder() - .schema(schema) - .writer(writer) - .codec(codec) - .marker(marker) - .has_header(true) - .build(); - w.resolved_schema = ResolvedSchema::try_from(schemata).ok(); - w - } + /// Creates a `Writer` with a specific `Codec` given a `Schema` and something implementing the + /// `io::Write` trait to write to. + pub fn with_codec(schema: &'a Schema, writer: W, codec: Codec) -> Self { + let mut w = Self::builder() + .schema(schema) + .writer(writer) + .codec(codec) + .build(); + w.resolved_schema = ResolvedSchema::try_from(schema).ok(); + w + } - /// Get a reference to the `Schema` associated to a `Writer`. - pub fn schema(&self) -> &'a Schema { - self.schema - } + /// Creates a `Writer` with a specific `Codec` given a `Schema` and something implementing the + /// `io::Write` trait to write to. + /// If the `schema` is incomplete, i.e. contains `Schema::Ref`s then all dependencies must + /// be provided in `schemata`. + pub fn with_schemata( + schema: &'a Schema, + schemata: Vec<&'a Schema>, + writer: W, + codec: Codec, + ) -> Self { + let mut w = Self::builder() + .schema(schema) + .writer(writer) + .codec(codec) + .build(); + w.resolved_schema = ResolvedSchema::try_from(schemata).ok(); + w + } - /// Append a compatible value (implementing the `ToAvro` trait) to a `Writer`, also performing - /// schema validation. - /// - /// Returns the number of bytes written (it might be 0, see below). - /// - /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on - /// internal buffering for performance reasons. If you want to be sure the value has been - /// written, then call [`flush`](Writer::flush). - pub fn append>(&mut self, value: T) -> AvroResult { - let n = self.maybe_write_header()?; + /// Creates a `Writer` that will append values to already populated + /// `std::io::Write` using the provided `marker` + /// No compression `Codec` will be used. + pub fn append_to(schema: &'a Schema, writer: W, marker: [u8; 16]) -> Self { + Writer::append_to_with_codec(schema, writer, Codec::Null, marker) + } - let avro = value.into(); - self.append_value_ref(&avro).map(|m| m + n) - } + /// Creates a `Writer` that will append values to already populated + /// `std::io::Write` using the provided `marker` + pub fn append_to_with_codec( + schema: &'a Schema, + writer: W, + codec: Codec, + marker: [u8; 16], + ) -> Self { + let mut w = Self::builder() + .schema(schema) + .writer(writer) + .codec(codec) + .marker(marker) + .has_header(true) + .build(); + w.resolved_schema = ResolvedSchema::try_from(schema).ok(); + w + } - /// Append a compatible value to a `Writer`, also performing schema validation. - /// - /// Returns the number of bytes written (it might be 0, see below). - /// - /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on - /// internal buffering for performance reasons. If you want to be sure the value has been - /// written, then call [`flush`](Writer::flush). - pub fn append_value_ref(&mut self, value: &Value) -> AvroResult { - let n = self.maybe_write_header()?; - - // Lazy init for users using the builder pattern with error throwing - match self.resolved_schema { - Some(ref rs) => { - write_value_ref_resolved(self.schema, rs, value, &mut self.buffer)?; - self.num_values += 1; - - if self.buffer.len() >= self.block_size { - return self.flush().map(|b| b + n); - } + /// Creates a `Writer` that will append values to already populated + /// `std::io::Write` using the provided `marker` + pub fn append_to_with_codec_schemata( + schema: &'a Schema, + schemata: Vec<&'a Schema>, + writer: W, + codec: Codec, + marker: [u8; 16], + ) -> Self { + let mut w = Self::builder() + .schema(schema) + .writer(writer) + .codec(codec) + .marker(marker) + .has_header(true) + .build(); + w.resolved_schema = ResolvedSchema::try_from(schemata).ok(); + w + } - Ok(n) - } - None => { - let rs = ResolvedSchema::try_from(self.schema)?; - self.resolved_schema = Some(rs); - self.append_value_ref(value) - } + /// Get a reference to the `Schema` associated to a `Writer`. + pub fn schema(&self) -> &'a Schema { + self.schema } - } - /// Append anything implementing the `Serialize` trait to a `Writer` for - /// [`serde`](https://docs.serde.rs/serde/index.html) compatibility, also performing schema - /// validation. - /// - /// Returns the number of bytes written. - /// - /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on - /// internal buffering for performance reasons. If you want to be sure the value has been - /// written, then call [`flush`](Writer::flush). - pub fn append_ser(&mut self, value: S) -> AvroResult { - let n = self.maybe_write_header()?; - - match self.resolved_schema { - Some(ref rs) => { - let mut serializer = SchemaAwareWriteSerializer::new( - &mut self.buffer, - self.schema, - rs.get_names(), - None, - ); - value.serialize(&mut serializer)?; - self.num_values += 1; - - if self.buffer.len() >= self.block_size { - return self.flush().map(|b| b + n); - } + /// Append a compatible value (implementing the `ToAvro` trait) to a `Writer`, also performing + /// schema validation. + /// + /// Returns the number of bytes written (it might be 0, see below). + /// + /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on + /// internal buffering for performance reasons. If you want to be sure the value has been + /// written, then call [`flush`](Writer::flush). + pub fn append>(&mut self, value: T) -> AvroResult { + let n = self.maybe_write_header()?; + + let avro = value.into(); + self.append_value_ref(&avro).map(|m| m + n) + } - Ok(n) - } - None => { - let rs = ResolvedSchema::try_from(self.schema)?; - self.resolved_schema = Some(rs); - self.append_ser(value) + /// Append a compatible value to a `Writer`, also performing schema validation. + /// + /// Returns the number of bytes written (it might be 0, see below). + /// + /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on + /// internal buffering for performance reasons. If you want to be sure the value has been + /// written, then call [`flush`](Writer::flush). + pub fn append_value_ref(&mut self, value: &Value) -> AvroResult { + let n = self.maybe_write_header()?; + + // Lazy init for users using the builder pattern with error throwing + match self.resolved_schema { + Some(ref rs) => { + write_value_ref_resolved(self.schema, rs, value, &mut self.buffer)?; + self.num_values += 1; + + if self.buffer.len() >= self.block_size { + return self.flush().map(|b| b + n); + } + + Ok(n) + } + None => { + let rs = ResolvedSchema::try_from(self.schema)?; + self.resolved_schema = Some(rs); + self.append_value_ref(value) + } } } - } - /// Extend a `Writer` with an `Iterator` of compatible values (implementing the `ToAvro` - /// trait), also performing schema validation. - /// - /// Returns the number of bytes written. - /// - /// **NOTE**: This function forces the written data to be flushed (an implicit - /// call to [`flush`](Writer::flush) is performed). - pub fn extend>(&mut self, values: I) -> AvroResult - where - I: IntoIterator, - { - /* - https://github.com/rust-lang/rfcs/issues/811 :( - let mut stream = values - .filter_map(|value| value.serialize(&mut self.serializer).ok()) - .map(|value| value.encode(self.schema)) - .collect::>>() - .ok_or_else(|| err_msg("value does not match given schema"))? - .into_iter() - .fold(Vec::new(), |mut acc, stream| { - num_values += 1; - acc.extend(stream); acc - }); - */ + /// Append anything implementing the `Serialize` trait to a `Writer` for + /// [`serde`](https://docs.serde.rs/serde/index.html) compatibility, also performing schema + /// validation. + /// + /// Returns the number of bytes written. + /// + /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on + /// internal buffering for performance reasons. If you want to be sure the value has been + /// written, then call [`flush`](Writer::flush). + pub fn append_ser(&mut self, value: S) -> AvroResult { + let n = self.maybe_write_header()?; + + match self.resolved_schema { + Some(ref rs) => { + let mut serializer = SchemaAwareWriteSerializer::new( + &mut self.buffer, + self.schema, + rs.get_names(), + None, + ); + value.serialize(&mut serializer)?; + self.num_values += 1; + + if self.buffer.len() >= self.block_size { + return self.flush().map(|b| b + n); + } - let mut num_bytes = 0; - for value in values { - num_bytes += self.append(value)?; + Ok(n) + } + None => { + let rs = ResolvedSchema::try_from(self.schema)?; + self.resolved_schema = Some(rs); + self.append_ser(value) + } + } } - num_bytes += self.flush()?; - - Ok(num_bytes) - } - /// Extend a `Writer` with an `Iterator` of anything implementing the `Serialize` trait for - /// [`serde`](https://docs.serde.rs/serde/index.html) compatibility, also performing schema - /// validation. - /// - /// Returns the number of bytes written. - /// - /// **NOTE**: This function forces the written data to be flushed (an implicit - /// call to [`flush`](Writer::flush) is performed). - pub fn extend_ser(&mut self, values: I) -> AvroResult - where - I: IntoIterator, - { - /* - https://github.com/rust-lang/rfcs/issues/811 :( - let mut stream = values - .filter_map(|value| value.serialize(&mut self.serializer).ok()) - .map(|value| value.encode(self.schema)) - .collect::>>() - .ok_or_else(|| err_msg("value does not match given schema"))? - .into_iter() - .fold(Vec::new(), |mut acc, stream| { - num_values += 1; - acc.extend(stream); acc - }); - */ + /// Extend a `Writer` with an `Iterator` of compatible values (implementing the `ToAvro` + /// trait), also performing schema validation. + /// + /// Returns the number of bytes written. + /// + /// **NOTE**: This function forces the written data to be flushed (an implicit + /// call to [`flush`](Writer::flush) is performed). + pub fn extend>(&mut self, values: I) -> AvroResult + where + I: IntoIterator, + { + /* + https://github.com/rust-lang/rfcs/issues/811 :( + let mut stream = values + .filter_map(|value| value.serialize(&mut self.serializer).ok()) + .map(|value| value.encode(self.schema)) + .collect::>>() + .ok_or_else(|| err_msg("value does not match given schema"))? + .into_iter() + .fold(Vec::new(), |mut acc, stream| { + num_values += 1; + acc.extend(stream); acc + }); + */ + + let mut num_bytes = 0; + for value in values { + num_bytes += self.append(value)?; + } + num_bytes += self.flush()?; - let mut num_bytes = 0; - for value in values { - num_bytes += self.append_ser(value)?; + Ok(num_bytes) } - num_bytes += self.flush()?; - Ok(num_bytes) - } + /// Extend a `Writer` with an `Iterator` of anything implementing the `Serialize` trait for + /// [`serde`](https://docs.serde.rs/serde/index.html) compatibility, also performing schema + /// validation. + /// + /// Returns the number of bytes written. + /// + /// **NOTE**: This function forces the written data to be flushed (an implicit + /// call to [`flush`](Writer::flush) is performed). + pub fn extend_ser(&mut self, values: I) -> AvroResult + where + I: IntoIterator, + { + /* + https://github.com/rust-lang/rfcs/issues/811 :( + let mut stream = values + .filter_map(|value| value.serialize(&mut self.serializer).ok()) + .map(|value| value.encode(self.schema)) + .collect::>>() + .ok_or_else(|| err_msg("value does not match given schema"))? + .into_iter() + .fold(Vec::new(), |mut acc, stream| { + num_values += 1; + acc.extend(stream); acc + }); + */ + + let mut num_bytes = 0; + for value in values { + num_bytes += self.append_ser(value)?; + } + num_bytes += self.flush()?; - /// Extend a `Writer` by appending each `Value` from a slice, while also performing schema - /// validation on each value appended. - /// - /// Returns the number of bytes written. - /// - /// **NOTE**: This function forces the written data to be flushed (an implicit - /// call to [`flush`](Writer::flush) is performed). - pub fn extend_from_slice(&mut self, values: &[Value]) -> AvroResult { - let mut num_bytes = 0; - for value in values { - num_bytes += self.append_value_ref(value)?; + Ok(num_bytes) } - num_bytes += self.flush()?; - Ok(num_bytes) - } + /// Extend a `Writer` by appending each `Value` from a slice, while also performing schema + /// validation on each value appended. + /// + /// Returns the number of bytes written. + /// + /// **NOTE**: This function forces the written data to be flushed (an implicit + /// call to [`flush`](Writer::flush) is performed). + pub fn extend_from_slice(&mut self, values: &[Value]) -> AvroResult { + let mut num_bytes = 0; + for value in values { + num_bytes += self.append_value_ref(value)?; + } + num_bytes += self.flush()?; - /// Flush the content to the inner `Writer`. - /// - /// Call this function to make sure all the content has been written before releasing the `Writer`. - /// This will also write the header if it wasn't written yet and hasn't been disabled using - /// [`WriterBuilder::has_header`]. - /// - /// Returns the number of bytes written. - pub fn flush(&mut self) -> AvroResult { - let mut num_bytes = self.maybe_write_header()?; - if self.num_values == 0 { - return Ok(num_bytes); + Ok(num_bytes) } - self.codec.compress(&mut self.buffer)?; + /// Flush the content to the inner `Writer`. + /// + /// Call this function to make sure all the content has been written before releasing the `Writer`. + /// This will also write the header if it wasn't written yet and hasn't been disabled using + /// [`WriterBuilder::has_header`]. + /// + /// Returns the number of bytes written. + pub fn flush(&mut self) -> AvroResult { + let mut num_bytes = self.maybe_write_header()?; + if self.num_values == 0 { + return Ok(num_bytes); + } - let num_values = self.num_values; - let stream_len = self.buffer.len(); + self.codec.compress(&mut self.buffer)?; - num_bytes += self.append_raw(&num_values.into(), &Schema::Long)? - + self.append_raw(&stream_len.into(), &Schema::Long)? - + self - .writer - .write(self.buffer.as_ref()) - .map_err(Details::WriteBytes)? - + self.append_marker()?; + let num_values = self.num_values; + let stream_len = self.buffer.len(); - self.buffer.clear(); - self.num_values = 0; + num_bytes += self.append_raw(&num_values.into(), &Schema::Long)? + + self.append_raw(&stream_len.into(), &Schema::Long)? + + self + .writer + .write(self.buffer.as_ref()) + .map_err(Details::WriteBytes)? + + self.append_marker()?; - self.writer.flush().map_err(Details::FlushWriter)?; + self.buffer.clear(); + self.num_values = 0; - Ok(num_bytes) - } + self.writer.flush().map_err(Details::FlushWriter)?; - /// Return what the `Writer` is writing to, consuming the `Writer` itself. - /// - /// **NOTE**: This function forces the written data to be flushed (an implicit - /// call to [`flush`](Writer::flush) is performed). - pub fn into_inner(mut self) -> AvroResult { - self.maybe_write_header()?; - self.flush()?; + Ok(num_bytes) + } - let mut this = ManuallyDrop::new(self); + /// Return what the `Writer` is writing to, consuming the `Writer` itself. + /// + /// **NOTE**: This function forces the written data to be flushed (an implicit + /// call to [`flush`](Writer::flush) is performed). + pub fn into_inner(mut self) -> AvroResult { + self.maybe_write_header()?; + self.flush()?; - // Extract every member that is not Copy and therefore should be dropped - let _resolved_schema = std::mem::take(&mut this.resolved_schema); - let _buffer = std::mem::take(&mut this.buffer); - let _user_metadata = std::mem::take(&mut this.user_metadata); + let mut this = ManuallyDrop::new(self); - // SAFETY: double-drops are prevented by putting `this` in a ManuallyDrop that is never dropped - let writer = unsafe { std::ptr::read(&this.writer) }; + // Extract every member that is not Copy and therefore should be dropped + let _resolved_schema = std::mem::take(&mut this.resolved_schema); + let _buffer = std::mem::take(&mut this.buffer); + let _user_metadata = std::mem::take(&mut this.user_metadata); - Ok(writer) - } + // SAFETY: double-drops are prevented by putting `this` in a ManuallyDrop that is never dropped + let writer = unsafe { std::ptr::read(&this.writer) }; - /// Gets a reference to the underlying writer. - /// - /// **NOTE**: There is likely data still in the buffer. To have all the data - /// in the writer call [`flush`](Writer::flush) first. - pub fn get_ref(&self) -> &W { - &self.writer - } + Ok(writer) + } - /// Gets a mutable reference to the underlying writer. - /// - /// It is inadvisable to directly write to the underlying writer. - /// - /// **NOTE**: There is likely data still in the buffer. To have all the data - /// in the writer call [`flush`](Writer::flush) first. - pub fn get_mut(&mut self) -> &mut W { - &mut self.writer - } + /// Gets a reference to the underlying writer. + /// + /// **NOTE**: There is likely data still in the buffer. To have all the data + /// in the writer call [`flush`](Writer::flush) first. + pub fn get_ref(&self) -> &W { + &self.writer + } - /// Generate and append synchronization marker to the payload. - fn append_marker(&mut self) -> AvroResult { - // using .writer.write directly to avoid mutable borrow of self - // with ref borrowing of self.marker - self.writer - .write(&self.marker) - .map_err(|e| Details::WriteMarker(e).into()) - } + /// Gets a mutable reference to the underlying writer. + /// + /// It is inadvisable to directly write to the underlying writer. + /// + /// **NOTE**: There is likely data still in the buffer. To have all the data + /// in the writer call [`flush`](Writer::flush) first. + pub fn get_mut(&mut self) -> &mut W { + &mut self.writer + } - /// Append a raw Avro Value to the payload avoiding to encode it again. - fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult { - self.append_bytes(encode_to_vec(value, schema)?.as_ref()) - } + /// Generate and append synchronization marker to the payload. + fn append_marker(&mut self) -> AvroResult { + // using .writer.write directly to avoid mutable borrow of self + // with ref borrowing of self.marker + self.writer + .write(&self.marker) + .map_err(|e| Details::WriteMarker(e).into()) + } - /// Append pure bytes to the payload. - fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult { - self.writer - .write(bytes) - .map_err(|e| Details::WriteBytes(e).into()) - } + /// Append a raw Avro Value to the payload avoiding to encode it again. + fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult { + self.append_bytes(encode_to_vec(value, schema)?.as_ref()) + } - /// Adds custom metadata to the file. - /// This method could be used only before adding the first record to the writer. - pub fn add_user_metadata>(&mut self, key: String, value: T) -> AvroResult<()> { - if !self.has_header { - if key.starts_with("avro.") { - return Err(Details::InvalidMetadataKey(key).into()); - } - self.user_metadata - .insert(key, Value::Bytes(value.as_ref().to_vec())); - Ok(()) - } else { - Err(Details::FileHeaderAlreadyWritten.into()) + /// Append pure bytes to the payload. + fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult { + self.writer + .write(bytes) + .map_err(|e| Details::WriteBytes(e).into()) } - } - /// Create an Avro header based on schema, codec and sync marker. - fn header(&self) -> Result, Error> { - let schema_bytes = serde_json::to_string(self.schema) - .map_err(Details::ConvertJsonToString)? - .into_bytes(); - - let mut metadata = HashMap::with_capacity(2); - metadata.insert("avro.schema", Value::Bytes(schema_bytes)); - metadata.insert("avro.codec", self.codec.into()); - match self.codec { - #[cfg(feature = "bzip")] - Codec::Bzip2(settings) => { - metadata.insert( - "avro.codec.compression_level", - Value::Bytes(vec![settings.compression_level]), - ); - } - #[cfg(feature = "xz")] - Codec::Xz(settings) => { - metadata.insert( - "avro.codec.compression_level", - Value::Bytes(vec![settings.compression_level]), - ); - } - #[cfg(feature = "zstandard")] - Codec::Zstandard(settings) => { - metadata.insert( - "avro.codec.compression_level", - Value::Bytes(vec![settings.compression_level]), - ); + /// Adds custom metadata to the file. + /// This method could be used only before adding the first record to the writer. + pub fn add_user_metadata>( + &mut self, + key: String, + value: T, + ) -> AvroResult<()> { + if !self.has_header { + if key.starts_with("avro.") { + return Err(Details::InvalidMetadataKey(key).into()); + } + self.user_metadata + .insert(key, Value::Bytes(value.as_ref().to_vec())); + Ok(()) + } else { + Err(Details::FileHeaderAlreadyWritten.into()) } - _ => {} } - for (k, v) in &self.user_metadata { - metadata.insert(k.as_str(), v.clone()); - } + /// Create an Avro header based on schema, codec and sync marker. + fn header(&self) -> Result, Error> { + let schema_bytes = serde_json::to_string(self.schema) + .map_err(Details::ConvertJsonToString)? + .into_bytes(); + + let mut metadata = HashMap::with_capacity(2); + metadata.insert("avro.schema", Value::Bytes(schema_bytes)); + metadata.insert("avro.codec", self.codec.into()); + match self.codec { + #[cfg(feature = "bzip")] + Codec::Bzip2(settings) => { + metadata.insert( + "avro.codec.compression_level", + Value::Bytes(vec![settings.compression_level]), + ); + } + #[cfg(feature = "xz")] + Codec::Xz(settings) => { + metadata.insert( + "avro.codec.compression_level", + Value::Bytes(vec![settings.compression_level]), + ); + } + #[cfg(feature = "zstandard")] + Codec::Zstandard(settings) => { + metadata.insert( + "avro.codec.compression_level", + Value::Bytes(vec![settings.compression_level]), + ); + } + _ => {} + } - let mut header = Vec::new(); - header.extend_from_slice(AVRO_OBJECT_HEADER); - encode(&metadata.into(), &Schema::map(Schema::Bytes), &mut header)?; - header.extend_from_slice(&self.marker); + for (k, v) in &self.user_metadata { + metadata.insert(k.as_str(), v.clone()); + } - Ok(header) - } + let mut header = Vec::new(); + header.extend_from_slice(AVRO_OBJECT_HEADER); + encode(&metadata.into(), &Schema::map(Schema::Bytes), &mut header)?; + header.extend_from_slice(&self.marker); - fn maybe_write_header(&mut self) -> AvroResult { - if !self.has_header { - let header = self.header()?; - let n = self.append_bytes(header.as_ref())?; - self.has_header = true; - Ok(n) - } else { - Ok(0) + Ok(header) } - } -} -impl Drop for Writer<'_, W> { - /// Drop the writer, will try to flush ignoring any errors. - fn drop(&mut self) { - let _ = self.maybe_write_header(); - let _ = self.flush(); + fn maybe_write_header(&mut self) -> AvroResult { + if !self.has_header { + let header = self.header()?; + let n = self.append_bytes(header.as_ref())?; + self.has_header = true; + Ok(n) + } else { + Ok(0) + } + } } -} -/// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also performing -/// schema validation. -/// -/// This is an internal function which gets the bytes buffer where to write as parameter instead of -/// creating a new one like `to_avro_datum`. -fn write_avro_datum, W: Write>( - schema: &Schema, - value: T, - writer: &mut W, -) -> Result<(), Error> { - let avro = value.into(); - if !avro.validate(schema) { - return Err(Details::Validation.into()); + impl Drop for Writer<'_, W> { + /// Drop the writer, will try to flush ignoring any errors. + fn drop(&mut self) { + let _ = self.maybe_write_header(); + let _ = self.flush(); + } } - encode(&avro, schema, writer)?; - Ok(()) -} -fn write_avro_datum_schemata>( - schema: &Schema, - schemata: Vec<&Schema>, - value: T, - buffer: &mut Vec, -) -> AvroResult { - let avro = value.into(); - let rs = ResolvedSchema::try_from(schemata)?; - let names = rs.get_names(); - let enclosing_namespace = schema.namespace(); - if let Some(_err) = avro.validate_internal(schema, names, &enclosing_namespace) { - return Err(Details::Validation.into()); + /// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also performing + /// schema validation. + /// + /// This is an internal function which gets the bytes buffer where to write as parameter instead of + /// creating a new one like `to_avro_datum`. + fn write_avro_datum, W: Write>( + schema: &Schema, + value: T, + writer: &mut W, + ) -> Result<(), Error> { + let avro = value.into(); + if !avro.validate(schema) { + return Err(Details::Validation.into()); + } + encode(&avro, schema, writer)?; + Ok(()) } - encode_internal(&avro, schema, names, &enclosing_namespace, buffer) -} - -/// Writer that encodes messages according to the single object encoding v1 spec -/// Uses an API similar to the current File Writer -/// Writes all object bytes at once, and drains internal buffer -pub struct GenericSingleObjectWriter { - buffer: Vec, - resolved: ResolvedOwnedSchema, -} -impl GenericSingleObjectWriter { - pub fn new_with_capacity( + fn write_avro_datum_schemata>( schema: &Schema, - initial_buffer_cap: usize, - ) -> AvroResult { - let header_builder = RabinFingerprintHeader::from_schema(schema); - Self::new_with_capacity_and_header_builder(schema, initial_buffer_cap, header_builder) + schemata: Vec<&Schema>, + value: T, + buffer: &mut Vec, + ) -> AvroResult { + let avro = value.into(); + let rs = ResolvedSchema::try_from(schemata)?; + let names = rs.get_names(); + let enclosing_namespace = schema.namespace(); + if let Some(_err) = avro.validate_internal(schema, names, &enclosing_namespace) { + return Err(Details::Validation.into()); + } + encode_internal(&avro, schema, names, &enclosing_namespace, buffer) } - pub fn new_with_capacity_and_header_builder( - schema: &Schema, - initial_buffer_cap: usize, - header_builder: HB, - ) -> AvroResult { - let mut buffer = Vec::with_capacity(initial_buffer_cap); - let header = header_builder.build_header(); - buffer.extend_from_slice(&header); - - Ok(GenericSingleObjectWriter { - buffer, - resolved: ResolvedOwnedSchema::try_from(schema.clone())?, - }) + /// Writer that encodes messages according to the single object encoding v1 spec + /// Uses an API similar to the current File Writer + /// Writes all object bytes at once, and drains internal buffer + pub struct GenericSingleObjectWriter { + buffer: Vec, + resolved: ResolvedOwnedSchema, } - const HEADER_LENGTH_RANGE: RangeInclusive = 10_usize..=20_usize; + impl GenericSingleObjectWriter { + pub fn new_with_capacity( + schema: &Schema, + initial_buffer_cap: usize, + ) -> AvroResult { + let header_builder = RabinFingerprintHeader::from_schema(schema); + Self::new_with_capacity_and_header_builder(schema, initial_buffer_cap, header_builder) + } - /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header - pub fn write_value_ref(&mut self, v: &Value, writer: &mut W) -> AvroResult { - let original_length = self.buffer.len(); - if !Self::HEADER_LENGTH_RANGE.contains(&original_length) { - Err(Details::IllegalSingleObjectWriterState.into()) - } else { - write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?; - writer - .write_all(&self.buffer) - .map_err(Details::WriteBytes)?; - let len = self.buffer.len(); - self.buffer.truncate(original_length); - Ok(len) + pub fn new_with_capacity_and_header_builder( + schema: &Schema, + initial_buffer_cap: usize, + header_builder: HB, + ) -> AvroResult { + let mut buffer = Vec::with_capacity(initial_buffer_cap); + let header = header_builder.build_header(); + buffer.extend_from_slice(&header); + + Ok(GenericSingleObjectWriter { + buffer, + resolved: ResolvedOwnedSchema::try_from(schema.clone())?, + }) + } + + const HEADER_LENGTH_RANGE: RangeInclusive = 10_usize..=20_usize; + + /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header + pub fn write_value_ref( + &mut self, + v: &Value, + writer: &mut W, + ) -> AvroResult { + let original_length = self.buffer.len(); + if !Self::HEADER_LENGTH_RANGE.contains(&original_length) { + Err(Details::IllegalSingleObjectWriterState.into()) + } else { + write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?; + writer + .write_all(&self.buffer) + .map_err(Details::WriteBytes)?; + let len = self.buffer.len(); + self.buffer.truncate(original_length); + Ok(len) + } } - } - /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header - pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { - self.write_value_ref(&v, writer) + /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header + pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { + self.write_value_ref(&v, writer) + } } -} -/// Writer that encodes messages according to the single object encoding v1 spec -pub struct SpecificSingleObjectWriter -where - T: AvroSchema, -{ - inner: GenericSingleObjectWriter, - schema: Schema, - header_written: bool, - _model: PhantomData, -} + /// Writer that encodes messages according to the single object encoding v1 spec + pub struct SpecificSingleObjectWriter + where + T: AvroSchema, + { + inner: GenericSingleObjectWriter, + schema: Schema, + header_written: bool, + _model: PhantomData, + } -impl SpecificSingleObjectWriter -where - T: AvroSchema, -{ - pub fn with_capacity(buffer_cap: usize) -> AvroResult> { - let schema = T::get_schema(); - Ok(SpecificSingleObjectWriter { - inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, - schema, - header_written: false, - _model: PhantomData, - }) + impl SpecificSingleObjectWriter + where + T: AvroSchema, + { + pub fn with_capacity(buffer_cap: usize) -> AvroResult> { + let schema = T::get_schema(); + Ok(SpecificSingleObjectWriter { + inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, + schema, + header_written: false, + _model: PhantomData, + }) + } } -} -impl SpecificSingleObjectWriter -where - T: AvroSchema + Into, -{ - /// Write the `Into` to the provided Write object. Returns a result with the number - /// of bytes written including the header - pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { - let v: Value = data.into(); - self.inner.write_value_ref(&v, writer) + impl SpecificSingleObjectWriter + where + T: AvroSchema + Into, + { + /// Write the `Into` to the provided Write object. Returns a result with the number + /// of bytes written including the header + pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { + let v: Value = data.into(); + self.inner.write_value_ref(&v, writer) + } } -} -impl SpecificSingleObjectWriter -where - T: AvroSchema + Serialize, -{ - /// Write the referenced `Serialize` object to the provided `Write` object. Returns a result with - /// the number of bytes written including the header - pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { - let mut bytes_written: usize = 0; + impl SpecificSingleObjectWriter + where + T: AvroSchema + Serialize, + { + /// Write the referenced `Serialize` object to the provided `Write` object. Returns a result with + /// the number of bytes written including the header + pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { + let mut bytes_written: usize = 0; + + if !self.header_written { + bytes_written += writer + .write(self.inner.buffer.as_slice()) + .map_err(Details::WriteBytes)?; + self.header_written = true; + } - if !self.header_written { - bytes_written += writer - .write(self.inner.buffer.as_slice()) - .map_err(Details::WriteBytes)?; - self.header_written = true; - } + bytes_written += write_avro_datum_ref(&self.schema, data, writer)?; - bytes_written += write_avro_datum_ref(&self.schema, data, writer)?; + Ok(bytes_written) + } - Ok(bytes_written) + /// Write the Serialize object to the provided Write object. Returns a result with the number + /// of bytes written including the header + pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + self.write_ref(&data, writer) + } } - /// Write the Serialize object to the provided Write object. Returns a result with the number - /// of bytes written including the header - pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { - self.write_ref(&data, writer) + fn write_value_ref_resolved( + schema: &Schema, + resolved_schema: &ResolvedSchema, + value: &Value, + buffer: &mut Vec, + ) -> AvroResult { + match value.validate_internal(schema, resolved_schema.get_names(), &schema.namespace()) { + Some(reason) => Err(Details::ValidationWithReason { + value: value.clone(), + schema: schema.clone(), + reason, + } + .into()), + None => encode_internal( + value, + schema, + resolved_schema.get_names(), + &schema.namespace(), + buffer, + ), + } } -} -fn write_value_ref_resolved( - schema: &Schema, - resolved_schema: &ResolvedSchema, - value: &Value, - buffer: &mut Vec, -) -> AvroResult { - match value.validate_internal(schema, resolved_schema.get_names(), &schema.namespace()) { - Some(reason) => Err(Details::ValidationWithReason { - value: value.clone(), - schema: schema.clone(), - reason, - } - .into()), - None => encode_internal( + fn write_value_ref_owned_resolved( + resolved_schema: &ResolvedOwnedSchema, + value: &Value, + buffer: &mut Vec, + ) -> AvroResult<()> { + let root_schema = resolved_schema.get_root_schema(); + if let Some(reason) = value.validate_internal( + root_schema, + resolved_schema.get_names(), + &root_schema.namespace(), + ) { + return Err(Details::ValidationWithReason { + value: value.clone(), + schema: root_schema.clone(), + reason, + } + .into()); + } + encode_internal( value, - schema, + root_schema, resolved_schema.get_names(), - &schema.namespace(), + &root_schema.namespace(), buffer, - ), + )?; + Ok(()) } -} -fn write_value_ref_owned_resolved( - resolved_schema: &ResolvedOwnedSchema, - value: &Value, - buffer: &mut Vec, -) -> AvroResult<()> { - let root_schema = resolved_schema.get_root_schema(); - if let Some(reason) = value.validate_internal( - root_schema, - resolved_schema.get_names(), - &root_schema.namespace(), - ) { - return Err(Details::ValidationWithReason { - value: value.clone(), - schema: root_schema.clone(), - reason, - } - .into()); + /// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also + /// performing schema validation. + /// + /// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync + /// markers; use [`Writer`] to be fully Avro-compatible if you don't know what + /// you are doing, instead. + pub fn to_avro_datum>(schema: &Schema, value: T) -> AvroResult> { + let mut buffer = Vec::new(); + write_avro_datum(schema, value, &mut buffer)?; + Ok(buffer) } - encode_internal( - value, - root_schema, - resolved_schema.get_names(), - &root_schema.namespace(), - buffer, - )?; - Ok(()) -} - -/// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also -/// performing schema validation. -/// -/// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync -/// markers; use [`Writer`] to be fully Avro-compatible if you don't know what -/// you are doing, instead. -pub fn to_avro_datum>(schema: &Schema, value: T) -> AvroResult> { - let mut buffer = Vec::new(); - write_avro_datum(schema, value, &mut buffer)?; - Ok(buffer) -} - -/// Write the referenced [Serialize]able object to the provided [Write] object. -/// Returns a result with the number of bytes written. -/// -/// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync -/// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible -/// if you don't know what you are doing, instead. -pub fn write_avro_datum_ref( - schema: &Schema, - data: &T, - writer: &mut W, -) -> AvroResult { - let names: HashMap = HashMap::new(); - let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, &names, None); - let bytes_written = data.serialize(&mut serializer)?; - Ok(bytes_written) -} -/// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also -/// performing schema validation. -/// If the provided `schema` is incomplete then its dependencies must be -/// provided in `schemata` -pub fn to_avro_datum_schemata>( - schema: &Schema, - schemata: Vec<&Schema>, - value: T, -) -> AvroResult> { - let mut buffer = Vec::new(); - write_avro_datum_schemata(schema, schemata, value, &mut buffer)?; - Ok(buffer) -} - -#[cfg(not(target_arch = "wasm32"))] -fn generate_sync_marker() -> [u8; 16] { - let mut marker = [0_u8; 16]; - std::iter::repeat_with(rand::random) - .take(16) - .enumerate() - .for_each(|(i, n)| marker[i] = n); - marker -} - -#[cfg(target_arch = "wasm32")] -fn generate_sync_marker() -> [u8; 16] { - let mut marker = [0_u8; 16]; - std::iter::repeat_with(quad_rand::rand) - .take(4) - .flat_map(|i| i.to_be_bytes()) - .enumerate() - .for_each(|(i, n)| marker[i] = n); - marker -} - -#[cfg(test)] -mod tests { - use std::{cell::RefCell, rc::Rc}; + /// Write the referenced [Serialize]able object to the provided [Write] object. + /// Returns a result with the number of bytes written. + /// + /// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync + /// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible + /// if you don't know what you are doing, instead. + pub fn write_avro_datum_ref( + schema: &Schema, + data: &T, + writer: &mut W, + ) -> AvroResult { + let names: HashMap = HashMap::new(); + let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, &names, None); + let bytes_written = data.serialize(&mut serializer)?; + Ok(bytes_written) + } - use super::*; - use crate::{ - Reader, - decimal::Decimal, - duration::{Days, Duration, Millis, Months}, - headers::GlueSchemaUuidHeader, - rabin::Rabin, - schema::{DecimalSchema, FixedSchema, Name}, - types::Record, - util::zig_i64, - }; - use pretty_assertions::assert_eq; - use serde::{Deserialize, Serialize}; - use uuid::Uuid; + /// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also + /// performing schema validation. + /// If the provided `schema` is incomplete then its dependencies must be + /// provided in `schemata` + pub fn to_avro_datum_schemata>( + schema: &Schema, + schemata: Vec<&Schema>, + value: T, + ) -> AvroResult> { + let mut buffer = Vec::new(); + write_avro_datum_schemata(schema, schemata, value, &mut buffer)?; + Ok(buffer) + } + + #[cfg(not(target_arch = "wasm32"))] + fn generate_sync_marker() -> [u8; 16] { + let mut marker = [0_u8; 16]; + std::iter::repeat_with(rand::random) + .take(16) + .enumerate() + .for_each(|(i, n)| marker[i] = n); + marker + } + + #[cfg(target_arch = "wasm32")] + fn generate_sync_marker() -> [u8; 16] { + let mut marker = [0_u8; 16]; + std::iter::repeat_with(quad_rand::rand) + .take(4) + .flat_map(|i| i.to_be_bytes()) + .enumerate() + .for_each(|(i, n)| marker[i] = n); + marker + } + + #[cfg(test)] + mod tests { + use std::{cell::RefCell, rc::Rc}; + + use super::*; + use crate::{ + Reader, + decimal::Decimal, + duration::{Days, Duration, Millis, Months}, + headers::GlueSchemaUuidHeader, + rabin::Rabin, + schema::{DecimalSchema, FixedSchema, Name}, + types::Record, + util::zig_i64, + }; + use pretty_assertions::assert_eq; + use serde::{Deserialize, Serialize}; + use uuid::Uuid; - use crate::{codec::DeflateSettings, error::Details}; - use apache_avro_test_helper::TestResult; + use crate::{codec::DeflateSettings, error::Details}; + use apache_avro_test_helper::TestResult; - const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); + const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); - const SCHEMA: &str = r#" + const SCHEMA: &str = r#" { "type": "record", "name": "test", @@ -809,456 +840,456 @@ mod tests { } "#; - const UNION_SCHEMA: &str = r#"["null", "long"]"#; - - #[test] - fn test_to_avro_datum() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); + const UNION_SCHEMA: &str = r#"["null", "long"]"#; - let mut expected = Vec::new(); - zig_i64(27, &mut expected)?; - zig_i64(3, &mut expected)?; - expected.extend([b'f', b'o', b'o']); + #[test] + fn test_to_avro_datum() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); - assert_eq!(to_avro_datum(&schema, record)?, expected); + let mut expected = Vec::new(); + zig_i64(27, &mut expected)?; + zig_i64(3, &mut expected)?; + expected.extend([b'f', b'o', b'o']); - Ok(()) - } + assert_eq!(to_avro_datum(&schema, record)?, expected); - #[test] - fn avro_rs_193_write_avro_datum_ref() -> TestResult { - #[derive(Serialize)] - struct TestStruct { - a: i64, - b: String, + Ok(()) } - let schema = Schema::parse_str(SCHEMA)?; - let mut writer: Vec = Vec::new(); - let data = TestStruct { - a: 27, - b: "foo".to_string(), - }; + #[test] + fn avro_rs_193_write_avro_datum_ref() -> TestResult { + #[derive(Serialize)] + struct TestStruct { + a: i64, + b: String, + } - let mut expected = Vec::new(); - zig_i64(27, &mut expected)?; - zig_i64(3, &mut expected)?; - expected.extend([b'f', b'o', b'o']); + let schema = Schema::parse_str(SCHEMA)?; + let mut writer: Vec = Vec::new(); + let data = TestStruct { + a: 27, + b: "foo".to_string(), + }; - let bytes = write_avro_datum_ref(&schema, &data, &mut writer)?; + let mut expected = Vec::new(); + zig_i64(27, &mut expected)?; + zig_i64(3, &mut expected)?; + expected.extend([b'f', b'o', b'o']); - assert_eq!(bytes, expected.len()); - assert_eq!(writer, expected); + let bytes = write_avro_datum_ref(&schema, &data, &mut writer)?; - Ok(()) - } + assert_eq!(bytes, expected.len()); + assert_eq!(writer, expected); - #[test] - fn avro_rs_220_flush_write_header() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - - // By default flush should write the header even if nothing was added yet - let mut writer = Writer::new(&schema, Vec::new()); - writer.flush()?; - let result = writer.into_inner()?; - assert_eq!(result.len(), 163); - - // Unless the user indicates via the builder that the header has already been written - let mut writer = Writer::builder() - .has_header(true) - .schema(&schema) - .writer(Vec::new()) - .build(); - writer.flush()?; - let result = writer.into_inner()?; - assert_eq!(result.len(), 0); + Ok(()) + } - Ok(()) - } + #[test] + fn avro_rs_220_flush_write_header() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + + // By default flush should write the header even if nothing was added yet + let mut writer = Writer::new(&schema, Vec::new()); + writer.flush()?; + let result = writer.into_inner()?; + assert_eq!(result.len(), 163); + + // Unless the user indicates via the builder that the header has already been written + let mut writer = Writer::builder() + .has_header(true) + .schema(&schema) + .writer(Vec::new()) + .build(); + writer.flush()?; + let result = writer.into_inner()?; + assert_eq!(result.len(), 0); - #[test] - fn test_union_not_null() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA)?; - let union = Value::Union(1, Box::new(Value::Long(3))); + Ok(()) + } - let mut expected = Vec::new(); - zig_i64(1, &mut expected)?; - zig_i64(3, &mut expected)?; + #[test] + fn test_union_not_null() -> TestResult { + let schema = Schema::parse_str(UNION_SCHEMA)?; + let union = Value::Union(1, Box::new(Value::Long(3))); - assert_eq!(to_avro_datum(&schema, union)?, expected); + let mut expected = Vec::new(); + zig_i64(1, &mut expected)?; + zig_i64(3, &mut expected)?; - Ok(()) - } + assert_eq!(to_avro_datum(&schema, union)?, expected); - #[test] - fn test_union_null() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA)?; - let union = Value::Union(0, Box::new(Value::Null)); + Ok(()) + } - let mut expected = Vec::new(); - zig_i64(0, &mut expected)?; + #[test] + fn test_union_null() -> TestResult { + let schema = Schema::parse_str(UNION_SCHEMA)?; + let union = Value::Union(0, Box::new(Value::Null)); - assert_eq!(to_avro_datum(&schema, union)?, expected); + let mut expected = Vec::new(); + zig_i64(0, &mut expected)?; - Ok(()) - } + assert_eq!(to_avro_datum(&schema, union)?, expected); - fn logical_type_test + Clone>( - schema_str: &'static str, - - expected_schema: &Schema, - value: Value, - - raw_schema: &Schema, - raw_value: T, - ) -> TestResult { - let schema = Schema::parse_str(schema_str)?; - assert_eq!(&schema, expected_schema); - // The serialized format should be the same as the schema. - let ser = to_avro_datum(&schema, value.clone())?; - let raw_ser = to_avro_datum(raw_schema, raw_value)?; - assert_eq!(ser, raw_ser); - - // Should deserialize from the schema into the logical type. - let mut r = ser.as_slice(); - let de = crate::from_avro_datum(&schema, &mut r, None)?; - assert_eq!(de, value); - Ok(()) - } + Ok(()) + } - #[test] - fn date() -> TestResult { - logical_type_test( - r#"{"type": "int", "logicalType": "date"}"#, - &Schema::Date, - Value::Date(1_i32), - &Schema::Int, - 1_i32, - ) - } + fn logical_type_test + Clone>( + schema_str: &'static str, + + expected_schema: &Schema, + value: Value, + + raw_schema: &Schema, + raw_value: T, + ) -> TestResult { + let schema = Schema::parse_str(schema_str)?; + assert_eq!(&schema, expected_schema); + // The serialized format should be the same as the schema. + let ser = to_avro_datum(&schema, value.clone())?; + let raw_ser = to_avro_datum(raw_schema, raw_value)?; + assert_eq!(ser, raw_ser); + + // Should deserialize from the schema into the logical type. + let mut r = ser.as_slice(); + let de = crate::from_avro_datum(&schema, &mut r, None)?; + assert_eq!(de, value); + Ok(()) + } - #[test] - fn time_millis() -> TestResult { - logical_type_test( - r#"{"type": "int", "logicalType": "time-millis"}"#, - &Schema::TimeMillis, - Value::TimeMillis(1_i32), - &Schema::Int, - 1_i32, - ) - } + #[test] + fn date() -> TestResult { + logical_type_test( + r#"{"type": "int", "logicalType": "date"}"#, + &Schema::Date, + Value::Date(1_i32), + &Schema::Int, + 1_i32, + ) + } - #[test] - fn time_micros() -> TestResult { - logical_type_test( - r#"{"type": "long", "logicalType": "time-micros"}"#, - &Schema::TimeMicros, - Value::TimeMicros(1_i64), - &Schema::Long, - 1_i64, - ) - } + #[test] + fn time_millis() -> TestResult { + logical_type_test( + r#"{"type": "int", "logicalType": "time-millis"}"#, + &Schema::TimeMillis, + Value::TimeMillis(1_i32), + &Schema::Int, + 1_i32, + ) + } - #[test] - fn timestamp_millis() -> TestResult { - logical_type_test( - r#"{"type": "long", "logicalType": "timestamp-millis"}"#, - &Schema::TimestampMillis, - Value::TimestampMillis(1_i64), - &Schema::Long, - 1_i64, - ) - } + #[test] + fn time_micros() -> TestResult { + logical_type_test( + r#"{"type": "long", "logicalType": "time-micros"}"#, + &Schema::TimeMicros, + Value::TimeMicros(1_i64), + &Schema::Long, + 1_i64, + ) + } - #[test] - fn timestamp_micros() -> TestResult { - logical_type_test( - r#"{"type": "long", "logicalType": "timestamp-micros"}"#, - &Schema::TimestampMicros, - Value::TimestampMicros(1_i64), - &Schema::Long, - 1_i64, - ) - } + #[test] + fn timestamp_millis() -> TestResult { + logical_type_test( + r#"{"type": "long", "logicalType": "timestamp-millis"}"#, + &Schema::TimestampMillis, + Value::TimestampMillis(1_i64), + &Schema::Long, + 1_i64, + ) + } - #[test] - fn decimal_fixed() -> TestResult { - let size = 30; - let inner = Schema::Fixed(FixedSchema { - name: Name::new("decimal")?, - aliases: None, - doc: None, - size, - default: None, - attributes: Default::default(), - }); - let value = vec![0u8; size]; - logical_type_test( - r#"{"type": {"type": "fixed", "size": 30, "name": "decimal"}, "logicalType": "decimal", "precision": 20, "scale": 5}"#, - &Schema::Decimal(DecimalSchema { - precision: 20, - scale: 5, - inner: Box::new(inner.clone()), - }), - Value::Decimal(Decimal::from(value.clone())), - &inner, - Value::Fixed(size, value), - ) - } + #[test] + fn timestamp_micros() -> TestResult { + logical_type_test( + r#"{"type": "long", "logicalType": "timestamp-micros"}"#, + &Schema::TimestampMicros, + Value::TimestampMicros(1_i64), + &Schema::Long, + 1_i64, + ) + } - #[test] - fn decimal_bytes() -> TestResult { - let inner = Schema::Bytes; - let value = vec![0u8; 10]; - logical_type_test( - r#"{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3}"#, - &Schema::Decimal(DecimalSchema { - precision: 4, - scale: 3, - inner: Box::new(inner.clone()), - }), - Value::Decimal(Decimal::from(value.clone())), - &inner, - value, - ) - } + #[test] + fn decimal_fixed() -> TestResult { + let size = 30; + let inner = Schema::Fixed(FixedSchema { + name: Name::new("decimal")?, + aliases: None, + doc: None, + size, + default: None, + attributes: Default::default(), + }); + let value = vec![0u8; size]; + logical_type_test( + r#"{"type": {"type": "fixed", "size": 30, "name": "decimal"}, "logicalType": "decimal", "precision": 20, "scale": 5}"#, + &Schema::Decimal(DecimalSchema { + precision: 20, + scale: 5, + inner: Box::new(inner.clone()), + }), + Value::Decimal(Decimal::from(value.clone())), + &inner, + Value::Fixed(size, value), + ) + } - #[test] - fn duration() -> TestResult { - let inner = Schema::Fixed(FixedSchema { - name: Name::new("duration")?, - aliases: None, - doc: None, - size: 12, - default: None, - attributes: Default::default(), - }); - let value = Value::Duration(Duration::new( - Months::new(256), - Days::new(512), - Millis::new(1024), - )); - logical_type_test( - r#"{"type": {"type": "fixed", "name": "duration", "size": 12}, "logicalType": "duration"}"#, - &Schema::Duration, - value, - &inner, - Value::Fixed(12, vec![0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0]), - ) - } + #[test] + fn decimal_bytes() -> TestResult { + let inner = Schema::Bytes; + let value = vec![0u8; 10]; + logical_type_test( + r#"{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3}"#, + &Schema::Decimal(DecimalSchema { + precision: 4, + scale: 3, + inner: Box::new(inner.clone()), + }), + Value::Decimal(Decimal::from(value.clone())), + &inner, + value, + ) + } - #[test] - fn test_writer_append() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); - - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - - let n1 = writer.append(record.clone())?; - let n2 = writer.append(record.clone())?; - let n3 = writer.flush()?; - let result = writer.into_inner()?; - - assert_eq!(n1 + n2 + n3, result.len()); - - let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; - data.extend(b"foo"); - data.extend(data.clone()); - - // starts with magic - assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); - // ends with data and sync marker - let last_data_byte = result.len() - 16; - assert_eq!( - &result[last_data_byte - data.len()..last_data_byte], - data.as_slice() - ); + #[test] + fn duration() -> TestResult { + let inner = Schema::Fixed(FixedSchema { + name: Name::new("duration")?, + aliases: None, + doc: None, + size: 12, + default: None, + attributes: Default::default(), + }); + let value = Value::Duration(Duration::new( + Months::new(256), + Days::new(512), + Millis::new(1024), + )); + logical_type_test( + r#"{"type": {"type": "fixed", "name": "duration", "size": 12}, "logicalType": "duration"}"#, + &Schema::Duration, + value, + &inner, + Value::Fixed(12, vec![0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0]), + ) + } - Ok(()) - } + #[test] + fn test_writer_append() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); + + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + + let n1 = writer.append(record.clone())?; + let n2 = writer.append(record.clone())?; + let n3 = writer.flush()?; + let result = writer.into_inner()?; + + assert_eq!(n1 + n2 + n3, result.len()); + + let mut data = Vec::new(); + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; + data.extend(b"foo"); + data.extend(data.clone()); + + // starts with magic + assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); + // ends with data and sync marker + let last_data_byte = result.len() - 16; + assert_eq!( + &result[last_data_byte - data.len()..last_data_byte], + data.as_slice() + ); - #[test] - fn test_writer_extend() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); - - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - let record_copy = record.clone(); - let records = vec![record, record_copy]; - - let n1 = writer.extend(records)?; - let n2 = writer.flush()?; - let result = writer.into_inner()?; - - assert_eq!(n1 + n2, result.len()); - - let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; - data.extend(b"foo"); - data.extend(data.clone()); - - // starts with magic - assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); - // ends with data and sync marker - let last_data_byte = result.len() - 16; - assert_eq!( - &result[last_data_byte - data.len()..last_data_byte], - data.as_slice() - ); + Ok(()) + } - Ok(()) - } + #[test] + fn test_writer_extend() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); + + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + let record_copy = record.clone(); + let records = vec![record, record_copy]; + + let n1 = writer.extend(records)?; + let n2 = writer.flush()?; + let result = writer.into_inner()?; + + assert_eq!(n1 + n2, result.len()); + + let mut data = Vec::new(); + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; + data.extend(b"foo"); + data.extend(data.clone()); + + // starts with magic + assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); + // ends with data and sync marker + let last_data_byte = result.len() - 16; + assert_eq!( + &result[last_data_byte - data.len()..last_data_byte], + data.as_slice() + ); - #[derive(Debug, Clone, Deserialize, Serialize)] - struct TestSerdeSerialize { - a: i64, - b: String, - } + Ok(()) + } - #[test] - fn test_writer_append_ser() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + #[derive(Debug, Clone, Deserialize, Serialize)] + struct TestSerdeSerialize { + a: i64, + b: String, + } - let record = TestSerdeSerialize { - a: 27, - b: "foo".to_owned(), - }; + #[test] + fn test_writer_append_ser() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); - let n1 = writer.append_ser(record)?; - let n2 = writer.flush()?; - let result = writer.into_inner()?; + let record = TestSerdeSerialize { + a: 27, + b: "foo".to_owned(), + }; - assert_eq!(n1 + n2, result.len()); + let n1 = writer.append_ser(record)?; + let n2 = writer.flush()?; + let result = writer.into_inner()?; - let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; - data.extend(b"foo"); + assert_eq!(n1 + n2, result.len()); - // starts with magic - assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); - // ends with data and sync marker - let last_data_byte = result.len() - 16; - assert_eq!( - &result[last_data_byte - data.len()..last_data_byte], - data.as_slice() - ); + let mut data = Vec::new(); + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; + data.extend(b"foo"); - Ok(()) - } + // starts with magic + assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); + // ends with data and sync marker + let last_data_byte = result.len() - 16; + assert_eq!( + &result[last_data_byte - data.len()..last_data_byte], + data.as_slice() + ); - #[test] - fn test_writer_extend_ser() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + Ok(()) + } - let record = TestSerdeSerialize { - a: 27, - b: "foo".to_owned(), - }; - let record_copy = record.clone(); - let records = vec![record, record_copy]; - - let n1 = writer.extend_ser(records)?; - let n2 = writer.flush()?; - let result = writer.into_inner()?; - - assert_eq!(n1 + n2, result.len()); - - let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; - data.extend(b"foo"); - data.extend(data.clone()); - - // starts with magic - assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); - // ends with data and sync marker - let last_data_byte = result.len() - 16; - assert_eq!( - &result[last_data_byte - data.len()..last_data_byte], - data.as_slice() - ); + #[test] + fn test_writer_extend_ser() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); + + let record = TestSerdeSerialize { + a: 27, + b: "foo".to_owned(), + }; + let record_copy = record.clone(); + let records = vec![record, record_copy]; + + let n1 = writer.extend_ser(records)?; + let n2 = writer.flush()?; + let result = writer.into_inner()?; + + assert_eq!(n1 + n2, result.len()); + + let mut data = Vec::new(); + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; + data.extend(b"foo"); + data.extend(data.clone()); + + // starts with magic + assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); + // ends with data and sync marker + let last_data_byte = result.len() - 16; + assert_eq!( + &result[last_data_byte - data.len()..last_data_byte], + data.as_slice() + ); - Ok(()) - } + Ok(()) + } - fn make_writer_with_codec(schema: &Schema) -> Writer<'_, Vec> { - Writer::with_codec( - schema, - Vec::new(), - Codec::Deflate(DeflateSettings::default()), - ) - } + fn make_writer_with_codec(schema: &Schema) -> Writer<'_, Vec> { + Writer::with_codec( + schema, + Vec::new(), + Codec::Deflate(DeflateSettings::default()), + ) + } - fn make_writer_with_builder(schema: &Schema) -> Writer<'_, Vec> { - Writer::builder() - .writer(Vec::new()) - .schema(schema) - .codec(Codec::Deflate(DeflateSettings::default())) - .block_size(100) - .build() - } + fn make_writer_with_builder(schema: &Schema) -> Writer<'_, Vec> { + Writer::builder() + .writer(Vec::new()) + .schema(schema) + .codec(Codec::Deflate(DeflateSettings::default())) + .block_size(100) + .build() + } - fn check_writer(mut writer: Writer<'_, Vec>, schema: &Schema) -> TestResult { - let mut record = Record::new(schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - - let n1 = writer.append(record.clone())?; - let n2 = writer.append(record.clone())?; - let n3 = writer.flush()?; - let result = writer.into_inner()?; - - assert_eq!(n1 + n2 + n3, result.len()); - - let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; - data.extend(b"foo"); - data.extend(data.clone()); - Codec::Deflate(DeflateSettings::default()).compress(&mut data)?; - - // starts with magic - assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); - // ends with data and sync marker - let last_data_byte = result.len() - 16; - assert_eq!( - &result[last_data_byte - data.len()..last_data_byte], - data.as_slice() - ); + fn check_writer(mut writer: Writer<'_, Vec>, schema: &Schema) -> TestResult { + let mut record = Record::new(schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + + let n1 = writer.append(record.clone())?; + let n2 = writer.append(record.clone())?; + let n3 = writer.flush()?; + let result = writer.into_inner()?; + + assert_eq!(n1 + n2 + n3, result.len()); + + let mut data = Vec::new(); + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; + data.extend(b"foo"); + data.extend(data.clone()); + Codec::Deflate(DeflateSettings::default()).compress(&mut data)?; + + // starts with magic + assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); + // ends with data and sync marker + let last_data_byte = result.len() - 16; + assert_eq!( + &result[last_data_byte - data.len()..last_data_byte], + data.as_slice() + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_writer_with_codec() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let writer = make_writer_with_codec(&schema); - check_writer(writer, &schema) - } + #[test] + fn test_writer_with_codec() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let writer = make_writer_with_codec(&schema); + check_writer(writer, &schema) + } - #[test] - fn test_writer_with_builder() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let writer = make_writer_with_builder(&schema); - check_writer(writer, &schema) - } + #[test] + fn test_writer_with_builder() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let writer = make_writer_with_builder(&schema); + check_writer(writer, &schema) + } - #[test] - fn test_logical_writer() -> TestResult { - const LOGICAL_TYPE_SCHEMA: &str = r#" + #[test] + fn test_logical_writer() -> TestResult { + const LOGICAL_TYPE_SCHEMA: &str = r#" { "type": "record", "name": "logical_type_test", @@ -1276,177 +1307,181 @@ mod tests { ] } "#; - let codec = Codec::Deflate(DeflateSettings::default()); - let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA)?; - let mut writer = Writer::builder() - .schema(&schema) - .codec(codec) - .writer(Vec::new()) - .build(); - - let mut record1 = Record::new(&schema).unwrap(); - record1.put( - "a", - Value::Union(1, Box::new(Value::TimestampMicros(1234_i64))), - ); - - let mut record2 = Record::new(&schema).unwrap(); - record2.put("a", Value::Union(0, Box::new(Value::Null))); - - let n1 = writer.append(record1)?; - let n2 = writer.append(record2)?; - let n3 = writer.flush()?; - let result = writer.into_inner()?; - - assert_eq!(n1 + n2 + n3, result.len()); - - let mut data = Vec::new(); - // byte indicating not null - zig_i64(1, &mut data)?; - zig_i64(1234, &mut data)?; - - // byte indicating null - zig_i64(0, &mut data)?; - codec.compress(&mut data)?; - - // starts with magic - assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); - // ends with data and sync marker - let last_data_byte = result.len() - 16; - assert_eq!( - &result[last_data_byte - data.len()..last_data_byte], - data.as_slice() - ); + let codec = Codec::Deflate(DeflateSettings::default()); + let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA)?; + let mut writer = Writer::builder() + .schema(&schema) + .codec(codec) + .writer(Vec::new()) + .build(); + + let mut record1 = Record::new(&schema).unwrap(); + record1.put( + "a", + Value::Union(1, Box::new(Value::TimestampMicros(1234_i64))), + ); + + let mut record2 = Record::new(&schema).unwrap(); + record2.put("a", Value::Union(0, Box::new(Value::Null))); + + let n1 = writer.append(record1)?; + let n2 = writer.append(record2)?; + let n3 = writer.flush()?; + let result = writer.into_inner()?; + + assert_eq!(n1 + n2 + n3, result.len()); + + let mut data = Vec::new(); + // byte indicating not null + zig_i64(1, &mut data)?; + zig_i64(1234, &mut data)?; + + // byte indicating null + zig_i64(0, &mut data)?; + codec.compress(&mut data)?; + + // starts with magic + assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER); + // ends with data and sync marker + let last_data_byte = result.len() - 16; + assert_eq!( + &result[last_data_byte - data.len()..last_data_byte], + data.as_slice() + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3405_writer_add_metadata_success() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + #[test] + fn test_avro_3405_writer_add_metadata_success() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); - writer.add_user_metadata("stringKey".to_string(), String::from("stringValue"))?; - writer.add_user_metadata("strKey".to_string(), "strValue")?; - writer.add_user_metadata("bytesKey".to_string(), b"bytesValue")?; - writer.add_user_metadata("vecKey".to_string(), vec![1, 2, 3])?; + writer.add_user_metadata("stringKey".to_string(), String::from("stringValue"))?; + writer.add_user_metadata("strKey".to_string(), "strValue")?; + writer.add_user_metadata("bytesKey".to_string(), b"bytesValue")?; + writer.add_user_metadata("vecKey".to_string(), vec![1, 2, 3])?; - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); - writer.append(record.clone())?; - writer.append(record.clone())?; - writer.flush()?; - let result = writer.into_inner()?; + writer.append(record.clone())?; + writer.append(record.clone())?; + writer.flush()?; + let result = writer.into_inner()?; - assert_eq!(result.len(), 260); + assert_eq!(result.len(), 260); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3881_metadata_empty_body() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); - writer.add_user_metadata("a".to_string(), "b")?; - let result = writer.into_inner()?; + #[test] + fn test_avro_3881_metadata_empty_body() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); + writer.add_user_metadata("a".to_string(), "b")?; + let result = writer.into_inner()?; - let reader = Reader::with_schema(&schema, &result[..])?; - let mut expected = HashMap::new(); - expected.insert("a".to_string(), vec![b'b']); - assert_eq!(reader.user_metadata(), &expected); - assert_eq!(reader.into_iter().count(), 0); + let reader = Reader::with_schema(&schema, &result[..])?; + let mut expected = HashMap::new(); + expected.insert("a".to_string(), vec![b'b']); + assert_eq!(reader.user_metadata(), &expected); + assert_eq!(reader.into_iter().count(), 0); - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3405_writer_add_metadata_failure() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + #[test] + fn test_avro_3405_writer_add_metadata_failure() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); - let mut record = Record::new(&schema).unwrap(); - record.put("a", 27i64); - record.put("b", "foo"); - writer.append(record.clone())?; + let mut record = Record::new(&schema).unwrap(); + record.put("a", 27i64); + record.put("b", "foo"); + writer.append(record.clone())?; - match writer - .add_user_metadata("stringKey".to_string(), String::from("value2")) - .map_err(Error::into_details) - { - Err(e @ Details::FileHeaderAlreadyWritten) => { - assert_eq!(e.to_string(), "The file metadata is already flushed.") + match writer + .add_user_metadata("stringKey".to_string(), String::from("value2")) + .map_err(Error::into_details) + { + Err(e @ Details::FileHeaderAlreadyWritten) => { + assert_eq!(e.to_string(), "The file metadata is already flushed.") + } + Err(e) => panic!("Unexpected error occurred while writing user metadata: {e:?}"), + Ok(_) => { + panic!("Expected an error that metadata cannot be added after adding data") + } } - Err(e) => panic!("Unexpected error occurred while writing user metadata: {e:?}"), - Ok(_) => panic!("Expected an error that metadata cannot be added after adding data"), - } - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + #[test] + fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); - let key = "avro.stringKey".to_string(); - match writer - .add_user_metadata(key.clone(), "value") - .map_err(Error::into_details) - { - Err(ref e @ Details::InvalidMetadataKey(_)) => { - assert_eq!( - e.to_string(), - format!( - "Metadata keys starting with 'avro.' are reserved for internal usage: {key}." + let key = "avro.stringKey".to_string(); + match writer + .add_user_metadata(key.clone(), "value") + .map_err(Error::into_details) + { + Err(ref e @ Details::InvalidMetadataKey(_)) => { + assert_eq!( + e.to_string(), + format!( + "Metadata keys starting with 'avro.' are reserved for internal usage: {key}." + ) ) - ) - } - Err(e) => panic!( - "Unexpected error occurred while writing user metadata with reserved prefix ('avro.'): {e:?}" - ), - Ok(_) => { - panic!("Expected an error that the metadata key cannot be prefixed with 'avro.'") + } + Err(e) => panic!( + "Unexpected error occurred while writing user metadata with reserved prefix ('avro.'): {e:?}" + ), + Ok(_) => { + panic!( + "Expected an error that the metadata key cannot be prefixed with 'avro.'" + ) + } } - } - Ok(()) - } + Ok(()) + } - #[test] - fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[test] + fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult { + let schema = Schema::parse_str(SCHEMA)?; - let mut user_meta_data: HashMap = HashMap::new(); - user_meta_data.insert( - "stringKey".to_string(), - Value::String("stringValue".to_string()), - ); - user_meta_data.insert("bytesKey".to_string(), Value::Bytes(b"bytesValue".to_vec())); - user_meta_data.insert("vecKey".to_string(), Value::Bytes(vec![1, 2, 3])); + let mut user_meta_data: HashMap = HashMap::new(); + user_meta_data.insert( + "stringKey".to_string(), + Value::String("stringValue".to_string()), + ); + user_meta_data.insert("bytesKey".to_string(), Value::Bytes(b"bytesValue".to_vec())); + user_meta_data.insert("vecKey".to_string(), Value::Bytes(vec![1, 2, 3])); - let writer: Writer<'_, Vec> = Writer::builder() - .writer(Vec::new()) - .schema(&schema) - .user_metadata(user_meta_data.clone()) - .build(); + let writer: Writer<'_, Vec> = Writer::builder() + .writer(Vec::new()) + .schema(&schema) + .user_metadata(user_meta_data.clone()) + .build(); - assert_eq!(writer.user_metadata, user_meta_data); + assert_eq!(writer.user_metadata, user_meta_data); - Ok(()) - } + Ok(()) + } - #[derive(Serialize, Clone)] - struct TestSingleObjectWriter { - a: i64, - b: f64, - c: Vec, - } + #[derive(Serialize, Clone)] + struct TestSingleObjectWriter { + a: i64, + b: f64, + c: Vec, + } - impl AvroSchema for TestSingleObjectWriter { - fn get_schema() -> Schema { - let schema = r#" + impl AvroSchema for TestSingleObjectWriter { + fn get_schema() -> Schema { + let schema = r#" { "type":"record", "name":"TestSingleObjectWrtierSerialize", @@ -1469,128 +1504,128 @@ mod tests { ] } "#; - Schema::parse_str(schema).unwrap() + Schema::parse_str(schema).unwrap() + } } - } - impl From for Value { - fn from(obj: TestSingleObjectWriter) -> Value { - Value::Record(vec![ - ("a".into(), obj.a.into()), - ("b".into(), obj.b.into()), - ( - "c".into(), - Value::Array(obj.c.into_iter().map(|s| s.into()).collect()), - ), - ]) + impl From for Value { + fn from(obj: TestSingleObjectWriter) -> Value { + Value::Record(vec![ + ("a".into(), obj.a.into()), + ("b".into(), obj.b.into()), + ( + "c".into(), + Value::Array(obj.c.into_iter().map(|s| s.into()).collect()), + ), + ]) + } } - } - #[test] - fn test_single_object_writer() -> TestResult { - let mut buf: Vec = Vec::new(); - let obj = TestSingleObjectWriter { - a: 300, - b: 34.555, - c: vec!["cat".into(), "dog".into()], - }; - let mut writer = GenericSingleObjectWriter::new_with_capacity( - &TestSingleObjectWriter::get_schema(), - 1024, - ) - .expect("Should resolve schema"); - let value = obj.into(); - let written_bytes = writer - .write_value_ref(&value, &mut buf) - .expect("Error serializing properly"); - - assert!(buf.len() > 10, "no bytes written"); - assert_eq!(buf.len(), written_bytes); - assert_eq!(buf[0], 0xC3); - assert_eq!(buf[1], 0x01); - assert_eq!( - &buf[2..10], - &TestSingleObjectWriter::get_schema() - .fingerprint::() - .bytes[..] - ); - let mut msg_binary = Vec::new(); - encode( - &value, - &TestSingleObjectWriter::get_schema(), - &mut msg_binary, - ) - .expect("encode should have failed by here as a dependency of any writing"); - assert_eq!(&buf[10..], &msg_binary[..]); + #[test] + fn test_single_object_writer() -> TestResult { + let mut buf: Vec = Vec::new(); + let obj = TestSingleObjectWriter { + a: 300, + b: 34.555, + c: vec!["cat".into(), "dog".into()], + }; + let mut writer = GenericSingleObjectWriter::new_with_capacity( + &TestSingleObjectWriter::get_schema(), + 1024, + ) + .expect("Should resolve schema"); + let value = obj.into(); + let written_bytes = writer + .write_value_ref(&value, &mut buf) + .expect("Error serializing properly"); + + assert!(buf.len() > 10, "no bytes written"); + assert_eq!(buf.len(), written_bytes); + assert_eq!(buf[0], 0xC3); + assert_eq!(buf[1], 0x01); + assert_eq!( + &buf[2..10], + &TestSingleObjectWriter::get_schema() + .fingerprint::() + .bytes[..] + ); + let mut msg_binary = Vec::new(); + encode( + &value, + &TestSingleObjectWriter::get_schema(), + &mut msg_binary, + ) + .expect("encode should have failed by here as a dependency of any writing"); + assert_eq!(&buf[10..], &msg_binary[..]); - Ok(()) - } + Ok(()) + } - #[test] - fn test_single_object_writer_with_header_builder() -> TestResult { - let mut buf: Vec = Vec::new(); - let obj = TestSingleObjectWriter { - a: 300, - b: 34.555, - c: vec!["cat".into(), "dog".into()], - }; - let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; - let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); - let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder( - &TestSingleObjectWriter::get_schema(), - 1024, - header_builder, - ) - .expect("Should resolve schema"); - let value = obj.into(); - writer - .write_value_ref(&value, &mut buf) - .expect("Error serializing properly"); - - assert_eq!(buf[0], 0x03); - assert_eq!(buf[1], 0x00); - assert_eq!(buf[2..18], schema_uuid.into_bytes()[..]); - Ok(()) - } + #[test] + fn test_single_object_writer_with_header_builder() -> TestResult { + let mut buf: Vec = Vec::new(); + let obj = TestSingleObjectWriter { + a: 300, + b: 34.555, + c: vec!["cat".into(), "dog".into()], + }; + let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; + let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); + let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder( + &TestSingleObjectWriter::get_schema(), + 1024, + header_builder, + ) + .expect("Should resolve schema"); + let value = obj.into(); + writer + .write_value_ref(&value, &mut buf) + .expect("Error serializing properly"); - #[test] - fn test_writer_parity() -> TestResult { - let obj1 = TestSingleObjectWriter { - a: 300, - b: 34.555, - c: vec!["cat".into(), "dog".into()], - }; + assert_eq!(buf[0], 0x03); + assert_eq!(buf[1], 0x00); + assert_eq!(buf[2..18], schema_uuid.into_bytes()[..]); + Ok(()) + } - let mut buf1: Vec = Vec::new(); - let mut buf2: Vec = Vec::new(); - let mut buf3: Vec = Vec::new(); - - let mut generic_writer = GenericSingleObjectWriter::new_with_capacity( - &TestSingleObjectWriter::get_schema(), - 1024, - ) - .expect("Should resolve schema"); - let mut specific_writer = - SpecificSingleObjectWriter::::with_capacity(1024) - .expect("Resolved should pass"); - specific_writer - .write(obj1.clone(), &mut buf1) - .expect("Serialization expected"); - specific_writer - .write_value(obj1.clone(), &mut buf2) - .expect("Serialization expected"); - generic_writer - .write_value(obj1.into(), &mut buf3) - .expect("Serialization expected"); - assert_eq!(buf1, buf2); - assert_eq!(buf1, buf3); + #[test] + fn test_writer_parity() -> TestResult { + let obj1 = TestSingleObjectWriter { + a: 300, + b: 34.555, + c: vec!["cat".into(), "dog".into()], + }; + + let mut buf1: Vec = Vec::new(); + let mut buf2: Vec = Vec::new(); + let mut buf3: Vec = Vec::new(); + + let mut generic_writer = GenericSingleObjectWriter::new_with_capacity( + &TestSingleObjectWriter::get_schema(), + 1024, + ) + .expect("Should resolve schema"); + let mut specific_writer = + SpecificSingleObjectWriter::::with_capacity(1024) + .expect("Resolved should pass"); + specific_writer + .write(obj1.clone(), &mut buf1) + .expect("Serialization expected"); + specific_writer + .write_value(obj1.clone(), &mut buf2) + .expect("Serialization expected"); + generic_writer + .write_value(obj1.into(), &mut buf3) + .expect("Serialization expected"); + assert_eq!(buf1, buf2); + assert_eq!(buf1, buf3); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { - const SCHEMA: &str = r#" + #[test] + fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { + const SCHEMA: &str = r#" { "type": "record", "name": "Conference", @@ -1600,30 +1635,30 @@ mod tests { ] }"#; - #[derive(Debug, PartialEq, Eq, Clone, Serialize)] - pub struct Conference { - pub name: String, - pub time: Option, - } + #[derive(Debug, PartialEq, Eq, Clone, Serialize)] + pub struct Conference { + pub name: String, + pub time: Option, + } - let conf = Conference { - name: "RustConf".to_string(), - time: Some(1234567890), - }; + let conf = Conference { + name: "RustConf".to_string(), + time: Some(1234567890), + }; - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); - let bytes = writer.append_ser(conf)?; + let bytes = writer.append_ser(conf)?; - assert_eq!(198, bytes); + assert_eq!(198, bytes); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_4014_validation_returns_a_detailed_error() -> TestResult { - const SCHEMA: &str = r#" + #[test] + fn avro_4014_validation_returns_a_detailed_error() -> TestResult { + const SCHEMA: &str = r#" { "type": "record", "name": "Conference", @@ -1633,35 +1668,35 @@ mod tests { ] }"#; - #[derive(Debug, PartialEq, Clone, Serialize)] - pub struct Conference { - pub name: String, - pub time: Option, // wrong type: f64 instead of i64 - } - - let conf = Conference { - name: "RustConf".to_string(), - time: Some(12345678.90), - }; - - let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, Vec::new()); + #[derive(Debug, PartialEq, Clone, Serialize)] + pub struct Conference { + pub name: String, + pub time: Option, // wrong type: f64 instead of i64 + } - match writer.append_ser(conf) { - Ok(bytes) => panic!("Expected an error, but got {bytes} bytes written"), - Err(e) => { - assert_eq!( - e.to_string(), - r#"Failed to serialize field 'time' for record Record(RecordSchema { name: Name { name: "Conference", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "name", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "date", doc: None, aliases: Some(["time2", "time"]), default: None, schema: Union(UnionSchema { schemas: [Null, Long], variant_index: {Null: 0, Long: 1} }), order: Ascending, position: 1, custom_attributes: {} }], lookup: {"date": 1, "name": 0, "time": 1, "time2": 1}, attributes: {} }): Failed to serialize value of type f64 using schema Long: 12345678.9. Cause: Expected: Long. Got: Double"# - ); + let conf = Conference { + name: "RustConf".to_string(), + time: Some(12345678.90), + }; + + let schema = Schema::parse_str(SCHEMA)?; + let mut writer = Writer::new(&schema, Vec::new()); + + match writer.append_ser(conf) { + Ok(bytes) => panic!("Expected an error, but got {bytes} bytes written"), + Err(e) => { + assert_eq!( + e.to_string(), + r#"Failed to serialize field 'time' for record Record(RecordSchema { name: Name { name: "Conference", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "name", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "date", doc: None, aliases: Some(["time2", "time"]), default: None, schema: Union(UnionSchema { schemas: [Null, Long], variant_index: {Null: 0, Long: 1} }), order: Ascending, position: 1, custom_attributes: {} }], lookup: {"date": 1, "name": 0, "time": 1, "time2": 1}, attributes: {} }): Failed to serialize value of type f64 using schema Long: 12345678.9. Cause: Expected: Long. Got: Double"# + ); + } } + Ok(()) } - Ok(()) - } - #[test] - fn avro_4063_flush_applies_to_inner_writer() -> TestResult { - const SCHEMA: &str = r#" + #[test] + fn avro_4063_flush_applies_to_inner_writer() -> TestResult { + const SCHEMA: &str = r#" { "type": "record", "name": "ExampleSchema", @@ -1671,45 +1706,46 @@ mod tests { } "#; - #[derive(Clone, Default)] - struct TestBuffer(Rc>>); + #[derive(Clone, Default)] + struct TestBuffer(Rc>>); - impl TestBuffer { - fn len(&self) -> usize { - self.0.borrow().len() + impl TestBuffer { + fn len(&self) -> usize { + self.0.borrow().len() + } } - } - impl Write for TestBuffer { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.borrow_mut().write(buf) - } + impl Write for TestBuffer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.borrow_mut().write(buf) + } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } - } - let shared_buffer = TestBuffer::default(); + let shared_buffer = TestBuffer::default(); - let buffered_writer = std::io::BufWriter::new(shared_buffer.clone()); + let buffered_writer = std::io::BufWriter::new(shared_buffer.clone()); - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA)?; - let mut writer = Writer::new(&schema, buffered_writer); + let mut writer = Writer::new(&schema, buffered_writer); - let mut record = Record::new(writer.schema()).unwrap(); - record.put("exampleField", "value"); + let mut record = Record::new(writer.schema()).unwrap(); + record.put("exampleField", "value"); - writer.append(record)?; - writer.flush()?; + writer.append(record)?; + writer.flush()?; - assert_eq!( - shared_buffer.len(), - 167, - "the test buffer was not fully written to after Writer::flush was called" - ); + assert_eq!( + shared_buffer.len(), + 167, + "the test buffer was not fully written to after Writer::flush was called" + ); - Ok(()) + Ok(()) + } } } From 136c33b97106d0d7cbd6c6147294005af185ff0d Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 1 Aug 2025 16:47:40 +0300 Subject: [PATCH 03/47] More WIP Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 2 -- avro/Cargo.toml | 9 +++--- avro/src/bigdecimal.rs | 14 ++++---- avro/src/bytes.rs | 5 ++- avro/src/codec.rs | 14 ++++---- avro/src/de.rs | 26 +++++++-------- avro/src/decimal.rs | 15 +++++---- avro/src/decode.rs | 44 ++++++++++++------------- avro/src/encode.rs | 16 +++++++--- avro/src/headers.rs | 13 +++++--- avro/src/lib.rs | 28 ++++++++++------ avro/src/reader.rs | 22 +++++++------ avro/src/schema.rs | 37 ++++++++++----------- avro/src/schema_compatibility.rs | 19 ++++++----- avro/src/schema_equality.rs | 18 +++++++---- avro/src/ser.rs | 19 ++++++----- avro/src/ser_schema.rs | 16 +++++++--- avro/src/types.rs | 20 +++++++----- avro/src/util.rs | 55 +++++++++++++++++--------------- avro/src/validator.rs | 16 +++++++--- avro/src/writer.rs | 19 ++++++----- avro/tests/avro-3786.rs | 26 ++++++++------- avro/tests/avro-3787.rs | 14 ++++---- avro/tests/schema.rs | 10 +++--- 24 files changed, 271 insertions(+), 206 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8466e416..e9c58ca9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1312,8 +1312,6 @@ dependencies = [ [[package]] name = "synca" version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dfc1a8bd0488fee2f2c457925766a8471564a6fbf7cb4211bd8a0e5f68c754a" dependencies = [ "proc-macro2", "quote", diff --git a/avro/Cargo.toml b/avro/Cargo.toml index b16e04f3..c5539370 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -29,14 +29,14 @@ categories.workspace = true documentation.workspace = true [features] -default = [ "tokio"] +default = ["tokio"] bzip = ["dep:bzip2"] derive = ["dep:apache-avro-derive"] snappy = ["dep:crc32fast", "dep:snap"] xz = ["dep:xz2"] zstandard = ["dep:zstd"] -sync = [ ] -tokio = [ "dep:tokio", "dep:futures" ] +sync = [] +tokio = ["dep:tokio", "dep:futures"] [lib] # disable benchmarks to allow passing criterion arguments to `cargo bench` @@ -77,7 +77,8 @@ tokio = { version = "1.47.0", features = [ "full" ], optional = true } uuid = { default-features = false, version = "1.17.0", features = ["serde", "std"] } xz2 = { default-features = false, version = "0.1.7", optional = true } zstd = { default-features = false, version = "0.13.3", optional = true } -synca = "0.5.3" +#synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing" } +synca = { path = "/home/martin/git/rust/rs_synca/synca" } futures = { version = "0.3.31", optional = true } diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 08a96f6d..981d9dbc 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -22,12 +22,14 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 8ac98b5c..0c084de9 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -288,7 +288,10 @@ pub mod serde_avro_slice_opt { #[cfg(test)] mod tests { use super::*; - use crate::{Schema, from_value, to_value, types::Value}; + use crate::schema::tokio::Schema; + use crate::ser::tokio::to_value; + use crate::de::tokio::from_value; + use crate::types::tokio::Value; use serde::{Deserialize, Serialize}; #[test] diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 93d661d4..7bae9a44 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -24,12 +24,14 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/de.rs b/avro/src/de.rs index 55601515..b9c4a578 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -24,13 +24,14 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - headers::tokio => headers::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } @@ -41,8 +42,6 @@ mod de { Error, bytes::DE_BYTES_BORROWED, error::tokio::Details, schema::tokio::SchemaKind, types::tokio::Value, }; - #[cfg(feature = "tokio")] - use futures::FutureExt; use serde::{ Deserialize, de::{self, DeserializeSeed, Deserializer as _, Visitor}, @@ -803,10 +802,11 @@ mod de { use serial_test::serial; use std::sync::atomic::Ordering; use uuid::Uuid; + use crate::ser::tokio::to_value; use apache_avro_test_helper::TestResult; - use crate::Decimal; + use crate::{Decimal, Schema}; use super::*; @@ -836,18 +836,18 @@ mod de { } "#; - let schema = crate::Schema::parse_str(schema_content)?; + let schema = Schema::parse_str(schema_content)?; let data = StringEnum { source: "SOZU".to_string(), }; // encode into avro - let value = crate::to_value(&data)?; + let value = to_value(&data)?; let mut buf = std::io::Cursor::new(crate::to_avro_datum(&schema, value)?); // decode from avro - let value = crate::from_avro_datum(&schema, &mut buf, None)?; + let value = from_avro_datum(&schema, &mut buf, None)?; let decoded_data: StringEnum = crate::from_value(&value)?; diff --git a/avro/src/decimal.rs b/avro/src/decimal.rs index e215c8c5..7354a14a 100644 --- a/avro/src/decimal.rs +++ b/avro/src/decimal.rs @@ -22,13 +22,14 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead + Unpin => std::io::Read, - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 1d0b29f1..4b308b87 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -22,22 +22,19 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decimal::tokio => decimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } )] mod decode { - #[cfg(feature = "tokio")] - use futures::FutureExt; - #[cfg(feature = "tokio")] - use futures::TryFutureExt; #[cfg(feature = "sync")] use std::io::Read as AvroRead; #[cfg(feature = "tokio")] @@ -45,8 +42,7 @@ mod decode { #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; - use crate::util::safe_len; - use crate::util::tokio::{zag_i32, zag_i64}; + use crate::util::tokio::{safe_len, zag_i32, zag_i64}; use crate::{ AvroResult, Error, bigdecimal::tokio::deserialize_big_decimal, @@ -65,12 +61,12 @@ mod decode { #[inline] pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { - zag_i64(reader).await?.map(Value::Long) + zag_i64(reader).await.map(Value::Long) } #[inline] async fn decode_int(reader: &mut R) -> AvroResult { - zag_i32(reader).await?.map(Value::Int) + zag_i32(reader).await.map(Value::Int) } #[inline] @@ -227,16 +223,16 @@ mod decode { Ok(Value::Uuid(uuid)) } Schema::Int => decode_int(reader).await, - Schema::Date => zag_i32(reader).await?.map(Value::Date), - Schema::TimeMillis => zag_i32(reader).await?.map(Value::TimeMillis), + Schema::Date => zag_i32(reader).await.map(Value::Date), + Schema::TimeMillis => zag_i32(reader).await.map(Value::TimeMillis), Schema::Long => decode_long(reader).await, - Schema::TimeMicros => zag_i64(reader).await?.map(Value::TimeMicros), - Schema::TimestampMillis => zag_i64(reader).await?.map(Value::TimestampMillis), - Schema::TimestampMicros => zag_i64(reader).await?.map(Value::TimestampMicros), - Schema::TimestampNanos => zag_i64(reader).await?.map(Value::TimestampNanos), - Schema::LocalTimestampMillis => zag_i64(reader).await?.map(Value::LocalTimestampMillis), - Schema::LocalTimestampMicros => zag_i64(reader).await?.map(Value::LocalTimestampMicros), - Schema::LocalTimestampNanos => zag_i64(reader).await?.map(Value::LocalTimestampNanos), + Schema::TimeMicros => zag_i64(reader).await.map(Value::TimeMicros), + Schema::TimestampMillis => zag_i64(reader).await.map(Value::TimestampMillis), + Schema::TimestampMicros => zag_i64(reader).await.map(Value::TimestampMicros), + Schema::TimestampNanos => zag_i64(reader).await.map(Value::TimestampNanos), + Schema::LocalTimestampMillis => zag_i64(reader).await.map(Value::LocalTimestampMillis), + Schema::LocalTimestampMicros => zag_i64(reader).await.map(Value::LocalTimestampMicros), + Schema::LocalTimestampNanos => zag_i64(reader).await.map(Value::LocalTimestampNanos), Schema::Duration => { let mut buf = [0u8; 12]; reader diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 98c603e6..f4efbd4d 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -22,8 +22,14 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead + Unpin => std::io::Read, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } @@ -34,13 +40,13 @@ mod encode { use crate::{ AvroResult, - bigdecimal::serialize_big_decimal, - error::Details, - schema::{ + bigdecimal::tokio::serialize_big_decimal, + error::tokio::Details, + schema::tokio::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, SchemaKind, UnionSchema, }, - types::{Value, ValueKind}, + types::tokio::{Value, ValueKind}, }; use log::error; use std::{borrow::Borrow, collections::HashMap, io::Write}; diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 149fcf80..1c505c70 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -24,11 +24,14 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 7016db89..c2c8848c 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -886,28 +886,36 @@ pub use crate::bytes::{ serde_avro_bytes, serde_avro_bytes_opt, serde_avro_fixed, serde_avro_fixed_opt, serde_avro_slice, serde_avro_slice_opt, }; -pub use crate::tokio::bigdecimal::BigDecimal; +#[cfg(feature = "tokio")] +pub use crate::bigdecimal::tokio::BigDecimal; #[cfg(feature = "bzip")] pub use codec::bzip::Bzip2Settings; #[cfg(feature = "xz")] pub use codec::xz::XzSettings; #[cfg(feature = "zstandard")] pub use codec::zstandard::ZstandardSettings; -pub use codec::{Codec, DeflateSettings}; -pub use de::from_value; -pub use decimal::Decimal; +#[cfg(feature = "tokio")] +pub use codec::tokio::{Codec, DeflateSettings}; +#[cfg(feature = "tokio")] +pub use de::tokio::from_value; +#[cfg(feature = "tokio")] +pub use decimal::tokio::Decimal; pub use duration::{Days, Duration, Millis, Months}; -pub use error::Error; - +#[cfg(feature = "tokio")] +pub use error::tokio::Error; +#[cfg(feature = "tokio")] pub use reader::tokio::{ GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, }; -pub use schema::{AvroSchema, Schema}; -pub use ser::to_value; +#[cfg(feature = "tokio")] +pub use schema::tokio::{AvroSchema, Schema}; +#[cfg(feature = "tokio")] +pub use ser::tokio::to_value; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; -pub use writer::{ +#[cfg(feature = "tokio")] +pub use writer::tokio::{ GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, to_avro_datum_schemata, write_avro_datum_ref, }; @@ -922,7 +930,7 @@ pub type AvroResult = Result; mod tests { use crate::{ Codec, Reader, Schema, Writer, from_avro_datum, - types::{Record, Value}, + types::tokio::{Record, Value}, }; use pretty_assertions::assert_eq; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 1a274535..a99ebee2 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -22,12 +22,14 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, #[tokio::test] => #[test] ); } @@ -39,10 +41,10 @@ mod reader { use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; - + use std::str::FromStr; use crate::decode::tokio::{decode, decode_internal}; use crate::{ - AvroResult, Codec, Error, + AvroResult, codec::tokio::Codec, error::tokio::Error, error::tokio::Details, from_value, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, @@ -63,6 +65,8 @@ mod reader { #[cfg(feature = "tokio")] use std::task::Poll; use std::{collections::HashMap, io::ErrorKind, marker::PhantomData}; + use crate::util::tokio::safe_len; + /// Internal Block reader. #[derive(Debug, Clone)] @@ -156,7 +160,7 @@ mod reader { // We need to resize to ensure that the buffer len is safe to read `n` elements. // // TODO: Figure out a way to avoid having to truncate for the second case. - self.buf.resize(crate::util::safe_len(n)?, 0); + self.buf.resize(safe_len(n)?, 0); self.reader .read_exact(&mut self.buf) .await diff --git a/avro/src/schema.rs b/avro/src/schema.rs index cad51d63..76283acf 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -24,15 +24,16 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead + Unpin => std::io::Read, bigdecimal::tokio => bigdecimal::sync, + decimal::tokio => decimal::sync, decode::tokio => decode::sync, encode::tokio => encode::sync, error::tokio => error::sync, schema::tokio => schema::sync, - schema_equality::tokio => schema_equality::sync, util::tokio => util::sync, types::tokio => types::sync, + schema_equality::tokio => schema_equality::sync, + util::tokio => util::sync, validator::tokio => validator::sync, #[tokio::test] => #[test] ); @@ -40,7 +41,7 @@ )] mod schema { - use crate::{ + use { AvroResult, error::tokio::{Details, Error}, schema_equality::tokio as schema_equality, @@ -211,7 +212,7 @@ mod schema { impl From<&types::Value> for SchemaKind { fn from(value: &types::Value) -> Self { - use crate::types::tokio::Value; + use types::tokio::Value; match value { Value::Null => Self::Null, Value::Boolean(_) => Self::Boolean, @@ -2698,7 +2699,7 @@ mod schema { #[cfg(test)] mod tests { use super::*; - use crate::{SpecificSingleObjectWriter, error::Details, rabin::Rabin}; + use {SpecificSingleObjectWriter, error::Details, rabin::Rabin}; use apache_avro_test_helper::{ TestResult, logger::{assert_logged, assert_not_logged}, @@ -3784,7 +3785,7 @@ mod schema { #[test] fn test_schema_fingerprint() -> TestResult { - use crate::rabin::Rabin; + use rabin::Rabin; use md5::Md5; use sha2::Sha256; @@ -5328,10 +5329,10 @@ mod schema { bar_use: Bar::Bar1, }; - let avro_value = crate::to_value(foo)?; + let avro_value = to_value(foo)?; assert!(avro_value.validate(&schema)); - let mut writer = crate::Writer::new(&schema, Vec::new()); + let mut writer = Writer::new(&schema, Vec::new()); // schema validation happens here writer.append(avro_value)?; @@ -5426,15 +5427,15 @@ mod schema { bar_init: Bar::Bar0, bar_use: Bar::Bar1, }; - let avro_value = crate::to_value(foo)?; + let avro_value = to_value(foo)?; assert!( avro_value.validate(&writer_schema), "value is valid for schema", ); - let datum = crate::to_avro_datum(&writer_schema, avro_value)?; + let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; let reader_schema = Schema::parse_str(reader_schema)?; - let deser_value = crate::from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; + let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { types::Value::Record(fields) => { assert_eq!(fields.len(), 2); @@ -6007,23 +6008,23 @@ mod schema { // Serialize using the writer schema. let writer_schema = Schema::parse(&writer_schema)?; - let avro_value = crate::to_value(s)?; + let avro_value = to_value(s)?; assert!( avro_value.validate(&writer_schema), "value is valid for schema", ); - let datum = crate::to_avro_datum(&writer_schema, avro_value)?; + let datum = to_avro_datum(&writer_schema, avro_value)?; // Now, attempt to deserialize using the reader schema. let reader_schema = Schema::parse(&reader_schema)?; let mut x = &datum[..]; // Deserialization should succeed and we should be able to resolve the schema. - let deser_value = crate::from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; + let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; assert!(deser_value.validate(&reader_schema)); // Verify that we can read a field from the record. - let d: MyRecordReader = crate::from_value(&deser_value)?; + let d: MyRecordReader = from_value(&deser_value)?; assert_eq!(d.inner_record.unwrap().a, "foo".to_string()); Ok(()) } @@ -6800,7 +6801,7 @@ mod schema { fn avro_rs_53_uuid_with_fixed() -> TestResult { #[derive(Debug, Serialize, Deserialize)] struct Comment { - id: crate::Uuid, + id: Uuid, } impl AvroSchema for Comment { @@ -6830,13 +6831,13 @@ mod schema { let mut buffer = Vec::new(); // serialize the Uuid as String - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let bytes = SpecificSingleObjectWriter::::with_capacity(64)? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 47); // serialize the Uuid as Bytes - crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); + util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); let bytes = SpecificSingleObjectWriter::::with_capacity(64)? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 27); diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index abee6156..b9128936 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -24,14 +24,17 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - headers::tokio => headers::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, - types::tokio => types::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/schema_equality.rs b/avro/src/schema_equality.rs index 9609e956..02bd42b4 100644 --- a/avro/src/schema_equality.rs +++ b/avro/src/schema_equality.rs @@ -22,13 +22,17 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead + Unpin => std::io::Read, - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 6a3d1a48..499d5a9e 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -24,14 +24,17 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - headers::tokio => headers::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, - types::tokio => types::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index f26d8ed0..04a73573 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -25,9 +25,17 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, - decode::tokio => decode::sync, - encode::tokio => encode::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -37,7 +45,7 @@ mod bigdecimal { use crate::{ bigdecimal::tokio::big_decimal_as_bytes, encode::tokio::{encode_int, encode_long}, - error::{Details, Error}, + error::tokio::{Details, Error}, schema::tokio::{Name, NamesRef, Namespace, RecordField, RecordSchema, Schema}, }; use bigdecimal::BigDecimal; diff --git a/avro/src/types.rs b/avro/src/types.rs index 6e1c3c02..90adb7a5 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -24,13 +24,17 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decimal::tokio => decimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -42,7 +46,7 @@ mod types { bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, decimal::tokio::Decimal, duration::Duration, - error::tokio::{Details, Error}, + error::tokio::Details, schema::tokio::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, diff --git a/avro/src/util.rs b/avro/src/util.rs index 379b4645..dddab248 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use crate::AvroResult; -use crate::error::sync::Details; use std::sync::atomic::Ordering; use std::sync::{ Once, @@ -55,19 +53,6 @@ pub fn max_allocation_bytes(num_bytes: usize) -> usize { }); MAX_ALLOCATION_BYTES.load(Ordering::Acquire) } -pub fn safe_len(len: usize) -> AvroResult { - let max_bytes = max_allocation_bytes(DEFAULT_MAX_ALLOCATION_BYTES); - - if len <= max_bytes { - Ok(len) - } else { - Err(Details::MemoryAllocation { - desired: len, - maximum: max_bytes, - } - .into()) - } -} /// Set whether serializing/deserializing is marked as human readable in serde traits. /// This will adjust the return value of `is_human_readable()` for both. @@ -90,12 +75,17 @@ pub fn set_serde_human_readable(human_readable: bool) { pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -221,10 +211,25 @@ mod util { Ok(i) } + pub fn safe_len(len: usize) -> crate::AvroResult { + let max_bytes = crate::util::max_allocation_bytes(crate::util::DEFAULT_MAX_ALLOCATION_BYTES); + + if len <= max_bytes { + Ok(len) + } else { + Err(Details::MemoryAllocation { + desired: len, + maximum: max_bytes, + } + .into()) + } + } + + #[cfg(test)] mod tests { use super::*; - use crate::util::safe_len; + use crate::util::tokio::safe_len; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; @@ -312,10 +317,10 @@ mod util { assert_eq!(s, [255, 255, 255, 255, 15]); } - #[test] - fn test_overflow() { + #[tokio::test] + async fn test_overflow() { let causes_left_shift_overflow: &[u8] = &[0xe1, 0xe1, 0xe1, 0xe1, 0xe1]; - assert!(decode_variable(&mut &*causes_left_shift_overflow).is_err()); + assert!(decode_variable(&mut &*causes_left_shift_overflow).await.is_err()); } #[test] diff --git a/avro/src/validator.rs b/avro/src/validator.rs index 94c6d359..0cd37001 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -22,11 +22,17 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index e9977f59..f8d8838a 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -24,14 +24,17 @@ pub mod sync { sync!(); replace!( - tokio::io::AsyncRead => std::io::Read, - bigdecimal::tokio => bigdecimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - headers::tokio => headers::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index c509a3f8..bcb3a861 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Schema, from_avro_datum, to_avro_datum, to_value, types}; +use apache_avro::schema::tokio::Schema; +use apache_avro::{from_avro_datum, to_avro_datum, to_value}; +use apache_avro::types::tokio::Value; use apache_avro_test_helper::TestResult; #[test] @@ -137,18 +139,18 @@ fn avro_3786_deserialize_union_with_different_enum_order() -> TestResult { let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 2); assert_eq!(fields[0].0, "barInit"); - assert_eq!(fields[0].1, types::Value::Enum(0, "bar1".to_string())); + assert_eq!(fields[0].1, Value::Enum(0, "bar1".to_string())); assert_eq!(fields[1].0, "barUseParent"); assert_eq!( fields[1].1, - types::Value::Union( + Value::Union( 1, - Box::new(types::Value::Record(vec![( + Box::new(Value::Record(vec![( "barUse".to_string(), - types::Value::Enum(0, "bar1".to_string()) + Value::Enum(0, "bar1".to_string()) )])) ) ); @@ -261,7 +263,7 @@ fn avro_3786_deserialize_union_with_different_enum_order_defined_in_record() -> let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 1); assert_eq!(fields[0].0, "barParent"); // TODO: better validation @@ -374,7 +376,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 1); assert_eq!(fields[0].0, "barParent"); // TODO: better validation @@ -487,7 +489,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 1); assert_eq!(fields[0].0, "barParent"); // TODO: better validation @@ -600,7 +602,7 @@ fn deserialize_union_with_different_enum_order_defined_in_record() -> TestResult let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 1); assert_eq!(fields[0].0, "barParent"); // TODO: better validation @@ -874,10 +876,10 @@ fn deserialize_union_with_record_with_enum_defined_inline_reader_has_different_i let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 3); assert_eq!(fields[0].0, "barInit"); - assert_eq!(fields[0].1, types::Value::Enum(0, "bar0".to_string())); + assert_eq!(fields[0].1, Value::Enum(0, "bar0".to_string())); // TODO: better validation } _ => panic!("Expected Value::Record"), diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index 1795fca6..273ed696 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Schema, from_avro_datum, to_avro_datum, to_value, types}; +use apache_avro::{from_avro_datum, to_avro_datum, to_value}; +use apache_avro::schema::tokio::Schema; +use apache_avro::types::tokio::Value; use apache_avro_test_helper::TestResult; #[test] @@ -138,10 +140,10 @@ fn avro_3787_deserialize_union_with_unknown_symbol() -> TestResult { let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 2); assert_eq!(fields[0].0, "barInit"); - assert_eq!(fields[0].1, types::Value::Enum(1, "bar1".to_string())); + assert_eq!(fields[0].1, Value::Enum(1, "bar1".to_string())); assert_eq!(fields[1].0, "barUseParent"); // TODO: test value } @@ -265,12 +267,12 @@ fn avro_3787_deserialize_union_with_unknown_symbol_no_ref() -> TestResult { let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 1); // assert_eq!(fields[0].0, "barInit"); - // assert_eq!(fields[0].1, types::Value::Enum(0, "bar0".to_string())); + // assert_eq!(fields[0].1, Value::Enum(0, "bar0".to_string())); assert_eq!(fields[0].0, "barParent"); - // assert_eq!(fields[1].1, types::Value::Enum(1, "bar1".to_string())); + // assert_eq!(fields[1].1, Value::Enum(1, "bar1".to_string())); } _ => panic!("Expected Value::Record"), } diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index 92306c3a..9b4a6536 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -21,12 +21,12 @@ use std::{ }; use apache_avro::{ - Codec, Error, Reader, Schema, Writer, - error::Details, + Codec, Reader, Writer, + error::tokio::{Details, Error}, from_avro_datum, from_value, - schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema}, + schema::tokio::{Schema, EnumSchema, FixedSchema, Name, RecordField, RecordSchema}, to_avro_datum, to_value, - types::{Record, Value}, + types::tokio::{Record, Value}, }; use apache_avro_test_helper::{ TestResult, @@ -885,7 +885,7 @@ fn avro_old_issue_47() -> TestResult { #[test] fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_type() -> TestResult { - use apache_avro::{from_avro_datum, to_avro_datum, types::Value}; + use apache_avro::{from_avro_datum, to_avro_datum, types::tokio::Value}; use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] From b61ba80eaf1fe43cf61c0a30cf79662c43ea65ba Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 4 Aug 2025 09:36:48 +0300 Subject: [PATCH 04/47] More WIP Convert more methods to async Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/schema.rs | 4 +- avro/src/types.rs | 92 +++++++++++++++++++++++++++------------------- avro/src/writer.rs | 58 +++++++++++++++-------------- 3 files changed, 86 insertions(+), 68 deletions(-) diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 76283acf..1fd613a4 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -41,7 +41,7 @@ )] mod schema { - use { + use crate::{ AvroResult, error::tokio::{Details, Error}, schema_equality::tokio as schema_equality, @@ -212,7 +212,7 @@ mod schema { impl From<&types::Value> for SchemaKind { fn from(value: &types::Value) -> Self { - use types::tokio::Value; + use crate::types::tokio::Value; match value { Value::Null => Self::Null, Value::Boolean(_) => Self::Boolean, diff --git a/avro/src/types.rs b/avro/src/types.rs index 90adb7a5..b8beec8a 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -391,18 +391,18 @@ mod types { /// /// See the [Avro specification](https://avro.apache.org/docs/current/specification) /// for the full set of rules of schema validation. - pub fn validate(&self, schema: &Schema) -> bool { - self.validate_schemata(vec![schema]) + pub async fn validate(&self, schema: &Schema) -> bool { + self.validate_schemata(vec![schema]).await } - pub fn validate_schemata(&self, schemata: Vec<&Schema>) -> bool { + pub async fn validate_schemata(&self, schemata: Vec<&Schema>) -> bool { let rs = ResolvedSchema::try_from(schemata.clone()) .expect("Schemata didn't successfully resolve"); let schemata_len = schemata.len(); - schemata.iter().any(|schema| { + for schema in schemata { let enclosing_namespace = schema.namespace(); - match self.validate_internal(schema, rs.get_names(), &enclosing_namespace) { + match self.validate_internal(schema, rs.get_names(), &enclosing_namespace).await { Some(reason) => { let log_message = format!( "Invalid value: {self:?} for schema: {schema:?}. Reason: {reason}" @@ -412,11 +412,11 @@ mod types { } else { debug!("{log_message}"); }; - false } - None => true, + None => return true, } - }) + }; + false } fn accumulate(accumulator: Option, other: Option) -> Option { @@ -429,7 +429,7 @@ mod types { } /// Validates the value against the provided schema. - pub(crate) fn validate_internal + Debug>( + pub(crate) async fn validate_internal + Debug>( &self, schema: &Schema, names: &HashMap, @@ -446,7 +446,7 @@ mod types { names.keys() )) }, - |s| self.validate_internal(s.borrow(), names, &name.namespace), + async |s: &S| self.validate_internal(s.borrow(), names, &name.namespace).await, ) } (&Value::Null, &Schema::Null) => None, @@ -541,30 +541,46 @@ mod types { (&Value::Union(i, ref value), Schema::Union(inner)) => inner .variants() .get(i as usize) - .map(|schema| value.validate_internal(schema, names, enclosing_namespace)) + .map(|schema| value.validate_internal(schema, names, enclosing_namespace).await) .unwrap_or_else(|| Some(format!("No schema in the union at position '{i}'"))), (v, Schema::Union(inner)) => { - match inner.find_schema_with_known_schemata(v, Some(names), enclosing_namespace) + match inner.find_schema_with_known_schemata(v, Some(names), enclosing_namespace).await { Some(_) => None, None => Some("Could not find matching type in union".to_string()), } } (Value::Array(items), Schema::Array(inner)) => { - items.iter().fold(None, |acc, item| { - Value::accumulate( + let mut acc = None; + for item in items.iter() { + acc = Value::accumulate( acc, - item.validate_internal(&inner.items, names, enclosing_namespace), - ) - }) + item.validate_internal(&inner.items, names, enclosing_namespace).await, + ); + } + acc + // items.iter().fold(None, |acc, item| { + // Value::accumulate( + // acc, + // item.validate_internal(&inner.items, names, enclosing_namespace).await, + // ) + // }) } (Value::Map(items), Schema::Map(inner)) => { - items.iter().fold(None, |acc, (_, value)| { - Value::accumulate( + let mut acc = None; + for (_, value) in items.iter() { + acc = Value::accumulate( acc, - value.validate_internal(&inner.types, names, enclosing_namespace), - ) - }) + value.validate_internal(&inner.types, names, enclosing_namespace).await, + ); + } + acc + // items.iter().fold(None, |acc, (_, value)| { + // Value::accumulate( + // acc, + // value.validate_internal(&inner.types, names, enclosing_namespace), + // ) + // }) } ( Value::Record(record_fields), @@ -593,15 +609,14 @@ mod types { )); } - record_fields - .iter() - .fold(None, |acc, (field_name, record_field)| { + let mut acc = None; + for (field_name, record_field) in record_fields.iter() { let record_namespace = if name.namespace.is_none() { enclosing_namespace } else { &name.namespace }; - match lookup.get(field_name) { + acc = match lookup.get(field_name) { Some(idx) => { let field = &fields[*idx]; Value::accumulate( @@ -610,7 +625,7 @@ mod types { &field.schema, names, record_namespace, - ), + ).await ) } None => Value::accumulate( @@ -619,27 +634,28 @@ mod types { "There is no schema field for field '{field_name}'" )), ), - } - }) + }; + } + acc } (Value::Map(items), Schema::Record(RecordSchema { fields, .. })) => { - fields.iter().fold(None, |acc, field| { + let mut acc = None; + for field in fields.iter() { if let Some(item) = items.get(&field.name) { let res = - item.validate_internal(&field.schema, names, enclosing_namespace); - Value::accumulate(acc, res) + item.validate_internal(&field.schema, names, enclosing_namespace).await; + acc = Value::accumulate(acc, res); } else if !field.is_nullable() { - Value::accumulate( + acc = Value::accumulate( acc, Some(format!( "Field with name '{:?}' is not a member of the map items", field.name )), - ) - } else { - acc + ); } - }) + }; + acc } (v, s) => Some(format!( "Unsupported value-schema combination! Value: {v:?}, schema: {s:?}" @@ -2819,7 +2835,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3460_validation_with_refs_real_struct() -> TestResult { - use crate::ser::Serializer; + use crate::ser::tokio::Serializer; use serde::Serialize; #[derive(Serialize, Clone)] diff --git a/avro/src/writer.rs b/avro/src/writer.rs index f8d8838a..4e9b98c3 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -55,6 +55,8 @@ mod writer { collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive, }; + #[cfg(feature = "tokio")] + use futures::TryFutureExt; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; @@ -187,11 +189,11 @@ mod writer { /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). - pub fn append>(&mut self, value: T) -> AvroResult { + pub async fn append>(&mut self, value: T) -> AvroResult { let n = self.maybe_write_header()?; let avro = value.into(); - self.append_value_ref(&avro).map(|m| m + n) + self.append_value_ref(&avro).map_ok(|m| m + n).await } /// Append a compatible value to a `Writer`, also performing schema validation. @@ -201,13 +203,13 @@ mod writer { /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). - pub fn append_value_ref(&mut self, value: &Value) -> AvroResult { + pub async fn append_value_ref(&mut self, value: &Value) -> AvroResult { let n = self.maybe_write_header()?; // Lazy init for users using the builder pattern with error throwing match self.resolved_schema { Some(ref rs) => { - write_value_ref_resolved(self.schema, rs, value, &mut self.buffer)?; + write_value_ref_resolved(self.schema, rs, value, &mut self.buffer).await?; self.num_values += 1; if self.buffer.len() >= self.block_size { @@ -219,7 +221,7 @@ mod writer { None => { let rs = ResolvedSchema::try_from(self.schema)?; self.resolved_schema = Some(rs); - self.append_value_ref(value) + self.append_value_ref(value).await } } } @@ -268,7 +270,7 @@ mod writer { /// /// **NOTE**: This function forces the written data to be flushed (an implicit /// call to [`flush`](Writer::flush) is performed). - pub fn extend>(&mut self, values: I) -> AvroResult + pub async fn extend>(&mut self, values: I) -> AvroResult where I: IntoIterator, { @@ -288,7 +290,7 @@ mod writer { let mut num_bytes = 0; for value in values { - num_bytes += self.append(value)?; + num_bytes += self.append(value).await?; } num_bytes += self.flush()?; @@ -337,10 +339,10 @@ mod writer { /// /// **NOTE**: This function forces the written data to be flushed (an implicit /// call to [`flush`](Writer::flush) is performed). - pub fn extend_from_slice(&mut self, values: &[Value]) -> AvroResult { + pub async fn extend_from_slice(&mut self, values: &[Value]) -> AvroResult { let mut num_bytes = 0; for value in values { - num_bytes += self.append_value_ref(value)?; + num_bytes += self.append_value_ref(value).await?; } num_bytes += self.flush()?; @@ -531,20 +533,20 @@ mod writer { /// /// This is an internal function which gets the bytes buffer where to write as parameter instead of /// creating a new one like `to_avro_datum`. - fn write_avro_datum, W: Write>( + async fn write_avro_datum, W: Write>( schema: &Schema, value: T, writer: &mut W, ) -> Result<(), Error> { let avro = value.into(); - if !avro.validate(schema) { + if !avro.validate(schema).await { return Err(Details::Validation.into()); } encode(&avro, schema, writer)?; Ok(()) } - fn write_avro_datum_schemata>( + async fn write_avro_datum_schemata>( schema: &Schema, schemata: Vec<&Schema>, value: T, @@ -554,7 +556,7 @@ mod writer { let rs = ResolvedSchema::try_from(schemata)?; let names = rs.get_names(); let enclosing_namespace = schema.namespace(); - if let Some(_err) = avro.validate_internal(schema, names, &enclosing_namespace) { + if let Some(_err) = avro.validate_internal(schema, names, &enclosing_namespace).await { return Err(Details::Validation.into()); } encode_internal(&avro, schema, names, &enclosing_namespace, buffer) @@ -595,7 +597,7 @@ mod writer { const HEADER_LENGTH_RANGE: RangeInclusive = 10_usize..=20_usize; /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header - pub fn write_value_ref( + pub async fn write_value_ref( &mut self, v: &Value, writer: &mut W, @@ -604,7 +606,7 @@ mod writer { if !Self::HEADER_LENGTH_RANGE.contains(&original_length) { Err(Details::IllegalSingleObjectWriterState.into()) } else { - write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?; + write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer).await?; writer .write_all(&self.buffer) .map_err(Details::WriteBytes)?; @@ -615,8 +617,8 @@ mod writer { } /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header - pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { - self.write_value_ref(&v, writer) + pub async fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { + self.write_value_ref(&v, writer).await } } @@ -652,9 +654,9 @@ mod writer { { /// Write the `Into` to the provided Write object. Returns a result with the number /// of bytes written including the header - pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { + pub async fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { let v: Value = data.into(); - self.inner.write_value_ref(&v, writer) + self.inner.write_value_ref(&v, writer).await } } @@ -686,13 +688,13 @@ mod writer { } } - fn write_value_ref_resolved( + async fn write_value_ref_resolved( schema: &Schema, - resolved_schema: &ResolvedSchema, + resolved_schema: &ResolvedSchema<'_>, value: &Value, buffer: &mut Vec, ) -> AvroResult { - match value.validate_internal(schema, resolved_schema.get_names(), &schema.namespace()) { + match value.validate_internal(schema, resolved_schema.get_names(), &schema.namespace()).await { Some(reason) => Err(Details::ValidationWithReason { value: value.clone(), schema: schema.clone(), @@ -709,7 +711,7 @@ mod writer { } } - fn write_value_ref_owned_resolved( + async fn write_value_ref_owned_resolved( resolved_schema: &ResolvedOwnedSchema, value: &Value, buffer: &mut Vec, @@ -719,7 +721,7 @@ mod writer { root_schema, resolved_schema.get_names(), &root_schema.namespace(), - ) { + ).await { return Err(Details::ValidationWithReason { value: value.clone(), schema: root_schema.clone(), @@ -743,9 +745,9 @@ mod writer { /// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync /// markers; use [`Writer`] to be fully Avro-compatible if you don't know what /// you are doing, instead. - pub fn to_avro_datum>(schema: &Schema, value: T) -> AvroResult> { + pub async fn to_avro_datum>(schema: &Schema, value: T) -> AvroResult> { let mut buffer = Vec::new(); - write_avro_datum(schema, value, &mut buffer)?; + write_avro_datum(schema, value, &mut buffer).await?; Ok(buffer) } @@ -770,13 +772,13 @@ mod writer { /// performing schema validation. /// If the provided `schema` is incomplete then its dependencies must be /// provided in `schemata` - pub fn to_avro_datum_schemata>( + pub async fn to_avro_datum_schemata>( schema: &Schema, schemata: Vec<&Schema>, value: T, ) -> AvroResult> { let mut buffer = Vec::new(); - write_avro_datum_schemata(schema, schemata, value, &mut buffer)?; + write_avro_datum_schemata(schema, schemata, value, &mut buffer).await?; Ok(buffer) } From 66fef27607e7c9a3c094e677e848cfb1b3226e14 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 4 Aug 2025 11:47:58 +0300 Subject: [PATCH 05/47] WIP Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/decode.rs | 4 +-- avro/src/reader.rs | 4 +-- avro/src/schema.rs | 31 +++++++++---------- avro/src/types.rs | 77 ++++++++++++++++++++++++---------------------- avro/src/util.rs | 4 +-- 5 files changed, 62 insertions(+), 58 deletions(-) diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 4b308b87..6baac335 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -35,9 +35,9 @@ } )] mod decode { - #[cfg(feature = "sync")] + #[synca::cfg(sync)] use std::io::Read as AvroRead; - #[cfg(feature = "tokio")] + #[synca::cfg(tokio)] use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index a99ebee2..1addcb80 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -35,9 +35,9 @@ } )] mod reader { - #[cfg(feature = "sync")] + #[synca::cfg(sync)] use std::io::Read as AvroRead; - #[cfg(feature = "tokio")] + #[synca::cfg(tokio)] use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 1fd613a4..5dcc5ca5 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -24,17 +24,17 @@ pub mod sync { sync!(); replace!( - bigdecimal::tokio => bigdecimal::sync, - decimal::tokio => decimal::sync, - decode::tokio => decode::sync, - encode::tokio => encode::sync, - error::tokio => error::sync, - schema::tokio => schema::sync, - util::tokio => util::sync, - types::tokio => types::sync, - schema_equality::tokio => schema_equality::sync, - util::tokio => util::sync, - validator::tokio => validator::sync, + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decimal::tokio => crate::decimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::schema_equality::tokio => crate::schema_equality::sync, + crate::util::tokio => crate::util::sync, + crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -62,7 +62,6 @@ mod schema { use std::{ borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, - fmt, fmt::Debug, hash::Hash, io::Read, @@ -77,8 +76,8 @@ mod schema { pub bytes: Vec, } - impl fmt::Display for SchemaFingerprint { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + impl std::fmt::Display for SchemaFingerprint { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, "{}", @@ -370,8 +369,8 @@ mod schema { } } - impl fmt::Display for Name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + impl std::fmt::Display for Name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.fullname(None)[..]) } } diff --git a/avro/src/types.rs b/avro/src/types.rs index b8beec8a..aff70224 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -53,8 +53,8 @@ mod types { }, }; use bigdecimal::BigDecimal; - #[cfg(feature = "tokio")] - use futures::future::try_join_all; + #[synca::cfg(tokio)] + use futures::FutureExt; use log::{debug, error}; use serde_json::{Number, Value as JsonValue}; use std::{ @@ -438,16 +438,15 @@ mod types { match (self, schema) { (_, Schema::Ref { name }) => { let name = name.fully_qualified_name(enclosing_namespace); - names.get(&name).map_or_else( - || { - Some(format!( - "Unresolved schema reference: '{:?}'. Parsed names: {:?}", - name, - names.keys() - )) - }, - async |s: &S| self.validate_internal(s.borrow(), names, &name.namespace).await, - ) + if let Some(s) = names.get(&name) { + Box::pin(self.validate_internal(s.borrow(), names, &name.namespace)).await + } else { + Some(format!( + "Unresolved schema reference: '{:?}'. Parsed names: {:?}", + name, + names.keys() + )) + } } (&Value::Null, &Schema::Null) => None, (&Value::Boolean(_), &Schema::Boolean) => None, @@ -538,11 +537,13 @@ mod types { None => Some(format!("No symbol at position '{i}'")), }), // (&Value::Union(None), &Schema::Union(_)) => None, - (&Value::Union(i, ref value), Schema::Union(inner)) => inner - .variants() - .get(i as usize) - .map(|schema| value.validate_internal(schema, names, enclosing_namespace).await) - .unwrap_or_else(|| Some(format!("No schema in the union at position '{i}'"))), + (&Value::Union(i, ref value), Schema::Union(inner)) => { + if let Some(schema) = inner.variants().get(i as usize) { + Box::pin(value.validate_internal(schema, names, enclosing_namespace)).await + } else { + Some(format!("No schema in the union at position '{i}'")) + } + }, (v, Schema::Union(inner)) => { match inner.find_schema_with_known_schemata(v, Some(names), enclosing_namespace).await { @@ -753,7 +754,7 @@ mod types { .await } Schema::Map(ref inner) => { - self.resolve_map(&inner.types, names, enclosing_namespace) + self.resolve_map(&inner.types, names, enclosing_namespace).await } Schema::Record(RecordSchema { ref fields, .. }) => { self.resolve_record(fields, names, enclosing_namespace) @@ -1128,12 +1129,14 @@ mod types { enclosing_namespace: &Namespace, ) -> Result { match self { - Value::Array(items) => Ok(Value::Array( - try_join_all(items.into_iter().map(|item| { + Value::Array(items) => { + let resolved = items.into_iter().map(|item| { item.resolve_internal(schema, names, enclosing_namespace, &None) - })) - .await?, - )), + }); + #[synca::cfg(tokio)] + let resolved = futures::future::try_join_all(resolved).await?; + Ok(Value::Array(resolved)) + }, other => Err(Details::GetArray { expected: schema.into(), other, @@ -1142,24 +1145,26 @@ mod types { } } - fn resolve_map + Debug>( + async fn resolve_map + Debug>( self, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, ) -> Result { match self { - Value::Map(items) => Ok(Value::Map( - items + Value::Map(items) => { + let resolved = items .into_iter() - .map(async |(key, value)| { + .map(|(key, value)| { value .resolve_internal(schema, names, enclosing_namespace, &None) - .await .map(|value| (key, value)) }) - .collect::>()?, - )), + .collect::>(); + #[synca::cfg(tokio)] + let resolved = futures::future::try_join_all(resolved).await?; + Ok(Value::Map(resolved)) + }, other => Err(Details::GetMap { expected: schema.into(), other, @@ -1186,7 +1191,7 @@ mod types { })), }?; - let new_fields = try_join_all(fields.iter().map(|field| async { + let new_fields = fields.iter().map(|field| { let value = match items.remove(&field.name) { Some(value) => value, None => match field.default { @@ -1215,8 +1220,7 @@ mod types { names, enclosing_namespace, &field.default, - ) - .await?, + ), ), ), } @@ -1228,13 +1232,14 @@ mod types { } }, }; + value .resolve_internal(&field.schema, names, enclosing_namespace, &field.default) - .await .map(|value| (field.name.clone(), value)) - })) - .await?; + }); + #[synca::cfg(tokio)] + let new_fields = futures::future::try_join_all(new_fields).await?; Ok(Value::Record(new_fields)) } diff --git a/avro/src/util.rs b/avro/src/util.rs index dddab248..2de5ff8d 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -93,9 +93,9 @@ pub fn set_serde_human_readable(human_readable: bool) { mod util { #[cfg(feature = "tokio")] use futures::future::TryFutureExt; - #[cfg(feature = "sync")] + #[synca::cfg(sync)] use std::io::Read as AvroRead; - #[cfg(feature = "tokio")] + #[synca::cfg(tokio)] use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; From 3e4b2b26c4930052903096ec5abcdac39d3774f2 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 4 Aug 2025 14:32:19 +0300 Subject: [PATCH 06/47] WIP Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/de.rs | 2 +- avro/src/lib.rs | 48 +++++++++-- avro/src/schema.rs | 91 ++++++++++---------- avro/src/types.rs | 201 ++++++++++++++++++++++----------------------- 4 files changed, 187 insertions(+), 155 deletions(-) diff --git a/avro/src/de.rs b/avro/src/de.rs index b9c4a578..5d80f5e4 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -39,7 +39,7 @@ mod de { use crate::{ - Error, bytes::DE_BYTES_BORROWED, error::tokio::Details, schema::tokio::SchemaKind, + error::tokio::Error, bytes::DE_BYTES_BORROWED, error::tokio::Details, schema::tokio::SchemaKind, types::tokio::Value, }; use serde::{ diff --git a/avro/src/lib.rs b/avro/src/lib.rs index c2c8848c..4c0a6b64 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -887,6 +887,8 @@ pub use crate::bytes::{ serde_avro_slice, serde_avro_slice_opt, }; #[cfg(feature = "tokio")] +pub use crate::bigdecimal::tokio::BigDecimal as AsyncBigDecimal; +#[cfg(feature = "sync")] pub use crate::bigdecimal::tokio::BigDecimal; #[cfg(feature = "bzip")] pub use codec::bzip::Bzip2Settings; @@ -895,27 +897,56 @@ pub use codec::xz::XzSettings; #[cfg(feature = "zstandard")] pub use codec::zstandard::ZstandardSettings; #[cfg(feature = "tokio")] -pub use codec::tokio::{Codec, DeflateSettings}; +pub use codec::tokio::{Codec as AsyncCodec, DeflateSettings as AsyncDeflateSettings}; +#[cfg(feature = "sync")] +pub use codec::sync::{Codec, DeflateSettings}; #[cfg(feature = "tokio")] -pub use de::tokio::from_value; +pub use de::tokio::from_value as async_from_value; +#[cfg(feature = "sync")] +pub use de::sync::from_value; #[cfg(feature = "tokio")] -pub use decimal::tokio::Decimal; +pub use decimal::tokio::Decimal as AsyncDecimal; +#[cfg(feature = "sync")] +pub use decimal::sync::Decimal; pub use duration::{Days, Duration, Millis, Months}; #[cfg(feature = "tokio")] -pub use error::tokio::Error; +pub use error::tokio::Error as AsyncError; +#[cfg(feature = "sync")] +pub use error::sync::Error; #[cfg(feature = "tokio")] pub use reader::tokio::{ - GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, + GenericSingleObjectReader as AsyncGenericSingleObjectReader, Reader as AsyncReader, + SpecificSingleObjectReader as AsyncSpecificSingleObjectReader, from_avro_datum as async_from_avro_datum, + from_avro_datum_reader_schemata as async_from_avro_datum_reader_schemata, + from_avro_datum_schemata as async_from_avro_datum_schemata, read_marker as async_read_marker, +}; +#[cfg(feature = "sync")] +pub use reader::sync::{ + GenericSingleObjectReader, Reader, + SpecificSingleObjectReader, from_avro_datum, from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, }; #[cfg(feature = "tokio")] -pub use schema::tokio::{AvroSchema, Schema}; +pub use schema::tokio::{AvroSchema as AsyncAvroSchema, Schema as AsyncSchema}; +#[cfg(feature = "sync")] +pub use schema::sync::{AvroSchema, Schema}; #[cfg(feature = "tokio")] -pub use ser::tokio::to_value; +pub use ser::tokio::to_value as async_to_value; +#[cfg(feature = "sync")] +pub use ser::sync::to_value; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; #[cfg(feature = "tokio")] pub use writer::tokio::{ + GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, + SpecificSingleObjectWriter as AsyncSpecificSingleObjectWriter, + Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, + to_avro_datum as async_to_avro_datum, + to_avro_datum_schemata as async_to_avro_datum_schemata, + write_avro_datum_ref as async_write_avro_datum_ref, +}; +#[cfg(feature = "sync")] +pub use writer::sync::{ GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, to_avro_datum_schemata, write_avro_datum_ref, }; @@ -924,6 +955,9 @@ pub use writer::tokio::{ pub use apache_avro_derive::*; /// A convenience type alias for `Result`s with `Error`s. +#[cfg(feature = "tokio")] +pub type AsyncAvroResult = Result; +#[cfg(feature = "sync")] pub type AvroResult = Result; #[cfg(test)] diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 5dcc5ca5..7bae38e6 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -44,8 +44,8 @@ mod schema { use crate::{ AvroResult, error::tokio::{Details, Error}, - schema_equality::tokio as schema_equality, - types::tokio as types, + schema_equality::tokio::compare_schemata, + types::tokio::{Value, ValueKind}, util::tokio::MapHelper, validator::tokio::{ validate_enum_symbol_name, validate_namespace, validate_record_field_name, @@ -58,7 +58,6 @@ mod schema { Deserialize, Serialize, Serializer, ser::{SerializeMap, SerializeSeq}, }; - use serde_json::{Map, Value}; use std::{ borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, @@ -167,13 +166,13 @@ mod schema { #[derive(Clone, Debug, PartialEq)] pub struct MapSchema { pub types: Box, - pub attributes: BTreeMap, + pub attributes: BTreeMap, } #[derive(Clone, Debug, PartialEq)] pub struct ArraySchema { pub items: Box, - pub attributes: BTreeMap, + pub attributes: BTreeMap, } impl PartialEq for Schema { @@ -182,7 +181,7 @@ mod schema { /// [Parsing Canonical Form]: /// https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas fn eq(&self, other: &Self) -> bool { - schema_equality::compare_schemata(self, other) + compare_schemata(self, other) } } @@ -209,8 +208,8 @@ mod schema { } } - impl From<&types::Value> for SchemaKind { - fn from(value: &types::Value) -> Self { + impl From<&Value> for SchemaKind { + fn from(value: &Value) -> Self { use crate::types::tokio::Value; match value { Value::Null => Self::Null, @@ -289,7 +288,7 @@ mod schema { /// Parse a `serde_json::Value` into a `Name`. pub(crate) fn parse( - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { let (name, namespace_from_name) = complex @@ -298,7 +297,7 @@ mod schema { .ok_or(Details::GetNameField)?; // FIXME Reading name from the type is wrong ! The name there is just a metadata (AVRO-3430) let type_name = match complex.get("type") { - Some(Value::Object(complex_type)) => complex_type.name().or(None), + Some(serde_json::Value::Object(complex_type)) => complex_type.name().or(None), _ => None, }; @@ -699,7 +698,7 @@ mod schema { impl RecordField { /// Parse a `serde_json::Value` into a `RecordField`. async fn parse( - field: &Map, + field: &serde_json::Map, position: usize, parser: &mut Parser, enclosing_record: &Name, @@ -721,7 +720,7 @@ mod schema { &name, &enclosing_record.fullname(None), &parser.parsed_schemas, - &default, + &default.into(), ) .await?; @@ -744,7 +743,7 @@ mod schema { Ok(RecordField { name, doc: field.doc(), - default, + default: default.into(), aliases, order, position, @@ -761,7 +760,7 @@ mod schema { default: &Option, ) -> AvroResult<()> { if let Some(value) = default { - let avro_value = types::Value::from(value.clone()); + let avro_value = Value::from(value.clone()); match field_schema { Schema::Union(union_schema) => { let schemas = &union_schema.schemas; @@ -784,7 +783,7 @@ mod schema { return match schema { Some(first_schema) => Err(Details::GetDefaultUnion( SchemaKind::from(first_schema), - types::ValueKind::from(avro_value), + ValueKind::from(avro_value), ) .into()), None => Err(Details::EmptyUnion.into()), @@ -813,7 +812,7 @@ mod schema { } fn get_field_custom_attributes( - field: &Map, + field: &serde_json::Map, schema: &Schema, ) -> BTreeMap { let mut custom_attributes: BTreeMap = BTreeMap::new(); @@ -988,7 +987,7 @@ mod schema { /// - `known_schemata` - mapping between `Name` and `Schema` - if passed, additional external schemas would be used to resolve references. pub async fn find_schema_with_known_schemata + Debug>( &self, - value: &types::Value, + value: &Value, known_schemata: Option<&HashMap>, enclosing_namespace: &Namespace, ) -> Option<(usize, &Schema)> { @@ -1500,14 +1499,14 @@ mod schema { } fn parse_precision_and_scale( - complex: &Map, + complex: &serde_json::Map, ) -> Result<(Precision, Scale), Error> { fn get_decimal_integer( - complex: &Map, + complex: &serde_json::Map, key: &'static str, ) -> Result { match complex.get(key) { - Some(Value::Number(value)) => parse_json_integer_for_decimal(value), + Some(serde_json::Value::Number(value)) => parse_json_integer_for_decimal(value), None => { if key == "scale" { Ok(0) @@ -1543,19 +1542,19 @@ mod schema { /// e.g: {"type": {"type": "string"}} fn parse_complex( &mut self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, parse_location: RecordSchemaParseLocation, ) -> AvroResult { // Try to parse this as a native complex type. fn parse_as_native_complex( - complex: &Map, + complex: &serde_json::Map, parser: &mut Parser, enclosing_namespace: &Namespace, ) -> AvroResult { match complex.get("type") { Some(value) => match value { - Value::String(s) if s == "fixed" => { + serde_json::Value::String(s) if s == "fixed" => { parser.parse_fixed(complex, enclosing_namespace) } _ => parser.parse(value, enclosing_namespace), @@ -1591,7 +1590,7 @@ mod schema { } match complex.get("logicalType") { - Some(Value::String(t)) => match t.as_str() { + Some(serde_json::Value::String(t)) => match t.as_str() { "decimal" => { return try_convert_to_logical_type( "decimal", @@ -1734,7 +1733,7 @@ mod schema { _ => {} } match complex.get("type") { - Some(Value::String(t)) => match t.as_str() { + Some(serde_json::Value::String(t)) => match t.as_str() { "record" => match parse_location { RecordSchemaParseLocation::Root => { self.parse_record(complex, enclosing_namespace) @@ -1749,10 +1748,10 @@ mod schema { "fixed" => self.parse_fixed(complex, enclosing_namespace), other => self.parse_known_schema(other, enclosing_namespace), }, - Some(Value::Object(data)) => { + Some(serde_json::Value::Object(data)) => { self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) } - Some(Value::Array(variants)) => self.parse_union(variants, enclosing_namespace), + Some(serde_json::Value::Array(variants)) => self.parse_union(variants, enclosing_namespace), Some(unknown) => Err(Details::GetComplexType(unknown.clone()).into()), None => Err(Details::GetComplexTypeField.into()), } @@ -1800,7 +1799,7 @@ mod schema { /// Returns already parsed schema or a schema that is currently being resolved. fn get_already_seen_schema( &self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> Option<&Schema> { match complex.get("type") { @@ -1820,7 +1819,7 @@ mod schema { /// `Schema`. fn parse_record( &mut self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { let fields_opt = complex.get("fields"); @@ -1881,10 +1880,10 @@ mod schema { fn get_custom_attributes( &self, - complex: &Map, + complex: &serde_json::Map, excluded: Vec<&'static str>, ) -> BTreeMap { - let mut custom_attributes: BTreeMap = BTreeMap::new(); + let mut custom_attributes: BTreeMap = BTreeMap::new(); for (key, value) in complex { match key.as_str() { "type" | "name" | "namespace" | "doc" | "aliases" => continue, @@ -1899,7 +1898,7 @@ mod schema { /// `Schema`. fn parse_enum( &mut self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { let symbols_opt = complex.get("symbols"); @@ -1946,7 +1945,7 @@ mod schema { } if let Some(ref value) = default { - let resolved = types::Value::from(value.clone()) + let resolved = Value::from(value.clone()) .resolve_enum(&symbols, &Some(value.to_string()), &None) .is_ok(); if !resolved { @@ -1976,7 +1975,7 @@ mod schema { /// `Schema`. fn parse_array( &mut self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { complex @@ -1995,7 +1994,7 @@ mod schema { /// `Schema`. fn parse_map( &mut self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { complex @@ -2043,7 +2042,7 @@ mod schema { /// `Schema`. fn parse_fixed( &mut self, - complex: &Map, + complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { let size_opt = complex.get("size"); @@ -2119,9 +2118,9 @@ mod schema { }) } - fn get_schema_type_name(name: Name, value: Value) -> Name { + fn get_schema_type_name(name: Name, value: serde_json::Value) -> Name { match value.get("type") { - Some(Value::Object(complex_type)) => match complex_type.name() { + Some(serde_json::Value::Object(complex_type)) => match complex_type.name() { Some(name) => Name::new(name.as_str()).unwrap(), _ => name, }, @@ -2373,7 +2372,7 @@ mod schema { } } - fn pcf_map(schema: &Map, defined_names: &mut HashSet) -> String { + fn pcf_map(schema: &serde_json::Map, defined_names: &mut HashSet) -> String { // Look for the namespace variant up front. let ns = schema.get("namespace").and_then(|v| v.as_str()); let typ = schema.get("type").and_then(|v| v.as_str()); @@ -2634,7 +2633,7 @@ mod schema { } } - impl AvroSchemaComponent for Map + impl AvroSchemaComponent for serde_json::Map where T: AvroSchemaComponent, { @@ -2698,7 +2697,7 @@ mod schema { #[cfg(test)] mod tests { use super::*; - use {SpecificSingleObjectWriter, error::Details, rabin::Rabin}; + use {SpecificSingleObjectWriter, error::tokio::Details, rabin::Rabin}; use apache_avro_test_helper::{ TestResult, logger::{assert_logged, assert_not_logged}, @@ -5436,12 +5435,12 @@ mod schema { let reader_schema = Schema::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { - types::Value::Record(fields) => { + Value::Record(fields) => { assert_eq!(fields.len(), 2); assert_eq!(fields[0].0, "barInit"); - assert_eq!(fields[0].1, types::Value::Enum(0, "bar0".to_string())); + assert_eq!(fields[0].1, Value::Enum(0, "bar0".to_string())); assert_eq!(fields[1].0, "barUse"); - assert_eq!(fields[1].1, types::Value::Enum(1, "bar1".to_string())); + assert_eq!(fields[1].1, Value::Enum(1, "bar1".to_string())); } _ => panic!("Expected Value::Record"), } @@ -7153,8 +7152,8 @@ mod schema { assert_eq!(field.name, "birthday"); assert_eq!(field.schema, Schema::Date); assert_eq!( - types::Value::from(field.default.clone().unwrap()), - types::Value::Int(1681601653) + Value::from(field.default.clone().unwrap()), + Value::Int(1681601653) ); } _ => unreachable!("Expected Schema::Record"), diff --git a/avro/src/types.rs b/avro/src/types.rs index aff70224..949da90a 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -56,7 +56,6 @@ mod types { #[synca::cfg(tokio)] use futures::FutureExt; use log::{debug, error}; - use serde_json::{Number, Value as JsonValue}; use std::{ borrow::Borrow, collections::{BTreeMap, HashMap}, @@ -295,12 +294,12 @@ mod types { } } - impl From for Value { - fn from(value: JsonValue) -> Self { + impl From for Value { + fn from(value: serde_json::Value) -> Self { match value { - JsonValue::Null => Self::Null, - JsonValue::Bool(b) => b.into(), - JsonValue::Number(ref n) if n.is_i64() => { + serde_json::Value::Null => Self::Null, + serde_json::Value::Bool(b) => b.into(), + serde_json::Value::Number(ref n) if n.is_i64() => { let n = n.as_i64().unwrap(); if n >= i32::MIN as i64 && n <= i32::MAX as i64 { Value::Int(n as i32) @@ -308,13 +307,13 @@ mod types { Value::Long(n) } } - JsonValue::Number(ref n) if n.is_f64() => Value::Double(n.as_f64().unwrap()), - JsonValue::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great - JsonValue::String(s) => s.into(), - JsonValue::Array(items) => { + serde_json::Value::Number(ref n) if n.is_f64() => Value::Double(n.as_f64().unwrap()), + serde_json::Value::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great + serde_json::Value::String(s) => s.into(), + serde_json::Value::Array(items) => { Value::Array(items.into_iter().map(Value::from).collect()) } - JsonValue::Object(items) => Value::Map( + serde_json::Value::Object(items) => Value::Map( items .into_iter() .map(|(key, value)| (key, value.into())) @@ -325,7 +324,7 @@ mod types { } /// Convert Avro values to Json values - impl TryFrom for JsonValue { + impl TryFrom for serde_json::Value { type Error = Error; fn try_from(value: Value) -> AvroResult { match value { @@ -333,10 +332,10 @@ mod types { Value::Boolean(b) => Ok(Self::Bool(b)), Value::Int(i) => Ok(Self::Number(i.into())), Value::Long(l) => Ok(Self::Number(l.into())), - Value::Float(f) => Number::from_f64(f.into()) + Value::Float(f) => serde_json::Number::from_f64(f.into()) .map(Self::Number) .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), - Value::Double(d) => Number::from_f64(d) + Value::Double(d) => serde_json::Number::from_f64(d) .map(Self::Number) .ok_or_else(|| Details::ConvertF64ToJson(d).into()), Value::Bytes(bytes) => { @@ -700,7 +699,7 @@ mod types { schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, - field_default: &Option, + field_default: &Option, ) -> AvroResult { // Check if this schema is a union, and if the reader schema is not. if SchemaKind::from(&self) == SchemaKind::Union @@ -1058,7 +1057,7 @@ mod types { self, symbols: &[String], enum_default: &Option, - _field_default: &Option, + _field_default: &Option, ) -> Result { let validate_symbol = |symbol: String, symbols: &[String]| { if let Some(index) = symbols.iter().position(|item| item == &symbol) { @@ -1097,7 +1096,7 @@ mod types { schema: &UnionSchema, names: &HashMap, enclosing_namespace: &Namespace, - field_default: &Option, + field_default: &Option, ) -> Result { let v = match self { // Both are unions case. @@ -1260,8 +1259,8 @@ mod types { use super::*; use crate::{ duration::{Days, Millis, Months}, - error::Details, - schema::RecordFieldOrder, + error::tokio::Details, + schema::tokio::RecordFieldOrder, }; use apache_avro_test_helper::{ TestResult, @@ -1628,7 +1627,7 @@ mod types { RecordField { name: "c".to_string(), doc: None, - default: Some(JsonValue::Null), + default: Some(serde_json::Value::Null), aliases: None, schema: Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), order: RecordFieldOrder::Ascending, @@ -2058,69 +2057,69 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn json_from_avro() -> TestResult { - assert_eq!(JsonValue::try_from(Value::Null)?, JsonValue::Null); + assert_eq!(serde_json::Value::try_from(Value::Null)?, serde_json::Value::Null); assert_eq!( - JsonValue::try_from(Value::Boolean(true))?, - JsonValue::Bool(true) + serde_json::Value::try_from(Value::Boolean(true))?, + serde_json::Value::Bool(true) ); assert_eq!( - JsonValue::try_from(Value::Int(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::Int(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::Long(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::Long(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::Float(1.0))?, - JsonValue::Number(Number::from_f64(1.0).unwrap()) + serde_json::Value::try_from(Value::Float(1.0))?, + serde_json::Value::Number(Number::from_f64(1.0).unwrap()) ); assert_eq!( - JsonValue::try_from(Value::Double(1.0))?, - JsonValue::Number(Number::from_f64(1.0).unwrap()) + serde_json::Value::try_from(Value::Double(1.0))?, + serde_json::Value::Number(Number::from_f64(1.0).unwrap()) ); assert_eq!( - JsonValue::try_from(Value::Bytes(vec![1, 2, 3]))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) + serde_json::Value::try_from(Value::Bytes(vec![1, 2, 3]))?, + serde_json::Value::Array(vec![ + serde_json::Value::Number(1.into()), + serde_json::Value::Number(2.into()), + serde_json::Value::Number(3.into()) ]) ); assert_eq!( - JsonValue::try_from(Value::String("test".into()))?, - JsonValue::String("test".into()) + serde_json::Value::try_from(Value::String("test".into()))?, + serde_json::Value::String("test".into()) ); assert_eq!( - JsonValue::try_from(Value::Fixed(3, vec![1, 2, 3]))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) + serde_json::Value::try_from(Value::Fixed(3, vec![1, 2, 3]))?, + serde_json::Value::Array(vec![ + serde_json::Value::Number(1.into()), + serde_json::Value::Number(2.into()), + serde_json::Value::Number(3.into()) ]) ); assert_eq!( - JsonValue::try_from(Value::Enum(1, "test_enum".into()))?, - JsonValue::String("test_enum".into()) + serde_json::Value::try_from(Value::Enum(1, "test_enum".into()))?, + serde_json::Value::String("test_enum".into()) ); assert_eq!( - JsonValue::try_from(Value::Union(1, Box::new(Value::String("test_enum".into()))))?, - JsonValue::String("test_enum".into()) + serde_json::Value::try_from(Value::Union(1, Box::new(Value::String("test_enum".into()))))?, + serde_json::Value::String("test_enum".into()) ); assert_eq!( - JsonValue::try_from(Value::Array(vec![ + serde_json::Value::try_from(Value::Array(vec![ Value::Int(1), Value::Int(2), Value::Int(3) ]))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) + serde_json::Value::Array(vec![ + serde_json::Value::Number(1.into()), + serde_json::Value::Number(2.into()), + serde_json::Value::Number(3.into()) ]) ); assert_eq!( - JsonValue::try_from(Value::Map( + serde_json::Value::try_from(Value::Map( vec![ ("v1".to_string(), Value::Int(1)), ("v2".to_string(), Value::Int(2)), @@ -2129,103 +2128,103 @@ Field with name '"b"' is not a member of the map items"#, .into_iter() .collect() ))?, - JsonValue::Object( + serde_json::Value::Object( vec![ - ("v1".to_string(), JsonValue::Number(1.into())), - ("v2".to_string(), JsonValue::Number(2.into())), - ("v3".to_string(), JsonValue::Number(3.into())) + ("v1".to_string(), serde_json::Value::Number(1.into())), + ("v2".to_string(), serde_json::Value::Number(2.into())), + ("v3".to_string(), serde_json::Value::Number(3.into())) ] .into_iter() .collect() ) ); assert_eq!( - JsonValue::try_from(Value::Record(vec![ + serde_json::Value::try_from(Value::Record(vec![ ("v1".to_string(), Value::Int(1)), ("v2".to_string(), Value::Int(2)), ("v3".to_string(), Value::Int(3)) ]))?, - JsonValue::Object( + serde_json::Value::Object( vec![ - ("v1".to_string(), JsonValue::Number(1.into())), - ("v2".to_string(), JsonValue::Number(2.into())), - ("v3".to_string(), JsonValue::Number(3.into())) + ("v1".to_string(), serde_json::Value::Number(1.into())), + ("v2".to_string(), serde_json::Value::Number(2.into())), + ("v3".to_string(), serde_json::Value::Number(3.into())) ] .into_iter() .collect() ) ); assert_eq!( - JsonValue::try_from(Value::Date(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::Date(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::Decimal(vec![1, 2, 3].into()))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()) + serde_json::Value::try_from(Value::Decimal(vec![1, 2, 3].into()))?, + serde_json::Value::Array(vec![ + serde_json::Value::Number(1.into()), + serde_json::Value::Number(2.into()), + serde_json::Value::Number(3.into()) ]) ); assert_eq!( - JsonValue::try_from(Value::TimeMillis(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::TimeMillis(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::TimeMicros(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::TimeMicros(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::TimestampMillis(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::TimestampMillis(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::TimestampMicros(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::TimestampMicros(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::TimestampNanos(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::TimestampNanos(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::LocalTimestampMillis(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::LocalTimestampMillis(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::LocalTimestampMicros(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::LocalTimestampMicros(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::LocalTimestampNanos(1))?, - JsonValue::Number(1.into()) + serde_json::Value::try_from(Value::LocalTimestampNanos(1))?, + serde_json::Value::Number(1.into()) ); assert_eq!( - JsonValue::try_from(Value::Duration( + serde_json::Value::try_from(Value::Duration( [ 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8, 10u8, 11u8, 12u8 ] .into() ))?, - JsonValue::Array(vec![ - JsonValue::Number(1.into()), - JsonValue::Number(2.into()), - JsonValue::Number(3.into()), - JsonValue::Number(4.into()), - JsonValue::Number(5.into()), - JsonValue::Number(6.into()), - JsonValue::Number(7.into()), - JsonValue::Number(8.into()), - JsonValue::Number(9.into()), - JsonValue::Number(10.into()), - JsonValue::Number(11.into()), - JsonValue::Number(12.into()), + serde_json::Value::Array(vec![ + serde_json::Value::Number(1.into()), + serde_json::Value::Number(2.into()), + serde_json::Value::Number(3.into()), + serde_json::Value::Number(4.into()), + serde_json::Value::Number(5.into()), + serde_json::Value::Number(6.into()), + serde_json::Value::Number(7.into()), + serde_json::Value::Number(8.into()), + serde_json::Value::Number(9.into()), + serde_json::Value::Number(10.into()), + serde_json::Value::Number(11.into()), + serde_json::Value::Number(12.into()), ]) ); assert_eq!( - JsonValue::try_from(Value::Uuid(Uuid::parse_str( + serde_json::Value::try_from(Value::Uuid(Uuid::parse_str( "936DA01F-9ABD-4D9D-80C7-02AF85C822A8" )?))?, - JsonValue::String("936da01f-9abd-4d9d-80c7-02af85c822a8".into()) + serde_json::Value::String("936da01f-9abd-4d9d-80c7-02af85c822a8".into()) ); Ok(()) From f65cfac5355bcd18755ecfdc2f32d511fe67bbeb Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 4 Aug 2025 16:39:50 +0300 Subject: [PATCH 07/47] WIP: Fix regression with the usage of fully qualified serde_json::** items We need to avoid using `syn::UseRename` because it may cause issues after `synca` replacements Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 6 ++- avro/src/codec.rs | 8 +++- avro/src/decimal.rs | 6 ++- avro/src/decode.rs | 7 +++- avro/src/encode.rs | 6 ++- avro/src/headers.rs | 7 +++- avro/src/reader.rs | 12 +++++- avro/src/schema.rs | 83 ++++++++++++++++++++++-------------------- avro/src/ser.rs | 2 +- avro/src/types.rs | 6 ++- avro/src/util.rs | 5 ++- avro/src/validator.rs | 6 ++- avro/src/writer.rs | 7 +++- 13 files changed, 107 insertions(+), 54 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 981d9dbc..b3abedf4 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -36,7 +36,6 @@ )] mod bigdecimal { use crate::{ - AvroResult, decode::tokio::{decode_len, decode_long}, encode::tokio::{encode_bytes, encode_long}, error::tokio::Details, @@ -46,6 +45,11 @@ mod bigdecimal { use num_bigint::BigInt; use std::io::Read; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + pub(crate) fn big_decimal_as_bytes(decimal: &BigDecimal) -> AvroResult> { let mut buffer: Vec = Vec::new(); let (big_int, exponent): (BigInt, i64) = decimal.as_bigint_and_exponent(); diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 7bae9a44..323dda13 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -37,9 +37,13 @@ } )] mod codec { - use crate::{AvroResult, error::tokio::Details, error::tokio::Error, types::tokio::Value}; + use crate::{error::tokio::Details, error::tokio::Error, types::tokio::Value}; use strum_macros::{EnumIter, EnumString, IntoStaticStr}; - + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + /// Settings for the `Deflate` codec. #[derive(Clone, Copy, Eq, PartialEq, Debug)] pub struct DeflateSettings { diff --git a/avro/src/decimal.rs b/avro/src/decimal.rs index 7354a14a..6307fc8b 100644 --- a/avro/src/decimal.rs +++ b/avro/src/decimal.rs @@ -36,9 +36,13 @@ )] mod decimal { - use crate::{AvroResult, error::tokio::Details, error::tokio::Error}; + use crate::{error::tokio::Details, error::tokio::Error}; use num_bigint::{BigInt, Sign}; use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; #[derive(Debug, Clone, Eq)] pub struct Decimal { diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 6baac335..cc8db8fe 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -44,7 +44,7 @@ mod decode { use crate::util::tokio::{safe_len, zag_i32, zag_i64}; use crate::{ - AvroResult, Error, + error::tokio::Error, bigdecimal::tokio::deserialize_big_decimal, decimal::tokio::Decimal, duration::Duration, @@ -58,6 +58,11 @@ mod decode { }; use std::{borrow::Borrow, collections::HashMap, io::ErrorKind, str::FromStr}; use uuid::Uuid; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + #[inline] pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { diff --git a/avro/src/encode.rs b/avro/src/encode.rs index f4efbd4d..1cf8f7c5 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -39,7 +39,6 @@ mod encode { use crate::util::tokio::{zig_i32, zig_i64}; use crate::{ - AvroResult, bigdecimal::tokio::serialize_big_decimal, error::tokio::Details, schema::tokio::{ @@ -48,6 +47,11 @@ mod encode { }, types::tokio::{Value, ValueKind}, }; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + use log::error; use std::{borrow::Borrow, collections::HashMap, io::Write}; diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 1c505c70..9b72f281 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -40,9 +40,14 @@ mod headers { use uuid::Uuid; use crate::{ - AvroResult, error::tokio::Details, rabin::Rabin, schema::tokio::Schema, + error::tokio::Details, rabin::Rabin, schema::tokio::Schema, schema::tokio::SchemaFingerprint, }; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + /// This trait represents that an object is able to construct an Avro message header. It is /// implemented for some known header types already. If you need a header type that is not already diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 1addcb80..19bdc213 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -43,10 +43,18 @@ mod reader { use tokio::io::AsyncReadExt; use std::str::FromStr; use crate::decode::tokio::{decode, decode_internal}; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + #[synca::cfg(tokio)] + use crate::async_from_value as from_value; + #[synca::cfg(sync)] + use crate::from_value; + use crate::{ - AvroResult, codec::tokio::Codec, error::tokio::Error, + codec::tokio::Codec, error::tokio::Error, error::tokio::Details, - from_value, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, schema::tokio::{ AvroSchema, Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 7bae38e6..3853cfb6 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -41,8 +41,11 @@ )] mod schema { + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; use crate::{ - AvroResult, error::tokio::{Details, Error}, schema_equality::tokio::compare_schemata, types::tokio::{Value, ValueKind}, @@ -670,7 +673,7 @@ mod schema { /// Default value of the field. /// This value will be used when reading Avro datum if schema resolution /// is enabled. - pub default: Option, + pub default: Option, /// Schema of the field. pub schema: Schema, /// Order of the field. @@ -683,7 +686,7 @@ mod schema { pub position: usize, /// A collection of all unknown fields in the record field. #[builder(default = BTreeMap::new())] - pub custom_attributes: BTreeMap, + pub custom_attributes: BTreeMap, } /// Represents any valid order for a `field` in a `record` Avro schema. @@ -720,7 +723,7 @@ mod schema { &name, &enclosing_record.fullname(None), &parser.parsed_schemas, - &default.into(), + &default, ) .await?; @@ -743,7 +746,7 @@ mod schema { Ok(RecordField { name, doc: field.doc(), - default: default.into(), + default, aliases, order, position, @@ -757,7 +760,7 @@ mod schema { field_name: &str, record_name: &str, names: &Names, - default: &Option, + default: &Option, ) -> AvroResult<()> { if let Some(value) = default { let avro_value = Value::from(value.clone()); @@ -814,8 +817,8 @@ mod schema { fn get_field_custom_attributes( field: &serde_json::Map, schema: &Schema, - ) -> BTreeMap { - let mut custom_attributes: BTreeMap = BTreeMap::new(); + ) -> BTreeMap { + let mut custom_attributes: BTreeMap = BTreeMap::new(); for (key, value) in field { match key.as_str() { "type" | "name" | "doc" | "default" | "order" | "position" | "aliases" @@ -855,7 +858,7 @@ mod schema { pub lookup: BTreeMap, /// The custom attributes of the schema #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, + pub attributes: BTreeMap, } /// A description of an Enum schema. @@ -875,7 +878,7 @@ mod schema { pub default: Option, /// The custom attributes of the schema #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, + pub attributes: BTreeMap, } /// A description of a Union schema. @@ -895,7 +898,7 @@ mod schema { pub default: Option, /// The custom attributes of the schema #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, + pub attributes: BTreeMap, } impl FixedSchema { @@ -1081,7 +1084,7 @@ mod schema { #[derive(Default)] struct Parser { - input_schemas: HashMap, + input_schemas: HashMap, /// A map of name -> Schema::Ref /// Used to resolve cyclic references, i.e. when a /// field's type is a reference to its record's type @@ -1147,12 +1150,12 @@ mod schema { ) -> AvroResult> { let input = input.into_iter(); let input_len = input.size_hint().0; - let mut input_schemas: HashMap = HashMap::with_capacity(input_len); + let mut input_schemas: HashMap = HashMap::with_capacity(input_len); let mut input_order: Vec = Vec::with_capacity(input_len); for json in input { let json = json.as_ref(); - let schema: Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; - if let Value::Object(inner) = &schema { + let schema: serde_json::Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; + if let serde_json::Value::Object(inner) = &schema { let name = Name::parse(inner, &None)?; let previous_value = input_schemas.insert(name.clone(), schema); if previous_value.is_some() { @@ -1190,12 +1193,12 @@ mod schema { ) -> AvroResult<(Schema, Vec)> { let schemata = schemata.into_iter(); let schemata_len = schemata.size_hint().0; - let mut input_schemas: HashMap = HashMap::with_capacity(schemata_len); + let mut input_schemas: HashMap = HashMap::with_capacity(schemata_len); let mut input_order: Vec = Vec::with_capacity(schemata_len); for json in schemata { let json = json.as_ref(); - let schema: Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; - if let Value::Object(inner) = &schema { + let schema: serde_json::Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; + if let serde_json::Value::Object(inner) = &schema { let name = Name::parse(inner, &None)?; if let Some(_previous) = input_schemas.insert(name.clone(), schema) { return Err(Details::NameCollision(name.fullname(None)).into()); @@ -1229,14 +1232,14 @@ mod schema { } /// Parses an Avro schema from JSON. - pub fn parse(value: &Value) -> AvroResult { + pub fn parse(value: &serde_json::Value) -> AvroResult { let mut parser = Parser::default(); parser.parse(value, &None) } /// Parses an Avro schema from JSON. /// Any `Schema::Ref`s must be known in the `names` map. - pub(crate) fn parse_with_names(value: &Value, names: Names) -> AvroResult { + pub(crate) fn parse_with_names(value: &serde_json::Value, names: Names) -> AvroResult { let mut parser = Parser { input_schemas: HashMap::with_capacity(1), resolving_schemas: Names::default(), @@ -1247,7 +1250,7 @@ mod schema { } /// Returns the custom attributes (metadata) if the schema supports them. - pub fn custom_attributes(&self) -> Option<&BTreeMap> { + pub fn custom_attributes(&self) -> Option<&BTreeMap> { match self { Schema::Record(RecordSchema { attributes, .. }) | Schema::Enum(EnumSchema { attributes, .. }) @@ -1303,7 +1306,7 @@ mod schema { } /// Returns a Schema::Map with the given types and custom attributes. - pub fn map_with_attributes(types: Schema, attributes: BTreeMap) -> Self { + pub fn map_with_attributes(types: Schema, attributes: BTreeMap) -> Self { Schema::Map(MapSchema { types: Box::new(types), attributes, @@ -1319,7 +1322,7 @@ mod schema { } /// Returns a Schema::Array with the given items and custom attributes. - pub fn array_with_attributes(items: Schema, attributes: BTreeMap) -> Self { + pub fn array_with_attributes(items: Schema, attributes: BTreeMap) -> Self { Schema::Array(ArraySchema { items: Box::new(items), attributes, @@ -1407,13 +1410,13 @@ mod schema { /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro /// schema. - fn parse(&mut self, value: &Value, enclosing_namespace: &Namespace) -> AvroResult { + fn parse(&mut self, value: &serde_json::Value, enclosing_namespace: &Namespace) -> AvroResult { match *value { - Value::String(ref t) => self.parse_known_schema(t.as_str(), enclosing_namespace), - Value::Object(ref data) => { + serde_json::Value::String(ref t) => self.parse_known_schema(t.as_str(), enclosing_namespace), + serde_json::Value::Object(ref data) => { self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) } - Value::Array(ref data) => self.parse_union(data, enclosing_namespace), + serde_json::Value::Array(ref data) => self.parse_union(data, enclosing_namespace), _ => Err(Details::ParseSchemaFromValidJson.into()), } } @@ -1803,7 +1806,7 @@ mod schema { enclosing_namespace: &Namespace, ) -> Option<&Schema> { match complex.get("type") { - Some(Value::String(typ)) => { + Some(serde_json::Value::String(typ)) => { let name = Name::new(typ.as_str()) .unwrap() .fully_qualified_name(enclosing_namespace); @@ -1882,7 +1885,7 @@ mod schema { &self, complex: &serde_json::Map, excluded: Vec<&'static str>, - ) -> BTreeMap { + ) -> BTreeMap { let mut custom_attributes: BTreeMap = BTreeMap::new(); for (key, value) in complex { match key.as_str() { @@ -1937,7 +1940,7 @@ mod schema { let mut default: Option = None; if let Some(value) = complex.get("default") { - if let Value::String(ref s) = *value { + if let serde_json::Value::String(ref s) = *value { default = Some(s.clone()); } else { return Err(Details::EnumDefaultWrongType(value.clone()).into()); @@ -2013,7 +2016,7 @@ mod schema { /// `Schema`. fn parse_union( &mut self, - items: &[Value], + items: &[serde_json::Value], enclosing_namespace: &Namespace, ) -> AvroResult { items @@ -2053,7 +2056,7 @@ mod schema { } let doc = complex.get("doc").and_then(|v| match &v { - &Value::String(docstr) => Some(docstr.clone()), + &serde_json::Value::String(docstr) => Some(docstr.clone()), _ => None, }); @@ -2065,7 +2068,7 @@ mod schema { }?; let default = complex.get("default").and_then(|v| match &v { - &Value::String(default) => Some(default.clone()), + &serde_json::Value::String(default) => Some(default.clone()), _ => None, }); @@ -2363,11 +2366,11 @@ mod schema { /// Parses a **valid** avro schema into the Parsing Canonical Form. /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - fn parsing_canonical_form(schema: &Value, defined_names: &mut HashSet) -> String { + fn parsing_canonical_form(schema: &serde_json::Value, defined_names: &mut HashSet) -> String { match schema { - Value::Object(map) => pcf_map(map, defined_names), - Value::String(s) => pcf_string(s), - Value::Array(v) => pcf_array(v, defined_names), + serde_json::Value::Object(map) => pcf_map(map, defined_names), + serde_json::Value::String(s) => pcf_string(s), + serde_json::Value::Array(v) => pcf_array(v, defined_names), json => panic!("got invalid JSON value for canonical form of schema: {json}"), } } @@ -2401,7 +2404,7 @@ mod schema { // Reduce primitive types to their simple form. ([PRIMITIVE] rule) if schema.len() == 1 && k == "type" { // Invariant: function is only callable from a valid schema, so this is acceptable. - if let Value::String(s) = v { + if let serde_json::Value::String(s) = v { return pcf_string(s); } } @@ -2462,7 +2465,7 @@ mod schema { ) } - fn pcf_array(arr: &[Value], defined_names: &mut HashSet) -> String { + fn pcf_array(arr: &[serde_json::Value], defined_names: &mut HashSet) -> String { let inter = arr .iter() .map(|a| parsing_canonical_form(a, defined_names)) @@ -3124,7 +3127,7 @@ mod schema { RecordField { name: "a".to_string(), doc: None, - default: Some(Value::Number(42i64.into())), + default: Some(serde_json::Value::Number(42i64.into())), aliases: None, schema: Schema::Long, order: RecordFieldOrder::Ascending, diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 499d5a9e..4cd1b23a 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -42,7 +42,7 @@ mod ser { use crate::{ - Error, + error::tokio::Error, bytes::{BytesType, SER_BYTES_TYPE}, types::tokio::Value, }; diff --git a/avro/src/types.rs b/avro/src/types.rs index 949da90a..5012c9b5 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -41,8 +41,12 @@ )] mod types { + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; use crate::{ - AvroResult, Error, + error::tokio::Error, bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, decimal::tokio::Decimal, duration::Duration, diff --git a/avro/src/util.rs b/avro/src/util.rs index 2de5ff8d..121e6d71 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -100,6 +100,9 @@ mod util { #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] use crate::AvroResult; use crate::error::tokio::Details; use crate::schema::tokio::Documentation; @@ -211,7 +214,7 @@ mod util { Ok(i) } - pub fn safe_len(len: usize) -> crate::AvroResult { + pub fn safe_len(len: usize) -> AvroResult { let max_bytes = crate::util::max_allocation_bytes(crate::util::DEFAULT_MAX_ALLOCATION_BYTES); if len <= max_bytes { diff --git a/avro/src/validator.rs b/avro/src/validator.rs index 0cd37001..553e3cdf 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -39,7 +39,11 @@ )] mod validator { - use crate::{AvroResult, error::tokio::Details, schema::tokio::Namespace}; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; + use crate::{error::tokio::Details, schema::tokio::Namespace}; use log::debug; use regex_lite::Regex; use std::sync::OnceLock; diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 4e9b98c3..b4ed97ef 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -25,6 +25,7 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::codec::tokio => crate::codec::sync, crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, @@ -41,8 +42,12 @@ )] mod writer { + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; use crate::{ - AvroResult, Codec, Error, + codec::tokio::Codec, error::tokio::Error, encode::tokio::{encode, encode_internal, encode_to_vec}, error::tokio::Details, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, From fa8d54baa82da4ab75cf6dbe2857354f9a5e6cb1 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 5 Aug 2025 10:42:35 +0300 Subject: [PATCH 08/47] Fix replacements for all modules Now `clear && cargo expand --no-default-features --features xyz 2>&1 > generated_xyz.rs` works for both xyz=sync and xyz=tokio in non-test code Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 2 +- avro/src/lib.rs | 2 +- avro/src/reader.rs | 2 ++ avro/src/writer.rs | 2 ++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index b3abedf4..36d65e87 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -49,7 +49,7 @@ mod bigdecimal { use crate::AsyncAvroResult as AvroResult; #[synca::cfg(sync)] use crate::AvroResult; - + pub(crate) fn big_decimal_as_bytes(decimal: &BigDecimal) -> AvroResult> { let mut buffer: Vec = Vec::new(); let (big_int, exponent): (BigInt, i64) = decimal.as_bigint_and_exponent(); diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 4c0a6b64..5108b60e 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -889,7 +889,7 @@ pub use crate::bytes::{ #[cfg(feature = "tokio")] pub use crate::bigdecimal::tokio::BigDecimal as AsyncBigDecimal; #[cfg(feature = "sync")] -pub use crate::bigdecimal::tokio::BigDecimal; +pub use crate::bigdecimal::sync::BigDecimal; #[cfg(feature = "bzip")] pub use codec::bzip::Bzip2Settings; #[cfg(feature = "xz")] diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 19bdc213..e237303e 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -23,10 +23,12 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::codec::tokio => crate::codec::sync, crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, + crate::headers::tokio => crate::headers::sync, crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, diff --git a/avro/src/writer.rs b/avro/src/writer.rs index b4ed97ef..cd6b735c 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -30,7 +30,9 @@ crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, + crate::headers::tokio => crate::headers::sync, crate::schema::tokio => crate::schema::sync, + crate::ser_schema::tokio => crate::ser_schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, crate::schema_equality::tokio => crate::schema_equality::sync, From 2ecbb64887e52ebbf182b7824c8a2dce388454c8 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 5 Aug 2025 15:59:23 +0300 Subject: [PATCH 09/47] WIP: Successful build for `clear && cargo build --no-default-features --features tokio` The `sync` feature build fails due to the new `Box::pin(...)` wrappers for all recursive calls Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bytes.rs | 2 +- avro/src/codec.rs | 6 +- avro/src/de.rs | 6 +- avro/src/decimal.rs | 6 +- avro/src/decode.rs | 11 +- avro/src/encode.rs | 8 +- avro/src/headers.rs | 9 +- avro/src/lib.rs | 55 ++++---- avro/src/reader.rs | 77 +++++------ avro/src/schema.rs | 299 +++++++++++++++++++++++----------------- avro/src/ser.rs | 2 +- avro/src/types.rs | 154 +++++++++++---------- avro/src/util.rs | 12 +- avro/src/writer.rs | 44 ++++-- avro/tests/avro-3786.rs | 2 +- avro/tests/avro-3787.rs | 2 +- avro/tests/schema.rs | 2 +- 17 files changed, 384 insertions(+), 313 deletions(-) diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 0c084de9..5da2844a 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -288,9 +288,9 @@ pub mod serde_avro_slice_opt { #[cfg(test)] mod tests { use super::*; + use crate::de::tokio::from_value; use crate::schema::tokio::Schema; use crate::ser::tokio::to_value; - use crate::de::tokio::from_value; use crate::types::tokio::Value; use serde::{Deserialize, Serialize}; diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 323dda13..d346b715 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -37,13 +37,13 @@ } )] mod codec { - use crate::{error::tokio::Details, error::tokio::Error, types::tokio::Value}; - use strum_macros::{EnumIter, EnumString, IntoStaticStr}; #[synca::cfg(tokio)] use crate::AsyncAvroResult as AvroResult; #[synca::cfg(sync)] use crate::AvroResult; - + use crate::{error::tokio::Details, error::tokio::Error, types::tokio::Value}; + use strum_macros::{EnumIter, EnumString, IntoStaticStr}; + /// Settings for the `Deflate` codec. #[derive(Clone, Copy, Eq, PartialEq, Debug)] pub struct DeflateSettings { diff --git a/avro/src/de.rs b/avro/src/de.rs index 5d80f5e4..d5f98827 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -39,8 +39,8 @@ mod de { use crate::{ - error::tokio::Error, bytes::DE_BYTES_BORROWED, error::tokio::Details, schema::tokio::SchemaKind, - types::tokio::Value, + bytes::DE_BYTES_BORROWED, error::tokio::Details, error::tokio::Error, + schema::tokio::SchemaKind, types::tokio::Value, }; use serde::{ Deserialize, @@ -796,13 +796,13 @@ mod de { #[cfg(test)] mod tests { + use crate::ser::tokio::to_value; use num_bigint::BigInt; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; use serial_test::serial; use std::sync::atomic::Ordering; use uuid::Uuid; - use crate::ser::tokio::to_value; use apache_avro_test_helper::TestResult; diff --git a/avro/src/decimal.rs b/avro/src/decimal.rs index 6307fc8b..94f699ca 100644 --- a/avro/src/decimal.rs +++ b/avro/src/decimal.rs @@ -36,13 +36,13 @@ )] mod decimal { - use crate::{error::tokio::Details, error::tokio::Error}; - use num_bigint::{BigInt, Sign}; - use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; #[synca::cfg(tokio)] use crate::AsyncAvroResult as AvroResult; #[synca::cfg(sync)] use crate::AvroResult; + use crate::{error::tokio::Details, error::tokio::Error}; + use num_bigint::{BigInt, Sign}; + use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; #[derive(Debug, Clone, Eq)] pub struct Decimal { diff --git a/avro/src/decode.rs b/avro/src/decode.rs index cc8db8fe..7b3f54d6 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -42,14 +42,18 @@ mod decode { #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; use crate::util::tokio::{safe_len, zag_i32, zag_i64}; use crate::{ - error::tokio::Error, bigdecimal::tokio::deserialize_big_decimal, decimal::tokio::Decimal, duration::Duration, encode::tokio::encode_long, error::tokio::Details, + error::tokio::Error, schema::tokio::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, @@ -58,11 +62,6 @@ mod decode { }; use std::{borrow::Borrow, collections::HashMap, io::ErrorKind, str::FromStr}; use uuid::Uuid; - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; - #[inline] pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 1cf8f7c5..35bc7ed2 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -38,6 +38,10 @@ mod encode { use crate::util::tokio::{zig_i32, zig_i64}; + #[synca::cfg(tokio)] + use crate::AsyncAvroResult as AvroResult; + #[synca::cfg(sync)] + use crate::AvroResult; use crate::{ bigdecimal::tokio::serialize_big_decimal, error::tokio::Details, @@ -47,10 +51,6 @@ mod encode { }, types::tokio::{Value, ValueKind}, }; - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; use log::error; use std::{borrow::Borrow, collections::HashMap, io::Write}; diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 9b72f281..8866a66e 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -39,15 +39,14 @@ mod headers { use uuid::Uuid; - use crate::{ - error::tokio::Details, rabin::Rabin, schema::tokio::Schema, - schema::tokio::SchemaFingerprint, - }; #[synca::cfg(tokio)] use crate::AsyncAvroResult as AvroResult; #[synca::cfg(sync)] use crate::AvroResult; - + use crate::{ + error::tokio::Details, rabin::Rabin, schema::tokio::Schema, + schema::tokio::SchemaFingerprint, + }; /// This trait represents that an object is able to construct an Avro message header. It is /// implemented for some known header types already. If you need a header type that is not already diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 5108b60e..ea0b51e2 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -882,74 +882,73 @@ pub mod schema_equality; pub mod types; pub mod validator; +#[cfg(feature = "sync")] +pub use crate::bigdecimal::sync::BigDecimal; +#[cfg(feature = "tokio")] +pub use crate::bigdecimal::tokio::BigDecimal as AsyncBigDecimal; pub use crate::bytes::{ serde_avro_bytes, serde_avro_bytes_opt, serde_avro_fixed, serde_avro_fixed_opt, serde_avro_slice, serde_avro_slice_opt, }; -#[cfg(feature = "tokio")] -pub use crate::bigdecimal::tokio::BigDecimal as AsyncBigDecimal; -#[cfg(feature = "sync")] -pub use crate::bigdecimal::sync::BigDecimal; #[cfg(feature = "bzip")] pub use codec::bzip::Bzip2Settings; +#[cfg(feature = "sync")] +pub use codec::sync::{Codec, DeflateSettings}; +#[cfg(feature = "tokio")] +pub use codec::tokio::{Codec as AsyncCodec, DeflateSettings as AsyncDeflateSettings}; #[cfg(feature = "xz")] pub use codec::xz::XzSettings; #[cfg(feature = "zstandard")] pub use codec::zstandard::ZstandardSettings; -#[cfg(feature = "tokio")] -pub use codec::tokio::{Codec as AsyncCodec, DeflateSettings as AsyncDeflateSettings}; #[cfg(feature = "sync")] -pub use codec::sync::{Codec, DeflateSettings}; +pub use de::sync::from_value; #[cfg(feature = "tokio")] pub use de::tokio::from_value as async_from_value; #[cfg(feature = "sync")] -pub use de::sync::from_value; +pub use decimal::sync::Decimal; #[cfg(feature = "tokio")] pub use decimal::tokio::Decimal as AsyncDecimal; -#[cfg(feature = "sync")] -pub use decimal::sync::Decimal; pub use duration::{Days, Duration, Millis, Months}; +#[cfg(feature = "sync")] +pub use error::sync::Error; #[cfg(feature = "tokio")] pub use error::tokio::Error as AsyncError; #[cfg(feature = "sync")] -pub use error::sync::Error; +pub use reader::sync::{ + GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, + from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, +}; #[cfg(feature = "tokio")] pub use reader::tokio::{ GenericSingleObjectReader as AsyncGenericSingleObjectReader, Reader as AsyncReader, - SpecificSingleObjectReader as AsyncSpecificSingleObjectReader, from_avro_datum as async_from_avro_datum, + SpecificSingleObjectReader as AsyncSpecificSingleObjectReader, + from_avro_datum as async_from_avro_datum, from_avro_datum_reader_schemata as async_from_avro_datum_reader_schemata, from_avro_datum_schemata as async_from_avro_datum_schemata, read_marker as async_read_marker, }; #[cfg(feature = "sync")] -pub use reader::sync::{ - GenericSingleObjectReader, Reader, - SpecificSingleObjectReader, from_avro_datum, - from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, -}; +pub use schema::sync::{AvroSchema, Schema}; #[cfg(feature = "tokio")] pub use schema::tokio::{AvroSchema as AsyncAvroSchema, Schema as AsyncSchema}; #[cfg(feature = "sync")] -pub use schema::sync::{AvroSchema, Schema}; +pub use ser::sync::to_value; #[cfg(feature = "tokio")] pub use ser::tokio::to_value as async_to_value; -#[cfg(feature = "sync")] -pub use ser::sync::to_value; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; +#[cfg(feature = "sync")] +pub use writer::sync::{ + GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, + to_avro_datum_schemata, write_avro_datum_ref, +}; #[cfg(feature = "tokio")] pub use writer::tokio::{ GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, - SpecificSingleObjectWriter as AsyncSpecificSingleObjectWriter, - Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, - to_avro_datum as async_to_avro_datum, + SpecificSingleObjectWriter as AsyncSpecificSingleObjectWriter, Writer as AsyncWriter, + WriterBuilder as AsyncWriterBuilder, to_avro_datum as async_to_avro_datum, to_avro_datum_schemata as async_to_avro_datum_schemata, write_avro_datum_ref as async_write_avro_datum_ref, }; -#[cfg(feature = "sync")] -pub use writer::sync::{ - GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, - to_avro_datum_schemata, write_avro_datum_ref, -}; #[cfg(feature = "derive")] pub use apache_avro_derive::*; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index e237303e..24286479 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -37,26 +37,28 @@ } )] mod reader { - #[synca::cfg(sync)] - use std::io::Read as AvroRead; - #[synca::cfg(tokio)] - use tokio::io::AsyncRead as AvroRead; - #[cfg(feature = "tokio")] - use tokio::io::AsyncReadExt; - use std::str::FromStr; - use crate::decode::tokio::{decode, decode_internal}; #[synca::cfg(tokio)] use crate::AsyncAvroResult as AvroResult; #[synca::cfg(sync)] use crate::AvroResult; #[synca::cfg(tokio)] use crate::async_from_value as from_value; + use crate::decode::tokio::{decode, decode_internal}; #[synca::cfg(sync)] use crate::from_value; + #[synca::cfg(sync)] + use std::io::Read as AvroRead; + use std::str::FromStr; + #[synca::cfg(tokio)] + use tokio::io::AsyncRead as AvroRead; + #[cfg(feature = "tokio")] + use tokio::io::AsyncReadExt; + use crate::util::tokio::safe_len; use crate::{ - codec::tokio::Codec, error::tokio::Error, + codec::tokio::Codec, error::tokio::Details, + error::tokio::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, schema::tokio::{ AvroSchema, Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, @@ -65,17 +67,10 @@ mod reader { types::tokio::Value, util::tokio::read_long, }; - #[cfg(feature = "tokio")] - use futures::Stream; use log::warn; use serde::de::DeserializeOwned; use serde_json::from_slice; - #[cfg(feature = "tokio")] - use std::pin::Pin; - #[cfg(feature = "tokio")] - use std::task::Poll; use std::{collections::HashMap, io::ErrorKind, marker::PhantomData}; - use crate::util::tokio::safe_len; /// Internal Block reader. @@ -130,7 +125,7 @@ mod reader { let meta_schema = Schema::map(Schema::Bytes); match decode(&meta_schema, &mut self.reader).await? { Value::Map(metadata) => { - self.read_writer_schema(&metadata)?; + self.read_writer_schema(&metadata).await?; self.codec = read_codec(&metadata)?; for (key, value) in metadata { @@ -261,7 +256,7 @@ mod reader { Ok(Some(item)) } - fn read_writer_schema(&mut self, metadata: &HashMap) -> AvroResult<()> { + async fn read_writer_schema(&mut self, metadata: &HashMap) -> AvroResult<()> { let json: serde_json::Value = metadata .get("avro.schema") .and_then(|bytes| { @@ -279,10 +274,10 @@ mod reader { .iter() .map(|(name, schema)| (name.clone(), (*schema).clone())) .collect(); - self.writer_schema = Schema::parse_with_names(&json, names)?; + self.writer_schema = Schema::parse_with_names(&json, names).await?; resolve_names_with_schemata(&self.schemata, &mut self.names_refs, &None)?; } else { - self.writer_schema = Schema::parse(&json)?; + self.writer_schema = Schema::parse(&json).await?; resolve_names(&self.writer_schema, &mut self.names_refs, &None)?; } Ok(()) @@ -467,28 +462,26 @@ mod reader { } } - #[cfg(feature = "tokio")] - impl Stream for Reader<'_, R> { - type Item = AvroResult; - fn poll_next( - mut self: Pin<&mut Self>, - _cx: &mut futures::task::Context<'_>, - ) -> Poll> { - // to prevent keep on reading after the first error occurs - if self.errored { - return Poll::Ready(None); - }; - async { - match self.read_next().await { - Ok(opt) => Poll::Ready(opt.map(Ok)), - Err(e) => { - self.errored = true; - Poll::Ready(Some(Err(e))) - } - } - } - } - } + // #[cfg(feature = "tokio")] + // impl Stream for Reader<'_, R> { + // type Item = AvroResult; + // fn poll_next( + // mut self: Pin<&mut Self>, + // _cx: &mut futures::task::Context<'_>, + // ) -> Poll> { + // // to prevent keep on reading after the first error occurs + // if self.errored { + // return Poll::Ready(None); + // }; + // match self.read_next().await { + // Ok(opt) => Poll::Ready(opt.map(Ok)), + // Err(e) => { + // self.errored = true; + // Poll::Ready(Some(Err(e))) + // } + // } + // } + // } #[cfg(feature = "sync")] impl Iterator for Reader<'_, R> { diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 3853cfb6..b43410e7 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -382,9 +382,9 @@ mod schema { where D: serde::de::Deserializer<'de>, { - Value::deserialize(deserializer).and_then(|value| { + serde_json::Value::deserialize(deserializer).and_then(|value| { use serde::de::Error; - if let Value::Object(json) = value { + if let serde_json::Value::Object(json) = value { Name::parse(&json, &None).map_err(Error::custom) } else { Err(Error::custom(format!("Expected a JSON object: {value:?}"))) @@ -711,11 +711,13 @@ mod schema { validate_record_field_name(&name)?; // TODO: "type" = "" - let schema = parser.parse_complex( - field, - &enclosing_record.namespace, - RecordSchemaParseLocation::FromField, - )?; + let schema = parser + .parse_complex( + field, + &enclosing_record.namespace, + RecordSchemaParseLocation::FromField, + ) + .await?; let default = field.get("default").cloned(); Self::resolve_default_value( @@ -1133,9 +1135,9 @@ mod schema { } /// Create a `Schema` from a string representing a JSON Avro schema. - pub fn parse_str(input: &str) -> Result { + pub async fn parse_str(input: &str) -> Result { let mut parser = Parser::default(); - parser.parse_str(input) + parser.parse_str(input).await } /// Create an array of `Schema`'s from a list of named JSON Avro schemas (Record, Enum, and @@ -1145,16 +1147,18 @@ mod schema { /// during parsing. /// /// If two of the input schemas have the same fullname, an Error will be returned. - pub fn parse_list( + pub async fn parse_list( input: impl IntoIterator>, ) -> AvroResult> { let input = input.into_iter(); let input_len = input.size_hint().0; - let mut input_schemas: HashMap = HashMap::with_capacity(input_len); + let mut input_schemas: HashMap = + HashMap::with_capacity(input_len); let mut input_order: Vec = Vec::with_capacity(input_len); for json in input { let json = json.as_ref(); - let schema: serde_json::Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; + let schema: serde_json::Value = + serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; if let serde_json::Value::Object(inner) = &schema { let name = Name::parse(inner, &None)?; let previous_value = input_schemas.insert(name.clone(), schema); @@ -1172,7 +1176,7 @@ mod schema { input_order, parsed_schemas: HashMap::with_capacity(input_len), }; - parser.parse_list() + parser.parse_list().await } /// Create a `Schema` from a string representing a JSON Avro schema, @@ -1187,17 +1191,19 @@ mod schema { /// # Arguments /// * `schema` - the JSON string of the schema to parse /// * `schemata` - a slice of additional schemas that is used to resolve cross-references - pub fn parse_str_with_list( + pub async fn parse_str_with_list( schema: &str, schemata: impl IntoIterator>, ) -> AvroResult<(Schema, Vec)> { let schemata = schemata.into_iter(); let schemata_len = schemata.size_hint().0; - let mut input_schemas: HashMap = HashMap::with_capacity(schemata_len); + let mut input_schemas: HashMap = + HashMap::with_capacity(schemata_len); let mut input_order: Vec = Vec::with_capacity(schemata_len); for json in schemata { let json = json.as_ref(); - let schema: serde_json::Value = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; + let schema: serde_json::Value = + serde_json::from_str(json).map_err(Details::ParseSchemaJson)?; if let serde_json::Value::Object(inner) = &schema { let name = Name::parse(inner, &None)?; if let Some(_previous) = input_schemas.insert(name.clone(), schema) { @@ -1214,39 +1220,42 @@ mod schema { input_order, parsed_schemas: HashMap::with_capacity(schemata_len), }; - parser.parse_input_schemas()?; + parser.parse_input_schemas().await?; let value = serde_json::from_str(schema).map_err(Details::ParseSchemaJson)?; - let schema = parser.parse(&value, &None)?; - let schemata = parser.parse_list()?; + let schema = parser.parse(&value, &None).await?; + let schemata = parser.parse_list().await?; Ok((schema, schemata)) } /// Create a `Schema` from a reader which implements [`Read`]. - pub fn parse_reader(reader: &mut (impl Read + ?Sized)) -> AvroResult { + pub async fn parse_reader(reader: &mut (impl Read + ?Sized)) -> AvroResult { let mut buf = String::new(); match reader.read_to_string(&mut buf) { - Ok(_) => Self::parse_str(&buf), + Ok(_) => Self::parse_str(&buf).await, Err(e) => Err(Details::ReadSchemaFromReader(e).into()), } } /// Parses an Avro schema from JSON. - pub fn parse(value: &serde_json::Value) -> AvroResult { + pub async fn parse(value: &serde_json::Value) -> AvroResult { let mut parser = Parser::default(); - parser.parse(value, &None) + parser.parse(value, &None).await } /// Parses an Avro schema from JSON. /// Any `Schema::Ref`s must be known in the `names` map. - pub(crate) fn parse_with_names(value: &serde_json::Value, names: Names) -> AvroResult { + pub(crate) async fn parse_with_names( + value: &serde_json::Value, + names: Names, + ) -> AvroResult { let mut parser = Parser { input_schemas: HashMap::with_capacity(1), resolving_schemas: Names::default(), input_order: Vec::with_capacity(1), parsed_schemas: names, }; - parser.parse(value, &None) + parser.parse(value, &None).await } /// Returns the custom attributes (metadata) if the schema supports them. @@ -1306,7 +1315,10 @@ mod schema { } /// Returns a Schema::Map with the given types and custom attributes. - pub fn map_with_attributes(types: Schema, attributes: BTreeMap) -> Self { + pub fn map_with_attributes( + types: Schema, + attributes: BTreeMap, + ) -> Self { Schema::Map(MapSchema { types: Box::new(types), attributes, @@ -1322,7 +1334,10 @@ mod schema { } /// Returns a Schema::Array with the given items and custom attributes. - pub fn array_with_attributes(items: Schema, attributes: BTreeMap) -> Self { + pub fn array_with_attributes( + items: Schema, + attributes: BTreeMap, + ) -> Self { Schema::Array(ArraySchema { items: Box::new(items), attributes, @@ -1367,15 +1382,15 @@ mod schema { impl Parser { /// Create a `Schema` from a string representing a JSON Avro schema. - fn parse_str(&mut self, input: &str) -> Result { + async fn parse_str(&mut self, input: &str) -> Result { let value = serde_json::from_str(input).map_err(Details::ParseSchemaJson)?; - self.parse(&value, &None) + self.parse(&value, &None).await } /// Create an array of `Schema`'s from an iterator of JSON Avro schemas. It is allowed that /// the schemas have cross-dependencies; these will be resolved during parsing. - fn parse_list(&mut self) -> Result, Error> { - self.parse_input_schemas()?; + async fn parse_list(&mut self) -> Result, Error> { + self.parse_input_schemas().await?; let mut parsed_schemas = Vec::with_capacity(self.parsed_schemas.len()); for name in self.input_order.drain(0..) { @@ -1389,7 +1404,7 @@ mod schema { } /// Convert the input schemas to parsed_schemas - fn parse_input_schemas(&mut self) -> Result<(), Error> { + async fn parse_input_schemas(&mut self) -> Result<(), Error> { while !self.input_schemas.is_empty() { let next_name = self .input_schemas @@ -1401,7 +1416,7 @@ mod schema { .input_schemas .remove_entry(&next_name) .expect("Key unexpectedly missing"); - let parsed = self.parse(&value, &None)?; + let parsed = self.parse(&value, &None).await?; self.parsed_schemas .insert(get_schema_type_name(name, value), parsed); } @@ -1410,13 +1425,26 @@ mod schema { /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro /// schema. - fn parse(&mut self, value: &serde_json::Value, enclosing_namespace: &Namespace) -> AvroResult { + async fn parse( + &mut self, + value: &serde_json::Value, + enclosing_namespace: &Namespace, + ) -> AvroResult { match *value { - serde_json::Value::String(ref t) => self.parse_known_schema(t.as_str(), enclosing_namespace), + serde_json::Value::String(ref t) => { + Box::pin(self.parse_known_schema(t.as_str(), enclosing_namespace)).await + } serde_json::Value::Object(ref data) => { - self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) + Box::pin(self.parse_complex( + data, + enclosing_namespace, + RecordSchemaParseLocation::Root, + )) + .await + } + serde_json::Value::Array(ref data) => { + Box::pin(self.parse_union(data, enclosing_namespace)).await } - serde_json::Value::Array(ref data) => self.parse_union(data, enclosing_namespace), _ => Err(Details::ParseSchemaFromValidJson.into()), } } @@ -1424,7 +1452,7 @@ mod schema { /// Parse a `serde_json::Value` representing an Avro type whose Schema is known into a /// `Schema`. A Schema for a `serde_json::Value` is known if it is primitive or has /// been parsed previously by the parsed and stored in its map of parsed_schemas. - fn parse_known_schema( + async fn parse_known_schema( &mut self, name: &str, enclosing_namespace: &Namespace, @@ -1438,7 +1466,7 @@ mod schema { "float" => Ok(Schema::Float), "bytes" => Ok(Schema::Bytes), "string" => Ok(Schema::String), - _ => self.fetch_schema_ref(name, enclosing_namespace), + _ => Box::pin(self.fetch_schema_ref(name, enclosing_namespace)).await, } } @@ -1451,7 +1479,7 @@ mod schema { /// /// This method allows schemas definitions that depend on other types to /// parse their dependencies (or look them up if already parsed). - fn fetch_schema_ref( + async fn fetch_schema_ref( &mut self, name: &str, enclosing_namespace: &Namespace, @@ -1494,7 +1522,7 @@ mod schema { .ok_or_else(|| Details::ParsePrimitive(fully_qualified_name.fullname(None)))?; // parsing a full schema from inside another schema. Other full schema will not inherit namespace - let parsed = self.parse(&value, &None)?; + let parsed = Box::pin(self.parse(&value, &None)).await?; self.parsed_schemas .insert(get_schema_type_name(name, value), parsed.clone()); @@ -1543,14 +1571,14 @@ mod schema { /// /// Avro supports "recursive" definition of types. /// e.g: {"type": {"type": "string"}} - fn parse_complex( + async fn parse_complex( &mut self, complex: &serde_json::Map, enclosing_namespace: &Namespace, parse_location: RecordSchemaParseLocation, ) -> AvroResult { // Try to parse this as a native complex type. - fn parse_as_native_complex( + async fn parse_as_native_complex( complex: &serde_json::Map, parser: &mut Parser, enclosing_namespace: &Namespace, @@ -1560,7 +1588,7 @@ mod schema { serde_json::Value::String(s) if s == "fixed" => { parser.parse_fixed(complex, enclosing_namespace) } - _ => parser.parse(value, enclosing_namespace), + _ => Box::pin(parser.parse(value, enclosing_namespace)).await, }, None => Err(Details::GetLogicalTypeField.into()), } @@ -1597,7 +1625,8 @@ mod schema { "decimal" => { return try_convert_to_logical_type( "decimal", - parse_as_native_complex(complex, self, enclosing_namespace)?, + Box::pin(parse_as_native_complex(complex, self, enclosing_namespace)) + .await?, &[SchemaKind::Fixed, SchemaKind::Bytes], |inner| -> AvroResult { match Self::parse_precision_and_scale(complex) { @@ -1617,7 +1646,7 @@ mod schema { "big-decimal" => { return try_convert_to_logical_type( "big-decimal", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Bytes], |_| -> AvroResult { Ok(Schema::BigDecimal) }, ); @@ -1625,7 +1654,7 @@ mod schema { "uuid" => { return try_convert_to_logical_type( "uuid", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::String, SchemaKind::Fixed], |schema| match schema { Schema::String => Ok(Schema::Uuid), @@ -1648,7 +1677,7 @@ mod schema { "date" => { return try_convert_to_logical_type( "date", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Int], |_| -> AvroResult { Ok(Schema::Date) }, ); @@ -1656,7 +1685,7 @@ mod schema { "time-millis" => { return try_convert_to_logical_type( "date", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Int], |_| -> AvroResult { Ok(Schema::TimeMillis) }, ); @@ -1664,7 +1693,7 @@ mod schema { "time-micros" => { return try_convert_to_logical_type( "time-micros", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::TimeMicros) }, ); @@ -1672,7 +1701,7 @@ mod schema { "timestamp-millis" => { return try_convert_to_logical_type( "timestamp-millis", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::TimestampMillis) }, ); @@ -1680,7 +1709,7 @@ mod schema { "timestamp-micros" => { return try_convert_to_logical_type( "timestamp-micros", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::TimestampMicros) }, ); @@ -1688,7 +1717,7 @@ mod schema { "timestamp-nanos" => { return try_convert_to_logical_type( "timestamp-nanos", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::TimestampNanos) }, ); @@ -1696,7 +1725,7 @@ mod schema { "local-timestamp-millis" => { return try_convert_to_logical_type( "local-timestamp-millis", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::LocalTimestampMillis) }, ); @@ -1704,7 +1733,7 @@ mod schema { "local-timestamp-micros" => { return try_convert_to_logical_type( "local-timestamp-micros", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::LocalTimestampMicros) }, ); @@ -1712,7 +1741,7 @@ mod schema { "local-timestamp-nanos" => { return try_convert_to_logical_type( "local-timestamp-nanos", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Long], |_| -> AvroResult { Ok(Schema::LocalTimestampNanos) }, ); @@ -1720,7 +1749,7 @@ mod schema { "duration" => { return try_convert_to_logical_type( "duration", - parse_as_native_complex(complex, self, enclosing_namespace)?, + parse_as_native_complex(complex, self, enclosing_namespace).await?, &[SchemaKind::Fixed], |_| -> AvroResult { Ok(Schema::Duration) }, ); @@ -1739,22 +1768,29 @@ mod schema { Some(serde_json::Value::String(t)) => match t.as_str() { "record" => match parse_location { RecordSchemaParseLocation::Root => { - self.parse_record(complex, enclosing_namespace) + Box::pin(self.parse_record(complex, enclosing_namespace)).await } RecordSchemaParseLocation::FromField => { - self.fetch_schema_ref(t, enclosing_namespace) + self.fetch_schema_ref(t, enclosing_namespace).await } }, "enum" => self.parse_enum(complex, enclosing_namespace), - "array" => self.parse_array(complex, enclosing_namespace), - "map" => self.parse_map(complex, enclosing_namespace), + "array" => self.parse_array(complex, enclosing_namespace).await, + "map" => self.parse_map(complex, enclosing_namespace).await, "fixed" => self.parse_fixed(complex, enclosing_namespace), - other => self.parse_known_schema(other, enclosing_namespace), + other => self.parse_known_schema(other, enclosing_namespace).await, }, Some(serde_json::Value::Object(data)) => { - self.parse_complex(data, enclosing_namespace, RecordSchemaParseLocation::Root) + Box::pin(self.parse_complex( + data, + enclosing_namespace, + RecordSchemaParseLocation::Root, + )) + .await + } + Some(serde_json::Value::Array(variants)) => { + self.parse_union(variants, enclosing_namespace).await } - Some(serde_json::Value::Array(variants)) => self.parse_union(variants, enclosing_namespace), Some(unknown) => Err(Details::GetComplexType(unknown.clone()).into()), None => Err(Details::GetComplexTypeField.into()), } @@ -1820,7 +1856,7 @@ mod schema { /// Parse a `serde_json::Value` representing a Avro record type into a /// `Schema`. - fn parse_record( + async fn parse_record( &mut self, complex: &serde_json::Map, enclosing_namespace: &Namespace, @@ -1842,21 +1878,23 @@ mod schema { debug!("Going to parse record schema: {:?}", &fully_qualified_name); - let fields: Vec = fields_opt - .and_then(|fields| fields.as_array()) - .ok_or_else(|| Error::new(Details::GetRecordFieldsJson)) - .and_then(|fields| { - fields - .iter() - .filter_map(|field| field.as_object()) - .enumerate() - .map(|(position, field)| { - RecordField::parse(field, position, self, &fully_qualified_name) - }) - .collect::>() - })?; + let fields: &Vec = + fields_opt + .and_then(|fields| fields.as_array()) + .ok_or_else(|| Error::new(Details::GetRecordFieldsJson))?; + + let mut record_fields: Vec = Vec::with_capacity(fields.len()); + let mut position = 0; + for field in fields.iter() { + if let Some(field) = field.as_object() { + let record_field = + Box::pin(RecordField::parse(field, position, self, &fully_qualified_name)).await?; + record_fields.push(record_field); + position += 1; + } + } - for field in &fields { + for field in &record_fields { if let Some(_old) = lookup.insert(field.name.clone(), field.position) { return Err(Details::FieldNameDuplicate(field.name.clone()).into()); } @@ -1872,7 +1910,7 @@ mod schema { name: fully_qualified_name.clone(), aliases: aliases.clone(), doc: complex.doc(), - fields, + fields: record_fields, lookup, attributes: self.get_custom_attributes(complex, vec!["fields"]), }); @@ -1976,69 +2014,66 @@ mod schema { /// Parse a `serde_json::Value` representing a Avro array type into a /// `Schema`. - fn parse_array( + async fn parse_array( &mut self, complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { - complex + let items = complex .get("items") - .ok_or_else(|| Details::GetArrayItemsField.into()) - .and_then(|items| self.parse(items, enclosing_namespace)) - .map(|items| { - Schema::array_with_attributes( - items, - self.get_custom_attributes(complex, vec!["items"]), - ) - }) + .ok_or_else(|| Error::new(Details::GetArrayItemsField))?; + let items = Box::pin(self.parse(items, enclosing_namespace)).await?; + Ok(Schema::array_with_attributes( + items, + self.get_custom_attributes(complex, vec!["items"]), + )) } /// Parse a `serde_json::Value` representing a Avro map type into a /// `Schema`. - fn parse_map( + async fn parse_map( &mut self, complex: &serde_json::Map, enclosing_namespace: &Namespace, ) -> AvroResult { - complex + let items = complex .get("values") - .ok_or_else(|| Details::GetMapValuesField.into()) - .and_then(|items| self.parse(items, enclosing_namespace)) - .map(|items| { - Schema::map_with_attributes( - items, - self.get_custom_attributes(complex, vec!["values"]), - ) - }) + .ok_or_else(|| Error::new(Details::GetMapValuesField))?; + + let items = self.parse(items, enclosing_namespace).await?; + Ok(Schema::map_with_attributes( + items, + self.get_custom_attributes(complex, vec!["values"]), + )) } /// Parse a `serde_json::Value` representing a Avro union type into a /// `Schema`. - fn parse_union( + async fn parse_union( &mut self, items: &[serde_json::Value], enclosing_namespace: &Namespace, ) -> AvroResult { - items - .iter() - .map(|v| self.parse(v, enclosing_namespace)) - .collect::, _>>() - .and_then(|schemas| { - if schemas.is_empty() { - error!( - "Union schemas should have at least two members! \ - Please enable debug logging to find out which Record schema \ - declares the union with 'RUST_LOG=apache_avro::schema=debug'." - ); - } else if schemas.len() == 1 { - warn!( - "Union schema with just one member! Consider dropping the union! \ - Please enable debug logging to find out which Record schema \ - declares the union with 'RUST_LOG=apache_avro::schema=debug'." - ); - } - Ok(Schema::Union(UnionSchema::new(schemas)?)) - }) + let mut schemas = Vec::with_capacity(items.len()); + for variant in items { + let schema = Box::pin(self.parse(variant, enclosing_namespace)).await?; + schemas.push(schema); + } + + if schemas.is_empty() { + error!( + "Union schemas should have at least two members! \ + Please enable debug logging to find out which Record schema \ + declares the union with 'RUST_LOG=apache_avro::schema=debug'." + ); + } else if schemas.len() == 1 { + warn!( + "Union schema with just one member! Consider dropping the union! \ + Please enable debug logging to find out which Record schema \ + declares the union with 'RUST_LOG=apache_avro::schema=debug'." + ); + } + Ok(Schema::Union(UnionSchema::new(schemas)?)) } /// Parse a `serde_json::Value` representing a Avro fixed type into a @@ -2366,7 +2401,10 @@ mod schema { /// Parses a **valid** avro schema into the Parsing Canonical Form. /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - fn parsing_canonical_form(schema: &serde_json::Value, defined_names: &mut HashSet) -> String { + fn parsing_canonical_form( + schema: &serde_json::Value, + defined_names: &mut HashSet, + ) -> String { match schema { serde_json::Value::Object(map) => pcf_map(map, defined_names), serde_json::Value::String(s) => pcf_string(s), @@ -2375,7 +2413,10 @@ mod schema { } } - fn pcf_map(schema: &serde_json::Map, defined_names: &mut HashSet) -> String { + fn pcf_map( + schema: &serde_json::Map, + defined_names: &mut HashSet, + ) -> String { // Look for the namespace variant up front. let ns = schema.get("namespace").and_then(|v| v.as_str()); let typ = schema.get("type").and_then(|v| v.as_str()); @@ -2700,7 +2741,6 @@ mod schema { #[cfg(test)] mod tests { use super::*; - use {SpecificSingleObjectWriter, error::tokio::Details, rabin::Rabin}; use apache_avro_test_helper::{ TestResult, logger::{assert_logged, assert_not_logged}, @@ -2708,6 +2748,7 @@ mod schema { use serde_json::json; use serial_test::serial; use std::sync::atomic::Ordering; + use {SpecificSingleObjectWriter, error::tokio::Details, rabin::Rabin}; #[test] fn test_invalid_schema() { @@ -2845,7 +2886,8 @@ mod schema { ] }"#; - let schema_c = Schema::parse_list([schema_str_a, schema_str_b, schema_str_c])? + let schema_c = Schema::parse_list([schema_str_a, schema_str_b, schema_str_c]) + .await? .last() .unwrap() .clone(); @@ -3027,7 +3069,7 @@ mod schema { "fields": [ {"name": "field_one", "type": "A"} ] }"#; - let list = Schema::parse_list([schema_str_a, schema_str_b])?; + let list = Schema::parse_list([schema_str_a, schema_str_b]).await?; let schema_a = list.first().unwrap().clone(); @@ -3067,7 +3109,8 @@ mod schema { ] }"#; - let schema_option_a = Schema::parse_list([schema_str_a, schema_str_option_a])? + let schema_option_a = Schema::parse_list([schema_str_a, schema_str_option_a]) + .await? .last() .unwrap() .clone(); @@ -3786,8 +3829,8 @@ mod schema { #[test] fn test_schema_fingerprint() -> TestResult { - use rabin::Rabin; use md5::Md5; + use rabin::Rabin; use sha2::Sha256; let raw_schema = r#" diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 4cd1b23a..73cb5e43 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -42,8 +42,8 @@ mod ser { use crate::{ - error::tokio::Error, bytes::{BytesType, SER_BYTES_TYPE}, + error::tokio::Error, types::tokio::Value, }; use serde::{Serialize, ser}; diff --git a/avro/src/types.rs b/avro/src/types.rs index 5012c9b5..4fe0c225 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -46,19 +46,17 @@ mod types { #[synca::cfg(sync)] use crate::AvroResult; use crate::{ - error::tokio::Error, bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, decimal::tokio::Decimal, duration::Duration, error::tokio::Details, + error::tokio::Error, schema::tokio::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, }, }; use bigdecimal::BigDecimal; - #[synca::cfg(tokio)] - use futures::FutureExt; use log::{debug, error}; use std::{ borrow::Borrow, @@ -311,7 +309,9 @@ mod types { Value::Long(n) } } - serde_json::Value::Number(ref n) if n.is_f64() => Value::Double(n.as_f64().unwrap()), + serde_json::Value::Number(ref n) if n.is_f64() => { + Value::Double(n.as_f64().unwrap()) + } serde_json::Value::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great serde_json::Value::String(s) => s.into(), serde_json::Value::Array(items) => { @@ -405,7 +405,10 @@ mod types { for schema in schemata { let enclosing_namespace = schema.namespace(); - match self.validate_internal(schema, rs.get_names(), &enclosing_namespace).await { + match self + .validate_internal(schema, rs.get_names(), &enclosing_namespace) + .await + { Some(reason) => { let log_message = format!( "Invalid value: {self:?} for schema: {schema:?}. Reason: {reason}" @@ -418,7 +421,7 @@ mod types { } None => return true, } - }; + } false } @@ -546,9 +549,11 @@ mod types { } else { Some(format!("No schema in the union at position '{i}'")) } - }, + } (v, Schema::Union(inner)) => { - match inner.find_schema_with_known_schemata(v, Some(names), enclosing_namespace).await + match inner + .find_schema_with_known_schemata(v, Some(names), enclosing_namespace) + .await { Some(_) => None, None => Some("Could not find matching type in union".to_string()), @@ -559,7 +564,8 @@ mod types { for item in items.iter() { acc = Value::accumulate( acc, - item.validate_internal(&inner.items, names, enclosing_namespace).await, + Box::pin(item.validate_internal(&inner.items, names, enclosing_namespace)) + .await, ); } acc @@ -575,7 +581,9 @@ mod types { for (_, value) in items.iter() { acc = Value::accumulate( acc, - value.validate_internal(&inner.types, names, enclosing_namespace).await, + Box::pin(value + .validate_internal(&inner.types, names, enclosing_namespace)) + .await, ); } acc @@ -615,39 +623,36 @@ mod types { let mut acc = None; for (field_name, record_field) in record_fields.iter() { - let record_namespace = if name.namespace.is_none() { - enclosing_namespace - } else { - &name.namespace - }; - acc = match lookup.get(field_name) { - Some(idx) => { - let field = &fields[*idx]; - Value::accumulate( - acc, - record_field.validate_internal( - &field.schema, - names, - record_namespace, - ).await - ) - } - None => Value::accumulate( + let record_namespace = if name.namespace.is_none() { + enclosing_namespace + } else { + &name.namespace + }; + acc = match lookup.get(field_name) { + Some(idx) => { + let field = &fields[*idx]; + Value::accumulate( acc, - Some(format!( - "There is no schema field for field '{field_name}'" - )), - ), - }; - } - acc + Box::pin(record_field + .validate_internal(&field.schema, names, record_namespace)) + .await, + ) + } + None => Value::accumulate( + acc, + Some(format!("There is no schema field for field '{field_name}'")), + ), + }; + } + acc } (Value::Map(items), Schema::Record(RecordSchema { fields, .. })) => { let mut acc = None; for field in fields.iter() { if let Some(item) = items.get(&field.name) { - let res = - item.validate_internal(&field.schema, names, enclosing_namespace).await; + let res = Box::pin(item + .validate_internal(&field.schema, names, enclosing_namespace)) + .await; acc = Value::accumulate(acc, res); } else if !field.is_nullable() { acc = Value::accumulate( @@ -658,7 +663,7 @@ mod types { )), ); } - }; + } acc } (v, s) => Some(format!( @@ -757,7 +762,8 @@ mod types { .await } Schema::Map(ref inner) => { - self.resolve_map(&inner.types, names, enclosing_namespace).await + self.resolve_map(&inner.types, names, enclosing_namespace) + .await } Schema::Record(RecordSchema { ref fields, .. }) => { self.resolve_record(fields, names, enclosing_namespace) @@ -1139,7 +1145,7 @@ mod types { #[synca::cfg(tokio)] let resolved = futures::future::try_join_all(resolved).await?; Ok(Value::Array(resolved)) - }, + } other => Err(Details::GetArray { expected: schema.into(), other, @@ -1156,18 +1162,26 @@ mod types { ) -> Result { match self { Value::Map(items) => { - let resolved = items - .into_iter() - .map(|(key, value)| { - value - .resolve_internal(schema, names, enclosing_namespace, &None) - .map(|value| (key, value)) - }) - .collect::>(); - #[synca::cfg(tokio)] - let resolved = futures::future::try_join_all(resolved).await?; + let mut resolved = HashMap::with_capacity(items.len()); + for (key, value) in items.into_iter() { + let v = Box::pin(value + .resolve_internal(schema, names, enclosing_namespace, &None)) + .await?; + resolved.insert(key.clone(), v); + } + + // let resolved = items + // .into_iter() + // .map(|(key, value)| { + // value + // .resolve_internal(schema, names, enclosing_namespace, &None) + // .map(|value| (key, value)) + // }) + // .collect::>(); + // #[synca::cfg(tokio)] + // let resolved = futures::future::try_join_all(resolved).await?; Ok(Value::Map(resolved)) - }, + } other => Err(Details::GetMap { expected: schema.into(), other, @@ -1194,7 +1208,8 @@ mod types { })), }?; - let new_fields = fields.iter().map(|field| { + let mut new_fields = Vec::with_capacity(fields.len()); + for field in fields.iter() { let value = match items.remove(&field.name) { Some(value) => value, None => match field.default { @@ -1216,15 +1231,12 @@ mod types { Schema::Null => Value::Union(0, Box::new(Value::Null)), _ => Value::Union( 0, - Box::new( - Value::from(value.clone()) - .resolve_internal( - first, - names, - enclosing_namespace, - &field.default, - ), - ), + Box::new(Box::pin(Value::from(value.clone()).resolve_internal( + first, + names, + enclosing_namespace, + &field.default, + )).await?), ), } } @@ -1236,13 +1248,11 @@ mod types { }, }; - value - .resolve_internal(&field.schema, names, enclosing_namespace, &field.default) - .map(|value| (field.name.clone(), value)) - }); + let v = Box::pin(value + .resolve_internal(&field.schema, names, enclosing_namespace, &field.default)).await?; + new_fields.push((field.name.clone(), v)); + } - #[synca::cfg(tokio)] - let new_fields = futures::future::try_join_all(new_fields).await?; Ok(Value::Record(new_fields)) } @@ -2061,7 +2071,10 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn json_from_avro() -> TestResult { - assert_eq!(serde_json::Value::try_from(Value::Null)?, serde_json::Value::Null); + assert_eq!( + serde_json::Value::try_from(Value::Null)?, + serde_json::Value::Null + ); assert_eq!( serde_json::Value::try_from(Value::Boolean(true))?, serde_json::Value::Bool(true) @@ -2107,7 +2120,10 @@ Field with name '"b"' is not a member of the map items"#, serde_json::Value::String("test_enum".into()) ); assert_eq!( - serde_json::Value::try_from(Value::Union(1, Box::new(Value::String("test_enum".into()))))?, + serde_json::Value::try_from(Value::Union( + 1, + Box::new(Value::String("test_enum".into())) + ))?, serde_json::Value::String("test_enum".into()) ); assert_eq!( diff --git a/avro/src/util.rs b/avro/src/util.rs index 121e6d71..44e0ebb4 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -215,7 +215,8 @@ mod util { } pub fn safe_len(len: usize) -> AvroResult { - let max_bytes = crate::util::max_allocation_bytes(crate::util::DEFAULT_MAX_ALLOCATION_BYTES); + let max_bytes = + crate::util::max_allocation_bytes(crate::util::DEFAULT_MAX_ALLOCATION_BYTES); if len <= max_bytes { Ok(len) @@ -224,11 +225,10 @@ mod util { desired: len, maximum: max_bytes, } - .into()) + .into()) } } - #[cfg(test)] mod tests { use super::*; @@ -323,7 +323,11 @@ mod util { #[tokio::test] async fn test_overflow() { let causes_left_shift_overflow: &[u8] = &[0xe1, 0xe1, 0xe1, 0xe1, 0xe1]; - assert!(decode_variable(&mut &*causes_left_shift_overflow).await.is_err()); + assert!( + decode_variable(&mut &*causes_left_shift_overflow) + .await + .is_err() + ); } #[test] diff --git a/avro/src/writer.rs b/avro/src/writer.rs index cd6b735c..826206a1 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -49,21 +49,22 @@ mod writer { #[synca::cfg(sync)] use crate::AvroResult; use crate::{ - codec::tokio::Codec, error::tokio::Error, + codec::tokio::Codec, encode::tokio::{encode, encode_internal, encode_to_vec}, error::tokio::Details, + error::tokio::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, schema::tokio::{AvroSchema, Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, ser_schema::tokio::SchemaAwareWriteSerializer, types::tokio::Value, }; + #[cfg(feature = "tokio")] + use futures::TryFutureExt; use serde::Serialize; use std::{ collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive, }; - #[cfg(feature = "tokio")] - use futures::TryFutureExt; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; @@ -228,7 +229,7 @@ mod writer { None => { let rs = ResolvedSchema::try_from(self.schema)?; self.resolved_schema = Some(rs); - self.append_value_ref(value).await + Box::pin(self.append_value_ref(value)).await } } } @@ -563,7 +564,10 @@ mod writer { let rs = ResolvedSchema::try_from(schemata)?; let names = rs.get_names(); let enclosing_namespace = schema.namespace(); - if let Some(_err) = avro.validate_internal(schema, names, &enclosing_namespace).await { + if let Some(_err) = avro + .validate_internal(schema, names, &enclosing_namespace) + .await + { return Err(Details::Validation.into()); } encode_internal(&avro, schema, names, &enclosing_namespace, buffer) @@ -624,7 +628,11 @@ mod writer { } /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header - pub async fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { + pub async fn write_value( + &mut self, + v: Value, + writer: &mut W, + ) -> AvroResult { self.write_value_ref(&v, writer).await } } @@ -661,7 +669,11 @@ mod writer { { /// Write the `Into` to the provided Write object. Returns a result with the number /// of bytes written including the header - pub async fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { + pub async fn write_value( + &mut self, + data: T, + writer: &mut W, + ) -> AvroResult { let v: Value = data.into(); self.inner.write_value_ref(&v, writer).await } @@ -701,7 +713,10 @@ mod writer { value: &Value, buffer: &mut Vec, ) -> AvroResult { - match value.validate_internal(schema, resolved_schema.get_names(), &schema.namespace()).await { + match value + .validate_internal(schema, resolved_schema.get_names(), &schema.namespace()) + .await + { Some(reason) => Err(Details::ValidationWithReason { value: value.clone(), schema: schema.clone(), @@ -724,11 +739,14 @@ mod writer { buffer: &mut Vec, ) -> AvroResult<()> { let root_schema = resolved_schema.get_root_schema(); - if let Some(reason) = value.validate_internal( - root_schema, - resolved_schema.get_names(), - &root_schema.namespace(), - ).await { + if let Some(reason) = value + .validate_internal( + root_schema, + resolved_schema.get_names(), + &root_schema.namespace(), + ) + .await + { return Err(Details::ValidationWithReason { value: value.clone(), schema: root_schema.clone(), diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index bcb3a861..e6bbeaa0 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -16,8 +16,8 @@ // under the License. use apache_avro::schema::tokio::Schema; -use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro::types::tokio::Value; +use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; #[test] diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index 273ed696..827765cb 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro::schema::tokio::Schema; use apache_avro::types::tokio::Value; +use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; #[test] diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index 9b4a6536..a911ad33 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -24,7 +24,7 @@ use apache_avro::{ Codec, Reader, Writer, error::tokio::{Details, Error}, from_avro_datum, from_value, - schema::tokio::{Schema, EnumSchema, FixedSchema, Name, RecordField, RecordSchema}, + schema::tokio::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, to_avro_datum, to_value, types::tokio::{Record, Value}, }; From 1a2e8628e3d23f775f4716647e8b7cb9a8ce62c4 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 6 Aug 2025 00:13:14 +0300 Subject: [PATCH 10/47] WIP: Replace some more Iterator combinators with plain for loops This way we could use .await without extra troubles Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/reader.rs | 5 ++- avro/src/types.rs | 85 ++++++++++++++++++++++++++++++---------------- avro/src/writer.rs | 2 +- 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 24286479..173f0244 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -256,7 +256,10 @@ mod reader { Ok(Some(item)) } - async fn read_writer_schema(&mut self, metadata: &HashMap) -> AvroResult<()> { + async fn read_writer_schema( + &mut self, + metadata: &HashMap, + ) -> AvroResult<()> { let json: serde_json::Value = metadata .get("avro.schema") .and_then(|bytes| { diff --git a/avro/src/types.rs b/avro/src/types.rs index 4fe0c225..a8fd6955 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -564,8 +564,12 @@ mod types { for item in items.iter() { acc = Value::accumulate( acc, - Box::pin(item.validate_internal(&inner.items, names, enclosing_namespace)) - .await, + Box::pin(item.validate_internal( + &inner.items, + names, + enclosing_namespace, + )) + .await, ); } acc @@ -581,9 +585,12 @@ mod types { for (_, value) in items.iter() { acc = Value::accumulate( acc, - Box::pin(value - .validate_internal(&inner.types, names, enclosing_namespace)) - .await, + Box::pin(value.validate_internal( + &inner.types, + names, + enclosing_namespace, + )) + .await, ); } acc @@ -633,9 +640,12 @@ mod types { let field = &fields[*idx]; Value::accumulate( acc, - Box::pin(record_field - .validate_internal(&field.schema, names, record_namespace)) - .await, + Box::pin(record_field.validate_internal( + &field.schema, + names, + record_namespace, + )) + .await, ) } None => Value::accumulate( @@ -650,9 +660,12 @@ mod types { let mut acc = None; for field in fields.iter() { if let Some(item) = items.get(&field.name) { - let res = Box::pin(item - .validate_internal(&field.schema, names, enclosing_namespace)) - .await; + let res = Box::pin(item.validate_internal( + &field.schema, + names, + enclosing_namespace, + )) + .await; acc = Value::accumulate(acc, res); } else if !field.is_nullable() { acc = Value::accumulate( @@ -1139,12 +1152,14 @@ mod types { ) -> Result { match self { Value::Array(items) => { - let resolved = items.into_iter().map(|item| { - item.resolve_internal(schema, names, enclosing_namespace, &None) - }); - #[synca::cfg(tokio)] - let resolved = futures::future::try_join_all(resolved).await?; - Ok(Value::Array(resolved)) + let mut resolved_values = Vec::with_capacity(items.len()); + for item in items.into_iter() { + let resolved = item + .resolve_internal(schema, names, enclosing_namespace, &None) + .await?; + resolved_values.push(resolved); + } + Ok(Value::Array(resolved_values)) } other => Err(Details::GetArray { expected: schema.into(), @@ -1164,9 +1179,13 @@ mod types { Value::Map(items) => { let mut resolved = HashMap::with_capacity(items.len()); for (key, value) in items.into_iter() { - let v = Box::pin(value - .resolve_internal(schema, names, enclosing_namespace, &None)) - .await?; + let v = Box::pin(value.resolve_internal( + schema, + names, + enclosing_namespace, + &None, + )) + .await?; resolved.insert(key.clone(), v); } @@ -1231,12 +1250,15 @@ mod types { Schema::Null => Value::Union(0, Box::new(Value::Null)), _ => Value::Union( 0, - Box::new(Box::pin(Value::from(value.clone()).resolve_internal( - first, - names, - enclosing_namespace, - &field.default, - )).await?), + Box::new( + Box::pin(Value::from(value.clone()).resolve_internal( + first, + names, + enclosing_namespace, + &field.default, + )) + .await?, + ), ), } } @@ -1248,9 +1270,14 @@ mod types { }, }; - let v = Box::pin(value - .resolve_internal(&field.schema, names, enclosing_namespace, &field.default)).await?; - new_fields.push((field.name.clone(), v)); + let v = Box::pin(value.resolve_internal( + &field.schema, + names, + enclosing_namespace, + &field.default, + )) + .await?; + new_fields.push((field.name.clone(), v)); } Ok(Value::Record(new_fields)) diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 826206a1..5d536963 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -201,7 +201,7 @@ mod writer { let n = self.maybe_write_header()?; let avro = value.into(); - self.append_value_ref(&avro).map_ok(|m| m + n).await + self.append_value_ref(&avro).await.map(|m| m + n) } /// Append a compatible value to a `Writer`, also performing schema validation. From ae85dda6ced8a411f5c4f0f5ae1c2838f601ac5e Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 6 Aug 2025 09:32:41 +0300 Subject: [PATCH 11/47] WIP: `cargo build` now passes for both `tokio` and `sync` features Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/decode.rs | 34 +++++++++++++++++++--------------- avro/src/reader.rs | 7 ++++--- avro/src/types.rs | 18 ++++++++---------- avro/src/writer.rs | 2 -- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 7b3f54d6..4d56496a 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -176,19 +176,23 @@ mod decode { encode_long(len as i64, &mut reader)?; reader.extend_from_slice(&bytes); - let decode_from_string = async |reader| match decode_internal( - &Schema::String, - names, - enclosing_namespace, - reader, - ) - .await? - { - Value::String(ref s) => { - Uuid::from_str(s).map_err(|e| Details::ConvertStrToUuid(e).into()) + async fn decode_from_string(reader: &mut R, names: &HashMap, enclosing_namespace: &Namespace) -> AvroResult + where R: AvroRead + Unpin, S: Borrow { + + match decode_internal( + &Schema::String, + names, + enclosing_namespace, + reader, + ) + .await? + { + Value::String(ref s) => { + Uuid::from_str(s).map_err(|e| Details::ConvertStrToUuid(e).into()) + } + value => Err(Error::new(Details::GetUuidFromStringValue(value))), } - value => Err(Error::new(Details::GetUuidFromStringValue(value))), - }; + } let uuid: Uuid = if len == 16 { // most probably a Fixed schema @@ -214,15 +218,15 @@ mod decode { } Uuid::from_slice(bytes).map_err(Details::ConvertSliceToUuid)? } - _ => Box::pin(decode_from_string(&mut reader.as_slice())).await?, + _ => Box::pin(decode_from_string(&mut reader.as_slice(), names, enclosing_namespace)).await?, } } else { // try to decode as string - Box::pin(decode_from_string(&mut reader.as_slice())).await? + Box::pin(decode_from_string(&mut reader.as_slice(), names, enclosing_namespace)).await? } } else { // definitely a string - Box::pin(decode_from_string(&mut reader.as_slice())).await? + Box::pin(decode_from_string(&mut reader.as_slice(), names, enclosing_namespace)).await? }; Ok(Value::Uuid(uuid)) } diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 173f0244..83aee665 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -381,7 +381,8 @@ mod reader { should_resolve_schema: bool, } - impl<'a, R: AvroRead + Unpin> Reader<'a, R> { + impl<'a, R> Reader<'a, R> + where R: AvroRead + Unpin { /// Creates a `Reader` given something implementing the `io::Read` trait to read from. /// No reader `Schema` will be set. /// @@ -486,8 +487,8 @@ mod reader { // } // } - #[cfg(feature = "sync")] - impl Iterator for Reader<'_, R> { + #[synca::cfg(sync)] + impl Iterator for Reader<'_, R> { type Item = AvroResult; fn next(&mut self) -> Option { diff --git a/avro/src/types.rs b/avro/src/types.rs index a8fd6955..af716a0f 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -771,15 +771,15 @@ mod types { .. }) => self.resolve_enum(symbols, default, field_default), Schema::Array(ref inner) => { - self.resolve_array(&inner.items, names, enclosing_namespace) + Box::pin(self.resolve_array(&inner.items, names, enclosing_namespace)) .await } Schema::Map(ref inner) => { - self.resolve_map(&inner.types, names, enclosing_namespace) + Box::pin(self.resolve_map(&inner.types, names, enclosing_namespace)) .await } Schema::Record(RecordSchema { ref fields, .. }) => { - self.resolve_record(fields, names, enclosing_namespace) + Box::pin(self.resolve_record(fields, names, enclosing_namespace)) .await } Schema::Decimal(DecimalSchema { @@ -1154,8 +1154,8 @@ mod types { Value::Array(items) => { let mut resolved_values = Vec::with_capacity(items.len()); for item in items.into_iter() { - let resolved = item - .resolve_internal(schema, names, enclosing_namespace, &None) + let resolved = Box::pin(item + .resolve_internal(schema, names, enclosing_namespace, &None)) .await?; resolved_values.push(resolved); } @@ -1342,7 +1342,7 @@ mod types { } ] }"#, - )?; + ).await?; let value = Value::Record(vec![( "outer_field_1".into(), Value::Record(vec![ @@ -1498,13 +1498,11 @@ mod types { for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { let err_message = - value.validate_internal::(&schema, &HashMap::default(), &None); + value.validate_internal::(&schema, &HashMap::default(), &None).await; assert_eq!(valid, err_message.is_none()); if !valid { let full_err_message = format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, - schema, + "Invalid value: {value:?} for schema: {schema:?}. Reason: {}", err_message.unwrap() ); assert_eq!(expected_err_message, full_err_message); diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 5d536963..a475fde9 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -58,8 +58,6 @@ mod writer { ser_schema::tokio::SchemaAwareWriteSerializer, types::tokio::Value, }; - #[cfg(feature = "tokio")] - use futures::TryFutureExt; use serde::Serialize; use std::{ collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, From 106512fc6bacdedb3a1d6dc9c92fae78d3003ef2 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 6 Aug 2025 09:57:37 +0300 Subject: [PATCH 12/47] WIP: Make the tests async for bytes.rs, de.rs, decode.rs and encode.rs Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bytes.rs | 20 ++++++----- avro/src/de.rs | 44 +++++++++++++----------- avro/src/decode.rs | 62 ++++++++++++++++------------------ avro/src/encode.rs | 84 +++++++++++++++++++++++++--------------------- 4 files changed, 110 insertions(+), 100 deletions(-) diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 5da2844a..168458ca 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -293,9 +293,10 @@ mod tests { use crate::ser::tokio::to_value; use crate::types::tokio::Value; use serde::{Deserialize, Serialize}; + use apache_avro_test_helper::TestResult; - #[test] - fn avro_3631_validate_schema_for_struct_with_byte_types() { + #[tokio::test] + async fn avro_3631_validate_schema_for_struct_with_byte_types() -> TestResult { #[derive(Debug, Serialize)] struct TestStructWithBytes<'a> { #[serde(with = "serde_avro_bytes")] @@ -357,12 +358,13 @@ mod tests { } ] }"#, ) - .unwrap(); - assert!(value.validate(&schema)); + .await?; + assert!(value.validate(&schema).await); + Ok(()) } - #[test] - fn avro_3631_deserialize_value_to_struct_with_byte_types() { + #[tokio::test] + async fn avro_3631_deserialize_value_to_struct_with_byte_types() -> TestResult { #[derive(Debug, Deserialize, PartialEq)] struct TestStructWithBytes<'a> { #[serde(with = "serde_avro_bytes")] @@ -485,10 +487,11 @@ mod tests { ), ]); assert_eq!(expected, from_value(&value).unwrap()); + Ok(()) } - #[test] - fn avro_3631_serialize_struct_to_value_with_byte_types() { + #[tokio::test] + async fn avro_3631_serialize_struct_to_value_with_byte_types() -> TestResult { #[derive(Debug, Serialize)] struct TestStructWithBytes<'a> { array_field: &'a [u8], @@ -683,5 +686,6 @@ mod tests { ), ]); assert_eq!(expected, to_value(test).unwrap()); + Ok(()) } } diff --git a/avro/src/de.rs b/avro/src/de.rs index d5f98827..4bde44c9 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -797,6 +797,8 @@ mod de { #[cfg(test)] mod tests { use crate::ser::tokio::to_value; + use crate::reader::tokio::from_avro_datum; + use crate::writer::tokio::to_avro_datum; use num_bigint::BigInt; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; @@ -805,8 +807,10 @@ mod de { use uuid::Uuid; use apache_avro_test_helper::TestResult; + use crate::de::tokio::from_value; - use crate::{Decimal, Schema}; + use crate::decimal::tokio::Decimal; + use crate::schema::tokio::Schema; use super::*; @@ -815,8 +819,8 @@ mod de { pub source: String, } - #[test] - fn avro_3955_decode_enum() -> TestResult { + #[tokio::test] + async fn avro_3955_decode_enum() -> TestResult { let schema_content = r#" { "name": "AccessLog", @@ -836,7 +840,7 @@ mod de { } "#; - let schema = Schema::parse_str(schema_content)?; + let schema = Schema::parse_str(schema_content).await?; let data = StringEnum { source: "SOZU".to_string(), }; @@ -844,20 +848,20 @@ mod de { // encode into avro let value = to_value(&data)?; - let mut buf = std::io::Cursor::new(crate::to_avro_datum(&schema, value)?); + let mut buf = std::io::Cursor::new(to_avro_datum(&schema, value).await?); // decode from avro - let value = from_avro_datum(&schema, &mut buf, None)?; + let value = from_avro_datum(&schema, &mut buf, None).await?; - let decoded_data: StringEnum = crate::from_value(&value)?; + let decoded_data: StringEnum = from_value(&value)?; assert_eq!(decoded_data, data); Ok(()) } - #[test] - fn avro_3955_encode_enum_data_with_wrong_content() -> TestResult { + #[tokio::test] + async fn avro_3955_encode_enum_data_with_wrong_content() -> TestResult { let schema_content = r#" { "name": "AccessLog", @@ -877,16 +881,16 @@ mod de { } "#; - let schema = crate::Schema::parse_str(schema_content)?; + let schema = Schema::parse_str(schema_content).await?; let data = StringEnum { source: "WRONG_ITEM".to_string(), }; // encode into avro - let value = crate::to_value(data)?; + let value = to_value(data)?; // The following sentence have to fail has the data is wrong. - let encoded_data = crate::to_avro_datum(&schema, value); + let encoded_data = to_avro_datum(&schema, value).await; assert!(encoded_data.is_err()); @@ -1269,7 +1273,7 @@ mod de { fn test_date() -> TestResult { let raw_value = 1; let value = Value::Date(raw_value); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result, raw_value); Ok(()) } @@ -1278,7 +1282,7 @@ mod de { fn test_time_millis() -> TestResult { let raw_value = 1; let value = Value::TimeMillis(raw_value); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result, raw_value); Ok(()) } @@ -1287,7 +1291,7 @@ mod de { fn test_time_micros() -> TestResult { let raw_value = 1; let value = Value::TimeMicros(raw_value); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result, raw_value); Ok(()) } @@ -1296,7 +1300,7 @@ mod de { fn test_timestamp_millis() -> TestResult { let raw_value = 1; let value = Value::TimestampMillis(raw_value); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result, raw_value); Ok(()) } @@ -1332,7 +1336,7 @@ mod de { fn test_avro_3853_local_timestamp_micros() -> TestResult { let raw_value = 1; let value = Value::LocalTimestampMicros(raw_value); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result, raw_value); Ok(()) } @@ -1341,7 +1345,7 @@ mod de { fn test_avro_3916_local_timestamp_nanos() -> TestResult { let raw_value = 1; let value = Value::LocalTimestampNanos(raw_value); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result, raw_value); Ok(()) } @@ -1359,7 +1363,7 @@ mod de { fn test_from_value_uuid_slice() -> TestResult { let raw_value = &[4, 54, 67, 12, 43, 2, 2, 76, 32, 50, 87, 5, 1, 33, 43, 87]; let value = Value::Uuid(Uuid::from_slice(raw_value)?); - let result = crate::from_value::(&value)?; + let result = from_value::(&value)?; assert_eq!(result.as_bytes(), raw_value); Ok(()) } @@ -1560,7 +1564,7 @@ mod de { ), ]); - let deserialized: StructWithMissingFields = crate::from_value(&record)?; + let deserialized: StructWithMissingFields = from_value(&record)?; let reference = StructWithMissingFields { a_string: "a valid message field".to_string(), a_record: Some(RecordInUnion { diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 4d56496a..b2a4b010 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -447,11 +447,11 @@ mod decode { #[allow(clippy::expect_fun_call)] mod tests { use crate::{ - Decimal, - decode::decode, - encode::{encode, tests::success}, - schema::{DecimalSchema, FixedSchema, Schema}, - types::{ + decimal::tokio::Decimal, + decode::tokio::decode, + encode::tokio::{encode, tests::success}, + schema::tokio::{DecimalSchema, FixedSchema, Schema, Name}, + types::tokio::{ Value, Value::{Array, Int, Map}, }, @@ -464,7 +464,7 @@ mod decode { #[tokio::test] async fn test_decode_array_without_size() -> TestResult { let mut input: &[u8] = &[6, 2, 4, 6, 0]; - let result = decode(&Schema::array(Schema::Int), &mut input); + let result = decode(&Schema::array(Schema::Int), &mut input).await; assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); Ok(()) @@ -473,7 +473,7 @@ mod decode { #[tokio::test] async fn test_decode_array_with_size() -> TestResult { let mut input: &[u8] = &[5, 6, 2, 4, 6, 0]; - let result = decode(&Schema::array(Schema::Int), &mut input); + let result = decode(&Schema::array(Schema::Int), &mut input).await; assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); Ok(()) @@ -482,7 +482,7 @@ mod decode { #[tokio::test] async fn test_decode_map_without_size() -> TestResult { let mut input: &[u8] = &[0x02, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; - let result = decode(&Schema::map(Schema::Int), &mut input); + let result = decode(&Schema::map(Schema::Int), &mut input).await; let mut expected = HashMap::new(); expected.insert(String::from("test"), Int(1)); assert_eq!(Map(expected), result?); @@ -493,7 +493,7 @@ mod decode { #[tokio::test] async fn test_decode_map_with_size() -> TestResult { let mut input: &[u8] = &[0x01, 0x0C, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; - let result = decode(&Schema::map(Schema::Int), &mut input); + let result = decode(&Schema::map(Schema::Int), &mut input).await; let mut expected = HashMap::new(); expected.insert(String::from("test"), Int(1)); assert_eq!(Map(expected), result?); @@ -503,7 +503,6 @@ mod decode { #[tokio::test] async fn test_negative_decimal_value() -> TestResult { - use crate::{encode::encode, schema::Name}; use num_bigint::ToBigInt; let inner = Box::new(Schema::Fixed( FixedSchema::builder() @@ -523,7 +522,7 @@ mod decode { encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); let mut bytes = &buffer[..]; - let result = decode(&schema, &mut bytes)?; + let result = decode(&schema, &mut bytes).await?; assert_eq!(result, value); Ok(()) @@ -531,7 +530,6 @@ mod decode { #[tokio::test] async fn test_decode_decimal_with_bigger_than_necessary_size() -> TestResult { - use crate::{encode::encode, schema::Name}; use num_bigint::ToBigInt; let inner = Box::new(Schema::Fixed(FixedSchema { size: 13, @@ -553,7 +551,7 @@ mod decode { encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); let mut bytes: &[u8] = &buffer[..]; - let result = decode(&schema, &mut bytes)?; + let result = decode(&schema, &mut bytes).await?; assert_eq!(result, value); Ok(()) @@ -585,7 +583,7 @@ mod decode { } ] }"#, - )?; + ).await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -599,7 +597,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_value1, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to decode using recursive definitions with schema:\n {:?}\n", &schema )) @@ -614,7 +612,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_value2, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to decode using recursive definitions with schema:\n {:?}\n", &schema )) @@ -651,7 +649,7 @@ mod decode { } ] }"#, - )?; + ).await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -664,7 +662,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_value, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to decode using recursive definitions with schema:\n {:?}\n", &schema )) @@ -701,7 +699,7 @@ mod decode { } ] }"#, - )?; + ).await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -717,7 +715,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_value, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to decode using recursive definitions with schema:\n {:?}\n", &schema )) @@ -771,7 +769,7 @@ mod decode { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -809,7 +807,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_record_variation_1, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", &schema )) @@ -821,7 +819,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_record_variation_2, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", &schema )) @@ -833,7 +831,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_record_variation_3, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", &schema )) @@ -888,7 +886,7 @@ mod decode { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -926,7 +924,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_record_variation_1, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", &schema )) @@ -938,7 +936,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_record_variation_2, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", &schema )) @@ -950,7 +948,7 @@ mod decode { let mut bytes = &buf[..]; assert_eq!( outer_record_variation_3, - decode(&schema, &mut bytes).expect(&format!( + decode(&schema, &mut bytes).await.expect(&format!( "Failed to Decode with recursively defined namespace with schema:\n {:?}\n", &schema )) @@ -961,15 +959,13 @@ mod decode { #[tokio::test] async fn avro_3926_encode_decode_uuid_to_string() -> TestResult { - use crate::encode::encode; - let schema = Schema::String; let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); - let result = decode(&Schema::Uuid, &mut &buffer[..])?; + let result = decode(&Schema::Uuid, &mut &buffer[..]).await?; assert_eq!(result, value); Ok(()) @@ -977,8 +973,6 @@ mod decode { #[tokio::test] async fn avro_3926_encode_decode_uuid_to_fixed() -> TestResult { - use crate::encode::encode; - let schema = Schema::Fixed(FixedSchema { size: 16, name: "uuid".into(), @@ -992,7 +986,7 @@ mod decode { let mut buffer = Vec::new(); encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); - let result = decode(&Schema::Uuid, &mut &buffer[..])?; + let result = decode(&Schema::Uuid, &mut &buffer[..]).await?; assert_eq!(result, value); Ok(()) diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 35bc7ed2..f45e4c83 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -30,7 +30,7 @@ crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, - #[tokio::test] => #[test] + #[tokio::test] => #[tokio::test] ); } )] @@ -379,7 +379,7 @@ mod encode { #[allow(clippy::expect_fun_call)] pub(crate) mod tests { use super::*; - use crate::error::{Details, Error}; + use crate::error::tokio::{Details, Error}; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use uuid::Uuid; @@ -391,8 +391,8 @@ mod encode { ) } - #[test] - fn test_encode_empty_array() { + #[tokio::test] + async fn test_encode_empty_array() { let mut buf = Vec::new(); let empty: Vec = Vec::new(); encode( @@ -404,8 +404,8 @@ mod encode { assert_eq!(vec![0u8], buf); } - #[test] - fn test_encode_empty_map() { + #[tokio::test] + async fn test_encode_empty_map() { let mut buf = Vec::new(); let empty: HashMap = HashMap::new(); encode( @@ -417,8 +417,8 @@ mod encode { assert_eq!(vec![0u8], buf); } - #[test] - fn test_avro_3433_recursive_definition_encode_record() { + #[tokio::test] + async fn test_avro_3433_recursive_definition_encode_record() -> TestResult { let mut buf = Vec::new(); let schema = Schema::parse_str( r#" @@ -443,8 +443,7 @@ mod encode { } ] }"#, - ) - .unwrap(); + ).await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -452,10 +451,11 @@ mod encode { Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3433_recursive_definition_encode_array() { + #[tokio::test] + async fn test_avro_3433_recursive_definition_encode_array() -> TestResult { let mut buf = Vec::new(); let schema = Schema::parse_str( r#" @@ -487,7 +487,7 @@ mod encode { ] }"#, ) - .unwrap(); + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -500,10 +500,11 @@ mod encode { ]); encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3433_recursive_definition_encode_map() { + #[tokio::test] + async fn test_avro_3433_recursive_definition_encode_map() -> TestResult { let mut buf = Vec::new(); let schema = Schema::parse_str( r#" @@ -532,7 +533,7 @@ mod encode { ] }"#, ) - .unwrap(); + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -545,10 +546,11 @@ mod encode { ]); encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3433_recursive_definition_encode_record_wrapper() { + #[tokio::test] + async fn test_avro_3433_recursive_definition_encode_record_wrapper() -> TestResult { let mut buf = Vec::new(); let schema = Schema::parse_str( r#" @@ -581,7 +583,7 @@ mod encode { ] }"#, ) - .unwrap(); + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![( @@ -592,10 +594,11 @@ mod encode { Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3433_recursive_definition_encode_map_and_array() { + #[tokio::test] + async fn test_avro_3433_recursive_definition_encode_map_and_array() -> TestResult { let mut buf = Vec::new(); let schema = Schema::parse_str( r#" @@ -627,7 +630,7 @@ mod encode { ] }"#, ) - .unwrap(); + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -640,10 +643,11 @@ mod encode { ]); encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3433_recursive_definition_encode_union() { + #[tokio::test] + async fn test_avro_3433_recursive_definition_encode_union() -> TestResult { let mut buf = Vec::new(); let schema = Schema::parse_str( r#" @@ -669,7 +673,7 @@ mod encode { ] }"#, ) - .unwrap(); + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -687,10 +691,11 @@ mod encode { ]); encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value1, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3448_proper_multi_level_encoding_outer_namespace() { + #[tokio::test] + async fn test_avro_3448_proper_multi_level_encoding_outer_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -732,7 +737,7 @@ mod encode { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -776,10 +781,11 @@ mod encode { encode(&outer_record_variation_3, &schema, &mut buf) .expect(&success(&outer_record_variation_3, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3448_proper_multi_level_encoding_middle_namespace() { + #[tokio::test] + async fn test_avro_3448_proper_multi_level_encoding_middle_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -822,7 +828,7 @@ mod encode { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -866,10 +872,11 @@ mod encode { encode(&outer_record_variation_3, &schema, &mut buf) .expect(&success(&outer_record_variation_3, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3448_proper_multi_level_encoding_inner_namespace() { + #[tokio::test] + async fn test_avro_3448_proper_multi_level_encoding_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -913,7 +920,7 @@ mod encode { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -957,10 +964,11 @@ mod encode { encode(&outer_record_variation_3, &schema, &mut buf) .expect(&success(&outer_record_variation_3, &schema)); assert!(!buf.is_empty()); + Ok(()) } - #[test] - fn test_avro_3585_encode_uuids() { + #[tokio::test] + async fn test_avro_3585_encode_uuids() { let value = Value::String(String::from("00000000-0000-0000-0000-000000000000")); let schema = Schema::Uuid; let mut buffer = Vec::new(); @@ -969,8 +977,8 @@ mod encode { assert!(!buffer.is_empty()); } - #[test] - fn avro_3926_encode_decode_uuid_to_fixed_wrong_schema_size() -> TestResult { + #[tokio::test] + async fn avro_3926_encode_decode_uuid_to_fixed_wrong_schema_size() -> TestResult { let schema = Schema::Fixed(FixedSchema { size: 15, name: "uuid".into(), From 4d917c1e82b804951d35fd33f5591891d7cfd28b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 6 Aug 2025 15:21:50 +0300 Subject: [PATCH 13/47] WIP: Unit tests now build OK TODO: examples, doc tests and IT tests Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 12 + avro/Cargo.toml | 3 +- avro/src/bigdecimal.rs | 15 +- avro/src/bytes.rs | 17 +- avro/src/de.rs | 6 +- avro/src/decode.rs | 76 ++- avro/src/encode.rs | 2 +- avro/src/headers.rs | 8 +- avro/src/lib.rs | 85 +-- avro/src/reader.rs | 88 +-- avro/src/schema.rs | 894 +++++++++++++++++-------------- avro/src/schema_compatibility.rs | 583 ++++++++++++-------- avro/src/schema_equality.rs | 2 +- avro/src/ser.rs | 2 +- avro/src/ser_schema.rs | 112 ++-- avro/src/types.rs | 149 +++--- avro/src/validator.rs | 2 +- avro/src/writer.rs | 247 +++++---- 18 files changed, 1312 insertions(+), 991 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9c58ca9..f48031d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,7 @@ dependencies = [ "anyhow", "apache-avro-derive", "apache-avro-test-helper", + "async-trait", "bigdecimal", "bon", "bzip2", @@ -110,6 +111,17 @@ dependencies = [ "log", ] +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.4.0" diff --git a/avro/Cargo.toml b/avro/Cargo.toml index c5539370..5577fadc 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -36,7 +36,7 @@ snappy = ["dep:crc32fast", "dep:snap"] xz = ["dep:xz2"] zstandard = ["dep:zstd"] sync = [] -tokio = ["dep:tokio", "dep:futures"] +tokio = ["dep:tokio", "dep:futures", "dep:async-trait"] [lib] # disable benchmarks to allow passing criterion arguments to `cargo bench` @@ -80,6 +80,7 @@ zstd = { default-features = false, version = "0.13.3", optional = true } #synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing" } synca = { path = "/home/martin/git/rust/rs_synca/synca" } futures = { version = "0.3.31", optional = true } +async-trait = { version = "0.1.88", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 36d65e87..38420c2f 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -23,13 +23,16 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::codec::tokio => crate::codec::sync, crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, crate::schema::tokio => crate::schema::sync, + crate::reader::tokio => crate::reader::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, + crate::writer::tokio => crate::writer::sync, #[tokio::test] => #[test] ); } @@ -125,7 +128,7 @@ mod bigdecimal { result.extend_from_slice(as_slice); let deserialize_big_decimal: Result = - deserialize_big_decimal(&result); + deserialize_big_decimal(&result).await; assert!( deserialize_big_decimal.is_ok(), "can't deserialize for iter {iter}" @@ -142,7 +145,7 @@ mod bigdecimal { result.extend_from_slice(as_slice); let deserialize_big_decimal: Result = - deserialize_big_decimal(&result); + deserialize_big_decimal(&result).await; assert!( deserialize_big_decimal.is_ok(), "can't deserialize for zero" @@ -171,7 +174,7 @@ mod bigdecimal { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; // build record with big decimal value let mut record = Record::new(&schema).unwrap(); @@ -186,12 +189,12 @@ mod bigdecimal { .writer(Vec::new()) .build(); - writer.append(record.clone())?; + writer.append(record.clone()).await?; writer.flush()?; // read record let wrote_data = writer.into_inner()?; - let mut reader = Reader::new(&wrote_data[..])?; + let mut reader = Reader::new(&wrote_data[..]).await?; let value = reader.next().await.unwrap()?; @@ -215,7 +218,7 @@ mod bigdecimal { // Open file generated with Java code to ensure compatibility // with Java big decimal logical type. let file: File = File::open("./tests/bigdec.avro")?; - let mut reader = Reader::new(BufReader::new(&file))?; + let mut reader = Reader::new(BufReader::new(&file)).await?; let next_element = reader.next().await; assert!(next_element.is_some()); let value = next_element.unwrap()?; diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 168458ca..933a8ae6 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -285,6 +285,21 @@ pub mod serde_avro_slice_opt { } } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio_tests { }, + #[cfg(feature = "sync")] + pub mod sync_tests { + sync!(); + replace!( + crate::de::tokio => crate::de::sync, + crate::schema::tokio => crate::schema::sync, + crate::ser::tokio => crate::ser::sync, + crate::types::tokio => crate::types::sync, + #[tokio::test] => #[test] + ); + } +)] #[cfg(test)] mod tests { use super::*; @@ -292,8 +307,8 @@ mod tests { use crate::schema::tokio::Schema; use crate::ser::tokio::to_value; use crate::types::tokio::Value; - use serde::{Deserialize, Serialize}; use apache_avro_test_helper::TestResult; + use serde::{Deserialize, Serialize}; #[tokio::test] async fn avro_3631_validate_schema_for_struct_with_byte_types() -> TestResult { diff --git a/avro/src/de.rs b/avro/src/de.rs index 4bde44c9..9f678bef 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -29,9 +29,12 @@ crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, + crate::ser::tokio => crate::ser::sync, crate::schema::tokio => crate::schema::sync, + crate::reader::tokio => crate::reader::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, + crate::writer::tokio => crate::writer::sync, #[tokio::test] => #[test] ); } @@ -796,8 +799,8 @@ mod de { #[cfg(test)] mod tests { - use crate::ser::tokio::to_value; use crate::reader::tokio::from_avro_datum; + use crate::ser::tokio::to_value; use crate::writer::tokio::to_avro_datum; use num_bigint::BigInt; use pretty_assertions::assert_eq; @@ -807,7 +810,6 @@ mod de { use uuid::Uuid; use apache_avro_test_helper::TestResult; - use crate::de::tokio::from_value; use crate::decimal::tokio::Decimal; use crate::schema::tokio::Schema; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index b2a4b010..78381bae 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -176,15 +176,16 @@ mod decode { encode_long(len as i64, &mut reader)?; reader.extend_from_slice(&bytes); - async fn decode_from_string(reader: &mut R, names: &HashMap, enclosing_namespace: &Namespace) -> AvroResult - where R: AvroRead + Unpin, S: Borrow { - - match decode_internal( - &Schema::String, - names, - enclosing_namespace, - reader, - ) + async fn decode_from_string( + reader: &mut R, + names: &HashMap, + enclosing_namespace: &Namespace, + ) -> AvroResult + where + R: AvroRead + Unpin, + S: Borrow, + { + match decode_internal(&Schema::String, names, enclosing_namespace, reader) .await? { Value::String(ref s) => { @@ -218,15 +219,32 @@ mod decode { } Uuid::from_slice(bytes).map_err(Details::ConvertSliceToUuid)? } - _ => Box::pin(decode_from_string(&mut reader.as_slice(), names, enclosing_namespace)).await?, + _ => { + Box::pin(decode_from_string( + &mut reader.as_slice(), + names, + enclosing_namespace, + )) + .await? + } } } else { // try to decode as string - Box::pin(decode_from_string(&mut reader.as_slice(), names, enclosing_namespace)).await? + Box::pin(decode_from_string( + &mut reader.as_slice(), + names, + enclosing_namespace, + )) + .await? } } else { // definitely a string - Box::pin(decode_from_string(&mut reader.as_slice(), names, enclosing_namespace)).await? + Box::pin(decode_from_string( + &mut reader.as_slice(), + names, + enclosing_namespace, + )) + .await? }; Ok(Value::Uuid(uuid)) } @@ -450,11 +468,8 @@ mod decode { decimal::tokio::Decimal, decode::tokio::decode, encode::tokio::{encode, tests::success}, - schema::tokio::{DecimalSchema, FixedSchema, Schema, Name}, - types::tokio::{ - Value, - Value::{Array, Int, Map}, - }, + schema::tokio::{DecimalSchema, FixedSchema, Name, Schema}, + types::tokio::Value, }; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; @@ -465,7 +480,10 @@ mod decode { async fn test_decode_array_without_size() -> TestResult { let mut input: &[u8] = &[6, 2, 4, 6, 0]; let result = decode(&Schema::array(Schema::Int), &mut input).await; - assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); + assert_eq!( + Value::Array(vec!(Value::Int(1), Value::Int(2), Value::Int(3))), + result? + ); Ok(()) } @@ -474,7 +492,10 @@ mod decode { async fn test_decode_array_with_size() -> TestResult { let mut input: &[u8] = &[5, 6, 2, 4, 6, 0]; let result = decode(&Schema::array(Schema::Int), &mut input).await; - assert_eq!(Array(vec!(Int(1), Int(2), Int(3))), result?); + assert_eq!( + Value::Array(vec!(Value::Int(1), Value::Int(2), Value::Int(3))), + result? + ); Ok(()) } @@ -484,8 +505,8 @@ mod decode { let mut input: &[u8] = &[0x02, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; let result = decode(&Schema::map(Schema::Int), &mut input).await; let mut expected = HashMap::new(); - expected.insert(String::from("test"), Int(1)); - assert_eq!(Map(expected), result?); + expected.insert(String::from("test"), Value::Int(1)); + assert_eq!(Value::Map(expected), result?); Ok(()) } @@ -495,8 +516,8 @@ mod decode { let mut input: &[u8] = &[0x01, 0x0C, 0x08, 0x74, 0x65, 0x73, 0x74, 0x02, 0x00]; let result = decode(&Schema::map(Schema::Int), &mut input).await; let mut expected = HashMap::new(); - expected.insert(String::from("test"), Int(1)); - assert_eq!(Map(expected), result?); + expected.insert(String::from("test"), Value::Int(1)); + assert_eq!(Value::Map(expected), result?); Ok(()) } @@ -583,7 +604,8 @@ mod decode { } ] }"#, - ).await?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -649,7 +671,8 @@ mod decode { } ] }"#, - ).await?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -699,7 +722,8 @@ mod decode { } ] }"#, - ).await?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); diff --git a/avro/src/encode.rs b/avro/src/encode.rs index f45e4c83..7b7d699a 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -30,7 +30,7 @@ crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, - #[tokio::test] => #[tokio::test] + #[tokio::test] => #[test] ); } )] diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 8866a66e..2a99e4d7 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -141,11 +141,11 @@ mod headers { #[cfg(test)] mod test { use super::*; - use crate::{Error, error::tokio::Details}; + use crate::error::tokio::{Error, Details}; use apache_avro_test_helper::TestResult; - #[test] - fn test_rabin_fingerprint_header() -> TestResult { + #[tokio::test] + async fn test_rabin_fingerprint_header() -> TestResult { let schema_str = r#" { "type": "record", @@ -163,7 +163,7 @@ mod headers { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let header_builder = RabinFingerprintHeader::from_schema(&schema); let computed_header = header_builder.build_header(); let expected_header: Vec = vec![195, 1, 232, 198, 194, 12, 97, 95, 44, 71]; diff --git a/avro/src/lib.rs b/avro/src/lib.rs index ea0b51e2..f829bb30 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -129,7 +129,7 @@ //! "#; //! //! // if the schema is not valid, this function will return an error -//! let schema = Schema::parse_str(raw_schema).unwrap(); +//! let schema = Schema::parse_str(raw_schema).await.unwrap(); //! //! // schemas can be printed for debugging //! println!("{:?}", schema); @@ -206,7 +206,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); //! // a writer needs a schema and something to write to //! let mut writer = Writer::new(&schema, Vec::new()); //! @@ -261,7 +261,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); //! // a writer needs a schema and something to write to //! let mut writer = Writer::new(&schema, Vec::new()); //! @@ -326,7 +326,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); //! let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate(DeflateSettings::default())); //! ``` //! @@ -352,7 +352,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let mut record = Record::new(writer.schema()).unwrap(); //! # record.put("a", 27i64); @@ -381,7 +381,7 @@ //! # ] //! # } //! # "#; -//! # let writer_schema = Schema::parse_str(writer_raw_schema).unwrap(); +//! # let writer_schema = Schema::parse_str(writer_raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&writer_schema, Vec::new()); //! # let mut record = Record::new(writer.schema()).unwrap(); //! # record.put("a", 27i64); @@ -401,7 +401,7 @@ //! } //! "#; //! -//! let reader_schema = Schema::parse_str(reader_raw_schema).unwrap(); +//! let reader_schema = Schema::parse_str(reader_raw_schema).await.unwrap(); //! //! // reader creation can fail in case the input to read from is not Avro-compatible or malformed //! let reader = Reader::with_schema(&reader_schema, &input[..]).unwrap(); @@ -440,8 +440,8 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).unwrap(); -//! # let schema = Schema::parse_str(raw_schema).unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let mut record = Record::new(writer.schema()).unwrap(); //! # record.put("a", 27i64); @@ -486,7 +486,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).unwrap(); +//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let test = Test { //! # a: 27, @@ -529,7 +529,7 @@ //! } //! "#; //! -//! let schema = Schema::parse_str(raw_schema)?; +//! let schema = Schema::parse_str(raw_schema).await?; //! //! println!("{:?}", schema); //! @@ -655,7 +655,7 @@ //! } //! "#; //! -//! let schema = Schema::parse_str(raw_schema)?; +//! let schema = Schema::parse_str(raw_schema).await?; //! //! println!("{:?}", schema); //! @@ -715,7 +715,7 @@ //! ] //! } //! "#; -//! let schema = Schema::parse_str(raw_schema)?; +//! let schema = Schema::parse_str(raw_schema).await?; //! println!("{}", schema.fingerprint::()); //! println!("{}", schema.fingerprint::()); //! println!("{}", schema.fingerprint::()); @@ -766,8 +766,8 @@ //! ```rust //! use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; //! -//! let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); -//! let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); +//! let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); +//! let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); //! assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_ok()); //! ``` //! @@ -779,8 +779,8 @@ //! ```rust //! use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; //! -//! let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); -//! let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); +//! let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); +//! let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); //! assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_err()); //! ``` //! ## Custom names validators @@ -959,11 +959,30 @@ pub type AsyncAvroResult = Result; #[cfg(feature = "sync")] pub type AvroResult = Result; +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio_tests { }, + #[cfg(feature = "sync")] + pub mod sync_tests { + sync!(); + replace!( + crate::codec::tokio => crate::codec::sync, + crate::reader::tokio => crate::reader::sync, + crate::schema::tokio => crate::schema::sync, + crate::types::tokio => crate::types::sync, + crate::writer::tokio => crate::writer::sync, + #[tokio::test] => #[test] + ); + } +)] #[cfg(test)] mod tests { use crate::{ - Codec, Reader, Schema, Writer, from_avro_datum, + codec::tokio::Codec, + reader::tokio::{Reader, from_avro_datum}, + schema::tokio::Schema, types::tokio::{Record, Value}, + writer::tokio::Writer, }; use pretty_assertions::assert_eq; @@ -999,13 +1018,13 @@ mod tests { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema).unwrap(); - let reader_schema = Schema::parse_str(reader_raw_schema).unwrap(); + let writer_schema = Schema::parse_str(writer_raw_schema).await.unwrap(); + let reader_schema = Schema::parse_str(reader_raw_schema).await.unwrap(); let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); record.put("b", "foo"); - writer.append(record).unwrap(); + writer.append(record).await.unwrap(); let input = writer.into_inner().unwrap(); let mut reader = Reader::with_schema(&reader_schema, &input[..]) .await @@ -1022,8 +1041,8 @@ mod tests { } //TODO: move where it fits better - #[test] - fn test_enum_string_value() { + #[tokio::test] + async fn test_enum_string_value() { let raw_schema = r#" { "type": "record", @@ -1043,15 +1062,15 @@ mod tests { ] } "#; - let schema = Schema::parse_str(raw_schema).unwrap(); + let schema = Schema::parse_str(raw_schema).await.unwrap(); let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); record.put("b", "foo"); record.put("c", "clubs"); - writer.append(record).unwrap(); + writer.append(record).await.unwrap(); let input = writer.into_inner().unwrap(); - let mut reader = Reader::with_schema(&schema, &input[..]).unwrap(); + let mut reader = Reader::with_schema(&schema, &input[..]).await.unwrap(); assert_eq!( reader.next().unwrap().unwrap(), Value::Record(vec![ @@ -1085,15 +1104,15 @@ mod tests { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema).unwrap(); + let writer_schema = Schema::parse_str(writer_raw_schema).await.unwrap(); let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); record.put("b", "foo"); record.put("c", "clubs"); - writer.append(record).unwrap(); + writer.append(record).await.unwrap(); let input = writer.into_inner().unwrap(); - let mut reader = Reader::new(&input[..]).unwrap(); + let mut reader = Reader::new(&input[..]).await.unwrap(); assert_eq!( reader.next().await.unwrap().unwrap(), Value::Record(vec![ @@ -1104,8 +1123,8 @@ mod tests { ); } - #[test] - fn test_illformed_length() { + #[tokio::test] + async fn test_illformed_length() { let raw_schema = r#" { "type": "record", @@ -1117,12 +1136,12 @@ mod tests { } "#; - let schema = Schema::parse_str(raw_schema).unwrap(); + let schema = Schema::parse_str(raw_schema).await.unwrap(); // Would allocated 18446744073709551605 bytes let illformed: &[u8] = &[0x3e, 0x15, 0xff, 0x1f, 0x15, 0xff]; - let value = from_avro_datum(&schema, &mut &*illformed, None); + let value = from_avro_datum(&schema, &mut &*illformed, None).await; assert!(value.is_err()); } } diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 83aee665..08ae037c 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -32,6 +32,7 @@ crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, + crate::writer::tokio => crate::writer::sync, #[tokio::test] => #[test] ); } @@ -382,7 +383,9 @@ mod reader { } impl<'a, R> Reader<'a, R> - where R: AvroRead + Unpin { + where + R: AvroRead + Unpin, + { /// Creates a `Reader` given something implementing the `io::Read` trait to read from. /// No reader `Schema` will be set. /// @@ -634,9 +637,9 @@ mod reader { where T: AvroSchema, { - pub fn new() -> AvroResult> { + pub async fn new() -> AvroResult> { Ok(SpecificSingleObjectReader { - inner: GenericSingleObjectReader::new(T::get_schema())?, + inner: GenericSingleObjectReader::new(T::get_schema().await)?, _model: PhantomData, }) } @@ -674,7 +677,10 @@ mod reader { #[cfg(test)] mod tests { use super::*; - use crate::{encode::encode, headers::GlueSchemaUuidHeader, rabin::Rabin, types::Record}; + use crate::{ + encode::tokio::encode, headers::tokio::GlueSchemaUuidHeader, rabin::Rabin, + types::tokio::Record, + }; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use serde::Deserialize; @@ -719,7 +725,7 @@ mod reader { #[tokio::test] async fn test_from_avro_datum() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; let mut record = Record::new(&schema).unwrap(); @@ -727,7 +733,10 @@ mod reader { record.put("b", "foo"); let expected = record.into(); - assert_eq!(from_avro_datum(&schema, &mut encoded, None)?, expected); + assert_eq!( + from_avro_datum(&schema, &mut encoded, None).await?, + expected + ); Ok(()) } @@ -776,7 +785,7 @@ mod reader { a_nullable_string: Option, } - let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240)?; + let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240).await?; let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; let expected_record: TestRecord3240 = TestRecord3240 { @@ -786,7 +795,7 @@ mod reader { a_nullable_string: None, }; - let avro_datum = from_avro_datum(&schema, &mut encoded, None)?; + let avro_datum = from_avro_datum(&schema, &mut encoded, None).await?; let parsed_record: TestRecord3240 = match &avro_datum { Value::Record(_) => from_value::(&avro_datum)?, unexpected => { @@ -801,11 +810,11 @@ mod reader { #[tokio::test] async fn test_null_union() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA)?; + let schema = Schema::parse_str(UNION_SCHEMA).await?; let mut encoded: &'static [u8] = &[2, 0]; assert_eq!( - from_avro_datum(&schema, &mut encoded, None)?, + from_avro_datum(&schema, &mut encoded, None).await?, Value::Union(1, Box::new(Value::Long(0))) ); @@ -814,8 +823,8 @@ mod reader { #[tokio::test] async fn test_reader_iterator() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; - let reader = Reader::with_schema(&schema, ENCODED)?; + let schema = Schema::parse_str(SCHEMA).await?; + let reader = Reader::with_schema(&schema, ENCODED).await?; let mut record1 = Record::new(&schema).unwrap(); record1.put("a", 27i64); @@ -836,16 +845,16 @@ mod reader { #[tokio::test] async fn test_reader_invalid_header() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let invalid = ENCODED.iter().copied().skip(1).collect::>(); - assert!(Reader::with_schema(&schema, &invalid[..]).is_err()); + assert!(Reader::with_schema(&schema, &invalid[..]).await.is_err()); Ok(()) } #[tokio::test] async fn test_reader_invalid_block() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let invalid = ENCODED .iter() .copied() @@ -855,7 +864,7 @@ mod reader { .into_iter() .rev() .collect::>(); - let reader = Reader::with_schema(&schema, &invalid[..])?; + let reader = Reader::with_schema(&schema, &invalid[..]).await?; for value in reader { assert!(value.is_err()); } @@ -866,7 +875,7 @@ mod reader { #[tokio::test] async fn test_reader_empty_buffer() -> TestResult { let empty = Cursor::new(Vec::new()); - assert!(Reader::new(empty).is_err()); + assert!(Reader::new(empty).await.is_err()); Ok(()) } @@ -874,7 +883,7 @@ mod reader { #[tokio::test] async fn test_reader_only_header() -> TestResult { let invalid = ENCODED.iter().copied().take(165).collect::>(); - let reader = Reader::new(&invalid[..])?; + let reader = Reader::new(&invalid[..]).await?; for value in reader { assert!(value.is_err()); } @@ -884,9 +893,9 @@ mod reader { #[tokio::test] async fn test_avro_3405_read_user_metadata_success() -> TestResult { - use crate::writer::Writer; + use crate::writer::tokio::Writer; - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut user_meta_data: HashMap> = HashMap::new(); @@ -905,12 +914,12 @@ mod reader { record.put("a", 27i64); record.put("b", "foo"); - writer.append(record.clone())?; - writer.append(record.clone())?; + writer.append(record.clone()).await?; + writer.append(record.clone()).await?; writer.flush()?; let result = writer.into_inner()?; - let reader = Reader::new(&result[..])?; + let reader = Reader::new(&result[..]).await?; assert_eq!(reader.user_metadata(), &user_meta_data); Ok(()) @@ -923,8 +932,11 @@ mod reader { c: Vec, } + #[synca::cfg(tokio)] + use async_trait::async_trait; + #[cfg_attr(feature = "tokio", async_trait)] impl AvroSchema for TestSingleObjectReader { - fn get_schema() -> Schema { + async fn get_schema() -> Schema { let schema = r#" { "type":"record", @@ -948,7 +960,7 @@ mod reader { ] } "#; - Schema::parse_str(schema).unwrap() + Schema::parse_str(schema).await.unwrap() } } @@ -1007,21 +1019,23 @@ mod reader { to_read.extend_from_slice(&[0xC3, 0x01]); to_read.extend_from_slice( &TestSingleObjectReader::get_schema() + .await .fingerprint::() .bytes[..], ); encode( &obj.clone().into(), - &TestSingleObjectReader::get_schema(), + &TestSingleObjectReader::get_schema().await, &mut to_read, ) .expect("Encode should succeed"); let mut to_read = &to_read[..]; let generic_reader = - GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) .expect("Schema should resolve"); let val = generic_reader .read_value(&mut to_read) + .await .expect("Should read"); let expected_value: Value = obj.into(); assert_eq!(expected_value, val); @@ -1041,22 +1055,24 @@ mod reader { let mut to_read_2 = Vec::::new(); to_read_2.extend_from_slice( &TestSingleObjectReader::get_schema() + .await .fingerprint::() .bytes[..], ); let mut to_read_3 = Vec::::new(); encode( &obj.clone().into(), - &TestSingleObjectReader::get_schema(), + &TestSingleObjectReader::get_schema().await, &mut to_read_3, ) .expect("Encode should succeed"); let mut to_read = (&to_read_1[..]).chain(&to_read_2[..]).chain(&to_read_3[..]); let generic_reader = - GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) .expect("Schema should resolve"); let val = generic_reader .read_value(&mut to_read) + .await .expect("Should read"); let expected_value: Value = obj.into(); assert_eq!(expected_value, val); @@ -1076,19 +1092,21 @@ mod reader { to_read.extend_from_slice(&[0xC3, 0x01]); to_read.extend_from_slice( &TestSingleObjectReader::get_schema() + .await .fingerprint::() .bytes[..], ); encode( &obj.clone().into(), - &TestSingleObjectReader::get_schema(), + &TestSingleObjectReader::get_schema().await, &mut to_read, ) .expect("Encode should succeed"); let generic_reader = - GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) .expect("Schema should resolve"); let specific_reader = SpecificSingleObjectReader::::new() + .await .expect("schema should resolve"); let mut to_read1 = &to_read[..]; let mut to_read2 = &to_read[..]; @@ -1096,12 +1114,15 @@ mod reader { let val = generic_reader .read_value(&mut to_read1) + .await .expect("Should read"); let read_obj1 = specific_reader .read_from_value(&mut to_read2) + .await .expect("Should read from value"); let read_obj2 = specific_reader .read(&mut to_read3) + .await .expect("Should read from deserilize"); let expected_value: Value = obj.clone().into(); assert_eq!(obj, read_obj1); @@ -1116,7 +1137,7 @@ mod reader { let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); let generic_reader = GenericSingleObjectReader::new_with_header_builder( - TestSingleObjectReader::get_schema(), + TestSingleObjectReader::get_schema().await, header_builder, ) .expect("failed to build reader"); @@ -1126,6 +1147,7 @@ mod reader { let mut to_read = &data_to_read[..]; let read_result = generic_reader .read_value(&mut to_read) + .await .map_err(Error::into_details); matches!(read_result, Err(Details::ReadBytes(_))); Ok(()) @@ -1148,7 +1170,7 @@ mod reader { 235, 161, 167, 195, 177, ]; - if let Err(err) = Reader::new(snappy_compressed_avro.as_slice()) { + if let Err(err) = Reader::new(snappy_compressed_avro.as_slice()).await { assert_eq!("Codec 'snappy' is not supported/enabled", err.to_string()); } else { panic!("Expected an error in the reading of the codec!"); diff --git a/avro/src/schema.rs b/avro/src/schema.rs index b43410e7..1ebfce85 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -25,11 +25,14 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::de::tokio => crate::de::sync, crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, + crate::reader::tokio => crate::reader::sync, + crate::writer::tokio => crate::writer::sync, + crate::ser::tokio => crate::ser::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, crate::schema_equality::tokio => crate::schema_equality::sync, @@ -1887,8 +1890,13 @@ mod schema { let mut position = 0; for field in fields.iter() { if let Some(field) = field.as_object() { - let record_field = - Box::pin(RecordField::parse(field, position, self, &fully_qualified_name)).await?; + let record_field = Box::pin(RecordField::parse( + field, + position, + self, + &fully_qualified_name, + )) + .await?; record_fields.push(record_field); position += 1; } @@ -2548,8 +2556,11 @@ mod schema { /// through `derive` feature. Do not implement directly! /// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait /// through a blanket implementation. + #[synca::cfg(tokio)] + use async_trait::async_trait; + #[cfg_attr(feature = "tokio", async_trait)] pub trait AvroSchema { - fn get_schema() -> Schema; + async fn get_schema() -> Schema; } #[cfg(feature = "derive")] @@ -2741,6 +2752,15 @@ mod schema { #[cfg(test)] mod tests { use super::*; + use crate::{ + Uuid, + de::tokio::from_value, + error::tokio::Details, + rabin::Rabin, + reader::tokio::from_avro_datum, + ser::tokio::to_value, + writer::tokio::{SpecificSingleObjectWriter, Writer, to_avro_datum}, + }; use apache_avro_test_helper::{ TestResult, logger::{assert_logged, assert_not_logged}, @@ -2748,38 +2768,37 @@ mod schema { use serde_json::json; use serial_test::serial; use std::sync::atomic::Ordering; - use {SpecificSingleObjectWriter, error::tokio::Details, rabin::Rabin}; - #[test] - fn test_invalid_schema() { - assert!(Schema::parse_str("invalid").is_err()); + #[tokio::test] + async fn test_invalid_schema() { + assert!(Schema::parse_str("invalid").await.is_err()); } - #[test] - fn test_primitive_schema() -> TestResult { - assert_eq!(Schema::Null, Schema::parse_str("\"null\"")?); - assert_eq!(Schema::Int, Schema::parse_str("\"int\"")?); - assert_eq!(Schema::Double, Schema::parse_str("\"double\"")?); + #[tokio::test] + async fn test_primitive_schema() -> TestResult { + assert_eq!(Schema::Null, Schema::parse_str("\"null\"").await?); + assert_eq!(Schema::Int, Schema::parse_str("\"int\"").await?); + assert_eq!(Schema::Double, Schema::parse_str("\"double\"").await?); Ok(()) } - #[test] - fn test_array_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "array", "items": "string"}"#)?; + #[tokio::test] + async fn test_array_schema() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "array", "items": "string"}"#).await?; assert_eq!(Schema::array(Schema::String), schema); Ok(()) } - #[test] - fn test_map_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "map", "values": "double"}"#)?; + #[tokio::test] + async fn test_map_schema() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "map", "values": "double"}"#).await?; assert_eq!(Schema::map(Schema::Double), schema); Ok(()) } - #[test] - fn test_union_schema() -> TestResult { - let schema = Schema::parse_str(r#"["null", "int"]"#)?; + #[tokio::test] + async fn test_union_schema() -> TestResult { + let schema = Schema::parse_str(r#"["null", "int"]"#).await?; assert_eq!( Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), schema @@ -2787,15 +2806,15 @@ mod schema { Ok(()) } - #[test] - fn test_union_unsupported_schema() { - let schema = Schema::parse_str(r#"["null", ["null", "int"], "string"]"#); + #[tokio::test] + async fn test_union_unsupported_schema() { + let schema = Schema::parse_str(r#"["null", ["null", "int"], "string"]"#).await; assert!(schema.is_err()); } - #[test] - fn test_multi_union_schema() -> TestResult { - let schema = Schema::parse_str(r#"["null", "int", "float", "string", "bytes"]"#); + #[tokio::test] + async fn test_multi_union_schema() -> TestResult { + let schema = Schema::parse_str(r#"["null", "int", "float", "string", "bytes"]"#).await; assert!(schema.is_ok()); let schema = schema?; assert_eq!(SchemaKind::from(&schema), SchemaKind::Union); @@ -2824,8 +2843,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3621_nullable_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3621_nullable_record_field() -> TestResult { let nullable_record_field = RecordField::builder() .name("next".to_string()) .schema(Schema::Union(UnionSchema::new(vec![ @@ -2856,8 +2875,8 @@ mod schema { } // AVRO-3248 - #[test] - fn test_union_of_records() -> TestResult { + #[tokio::test] + async fn test_union_of_records() -> TestResult { use std::iter::FromIterator; // A and B are the same except the name. @@ -2916,8 +2935,8 @@ mod schema { Ok(()) } - #[test] - fn avro_rs_104_test_root_union_of_records() -> TestResult { + #[tokio::test] + async fn avro_rs_104_test_root_union_of_records() -> TestResult { // A and B are the same except the name. let schema_str_a = r#"{ "name": "A", @@ -2938,7 +2957,7 @@ mod schema { let schema_str_c = r#"["A", "B"]"#; let (schema_c, schemata) = - Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b])?; + Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]).await?; let schema_a_expected = Schema::Record(RecordSchema { name: Name::new("A")?, @@ -2992,8 +3011,8 @@ mod schema { Ok(()) } - #[test] - fn avro_rs_104_test_root_union_of_records_name_collision() -> TestResult { + #[tokio::test] + async fn avro_rs_104_test_root_union_of_records_name_collision() -> TestResult { // A and B are exactly the same. let schema_str_a1 = r#"{ "name": "A", @@ -3013,7 +3032,7 @@ mod schema { let schema_str_c = r#"["A", "A"]"#; - match Schema::parse_str_with_list(schema_str_c, [schema_str_a1, schema_str_a2]) { + match Schema::parse_str_with_list(schema_str_c, [schema_str_a1, schema_str_a2]).await { Ok(_) => unreachable!("Expected an error that the name is already defined"), Err(e) => assert_eq!( e.to_string(), @@ -3024,8 +3043,8 @@ mod schema { Ok(()) } - #[test] - fn avro_rs_104_test_root_union_of_records_no_name() -> TestResult { + #[tokio::test] + async fn avro_rs_104_test_root_union_of_records_no_name() -> TestResult { let schema_str_a = r#"{ "name": "A", "type": "record", @@ -3044,7 +3063,7 @@ mod schema { let schema_str_c = r#"["A", "A"]"#; - match Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]) { + match Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]).await { Ok(_) => { unreachable!("Expected an error that schema_str_b is missing a name field") } @@ -3054,8 +3073,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3584_test_recursion_records() -> TestResult { + #[tokio::test] + async fn avro_3584_test_recursion_records() -> TestResult { // A and B are the same except the name. let schema_str_a = r#"{ "name": "A", @@ -3088,8 +3107,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3248_nullable_record() -> TestResult { + #[tokio::test] + async fn test_avro_3248_nullable_record() -> TestResult { use std::iter::FromIterator; let schema_str_a = r#"{ @@ -3122,7 +3141,7 @@ mod schema { fields: vec![RecordField { name: "field_one".to_string(), doc: None, - default: Some(Value::Null), + default: Some(serde_json::Value::Null), aliases: None, schema: Schema::Union(UnionSchema::new(vec![ Schema::Null, @@ -3143,8 +3162,8 @@ mod schema { Ok(()) } - #[test] - fn test_record_schema() -> TestResult { + #[tokio::test] + async fn test_record_schema() -> TestResult { let parsed = Schema::parse_str( r#" { @@ -3156,7 +3175,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let mut lookup = BTreeMap::new(); lookup.insert("a".to_owned(), 0); @@ -3197,8 +3217,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3302_record_schema_with_currently_parsing_schema() -> TestResult { + #[tokio::test] + async fn test_avro_3302_record_schema_with_currently_parsing_schema() -> TestResult { let schema = Schema::parse_str( r#" { @@ -3217,7 +3237,8 @@ mod schema { }] } "#, - )?; + ) + .await?; let mut lookup = BTreeMap::new(); lookup.insert("recordField".to_owned(), 0); @@ -3283,8 +3304,8 @@ mod schema { } // https://github.com/flavray/avro-rs/pull/99#issuecomment-1016948451 - #[test] - fn test_parsing_of_recursive_type_enum() -> TestResult { + #[tokio::test] + async fn test_parsing_of_recursive_type_enum() -> TestResult { let schema = r#" { "type": "record", @@ -3328,7 +3349,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let schema_str = schema.canonical_form(); let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"gender","type":{"name":"office.Gender","type":"enum","symbols":["male","female"]}}]},{"name":"office.Manager","type":"record","fields":[{"name":"gender","type":"office.Gender"}]}]}]}"#; assert_eq!(schema_str, expected); @@ -3336,8 +3357,8 @@ mod schema { Ok(()) } - #[test] - fn test_parsing_of_recursive_type_fixed() -> TestResult { + #[tokio::test] + async fn test_parsing_of_recursive_type_fixed() -> TestResult { let schema = r#" { "type": "record", @@ -3378,7 +3399,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let schema_str = schema.canonical_form(); let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"id","type":{"name":"office.EmployeeId","type":"fixed","size":16}}]},{"name":"office.Manager","type":"record","fields":[{"name":"id","type":"office.EmployeeId"}]}]}]}"#; assert_eq!(schema_str, expected); @@ -3386,8 +3407,9 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3302_record_schema_with_currently_parsing_schema_aliases() -> TestResult { + #[tokio::test] + async fn test_avro_3302_record_schema_with_currently_parsing_schema_aliases() -> TestResult + { let schema = Schema::parse_str( r#" { @@ -3400,7 +3422,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let mut lookup = BTreeMap::new(); lookup.insert("value".to_owned(), 0); @@ -3455,8 +3478,9 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3370_record_schema_with_currently_parsing_schema_named_record() -> TestResult { + #[tokio::test] + async fn test_avro_3370_record_schema_with_currently_parsing_schema_named_record() + -> TestResult { let schema = Schema::parse_str( r#" { @@ -3468,7 +3492,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let mut lookup = BTreeMap::new(); lookup.insert("value".to_owned(), 0); @@ -3520,8 +3545,9 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3370_record_schema_with_currently_parsing_schema_named_enum() -> TestResult { + #[tokio::test] + async fn test_avro_3370_record_schema_with_currently_parsing_schema_named_enum() + -> TestResult { let schema = Schema::parse_str( r#" { @@ -3537,7 +3563,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let mut lookup = BTreeMap::new(); lookup.insert("enum".to_owned(), 0); @@ -3607,8 +3634,9 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3370_record_schema_with_currently_parsing_schema_named_fixed() -> TestResult { + #[tokio::test] + async fn test_avro_3370_record_schema_with_currently_parsing_schema_named_fixed() + -> TestResult { let schema = Schema::parse_str( r#" { @@ -3624,7 +3652,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let mut lookup = BTreeMap::new(); lookup.insert("fixed".to_owned(), 0); @@ -3691,11 +3720,11 @@ mod schema { Ok(()) } - #[test] - fn test_enum_schema() -> TestResult { + #[tokio::test] + async fn test_enum_schema() -> TestResult { let schema = Schema::parse_str( r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "hearts"]}"#, - )?; + ).await?; let expected = Schema::Enum(EnumSchema { name: Name::new("Suit")?, @@ -3716,31 +3745,33 @@ mod schema { Ok(()) } - #[test] - fn test_enum_schema_duplicate() -> TestResult { + #[tokio::test] + async fn test_enum_schema_duplicate() -> TestResult { // Duplicate "diamonds" let schema = Schema::parse_str( r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "diamonds"]}"#, - ); + ).await; assert!(schema.is_err()); Ok(()) } - #[test] - fn test_enum_schema_name() -> TestResult { + #[tokio::test] + async fn test_enum_schema_name() -> TestResult { // Invalid name "0000" does not match [A-Za-z_][A-Za-z0-9_]* let schema = Schema::parse_str( r#"{"type": "enum", "name": "Enum", "symbols": ["0000", "variant"]}"#, - ); + ) + .await; assert!(schema.is_err()); Ok(()) } - #[test] - fn test_fixed_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "fixed", "name": "test", "size": 16}"#)?; + #[tokio::test] + async fn test_fixed_schema() -> TestResult { + let schema = + Schema::parse_str(r#"{"type": "fixed", "name": "test", "size": 16}"#).await?; let expected = Schema::Fixed(FixedSchema { name: Name::new("test")?, @@ -3756,11 +3787,11 @@ mod schema { Ok(()) } - #[test] - fn test_fixed_schema_with_documentation() -> TestResult { + #[tokio::test] + async fn test_fixed_schema_with_documentation() -> TestResult { let schema = Schema::parse_str( r#"{"type": "fixed", "name": "test", "size": 16, "doc": "FixedSchema documentation"}"#, - )?; + ).await?; let expected = Schema::Fixed(FixedSchema { name: Name::new("test")?, @@ -3776,11 +3807,12 @@ mod schema { Ok(()) } - #[test] - fn test_no_documentation() -> TestResult { + #[tokio::test] + async fn test_no_documentation() -> TestResult { let schema = Schema::parse_str( r#"{"type": "enum", "name": "Coin", "symbols": ["heads", "tails"]}"#, - )?; + ) + .await?; let doc = match schema { Schema::Enum(EnumSchema { doc, .. }) => doc, @@ -3792,11 +3824,11 @@ mod schema { Ok(()) } - #[test] - fn test_documentation() -> TestResult { + #[tokio::test] + async fn test_documentation() -> TestResult { let schema = Schema::parse_str( r#"{"type": "enum", "name": "Coin", "doc": "Some documentation", "symbols": ["heads", "tails"]}"#, - )?; + ).await?; let doc = match schema { Schema::Enum(EnumSchema { doc, .. }) => doc, @@ -3810,16 +3842,16 @@ mod schema { // Tests to ensure Schema is Send + Sync. These tests don't need to _do_ anything, if they can // compile, they pass. - #[test] - fn test_schema_is_send() { + #[tokio::test] + async fn test_schema_is_send() { fn send(_s: S) {} let schema = Schema::Null; send(schema); } - #[test] - fn test_schema_is_sync() { + #[tokio::test] + async fn test_schema_is_sync() { fn sync(_s: S) {} let schema = Schema::Null; @@ -3827,10 +3859,10 @@ mod schema { sync(schema); } - #[test] - fn test_schema_fingerprint() -> TestResult { + #[tokio::test] + async fn test_schema_fingerprint() -> TestResult { + use crate::rabin::Rabin; use md5::Md5; - use rabin::Rabin; use sha2::Sha256; let raw_schema = r#" @@ -3845,7 +3877,7 @@ mod schema { } "#; - let schema = Schema::parse_str(raw_schema)?; + let schema = Schema::parse_str(raw_schema).await?; assert_eq!( "7eb3b28d73dfc99bdd9af1848298b40804a2f8ad5d2642be2ecc2ad34842b987", format!("{}", schema.fingerprint::()) @@ -3863,23 +3895,24 @@ mod schema { Ok(()) } - #[test] - fn test_logical_types() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "int", "logicalType": "date"}"#)?; + #[tokio::test] + async fn test_logical_types() -> TestResult { + let schema = Schema::parse_str(r#"{"type": "int", "logicalType": "date"}"#).await?; assert_eq!(schema, Schema::Date); let schema = - Schema::parse_str(r#"{"type": "long", "logicalType": "timestamp-micros"}"#)?; + Schema::parse_str(r#"{"type": "long", "logicalType": "timestamp-micros"}"#).await?; assert_eq!(schema, Schema::TimestampMicros); Ok(()) } - #[test] - fn test_nullable_logical_type() -> TestResult { + #[tokio::test] + async fn test_nullable_logical_type() -> TestResult { let schema = Schema::parse_str( r#"{"type": ["null", {"type": "long", "logicalType": "timestamp-micros"}]}"#, - )?; + ) + .await?; assert_eq!( schema, Schema::Union(UnionSchema::new(vec![ @@ -3912,8 +3945,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3374_preserve_namespace_for_primitive() -> TestResult { + #[tokio::test] + async fn test_avro_3374_preserve_namespace_for_primitive() -> TestResult { let schema = Schema::parse_str( r#" { @@ -3925,7 +3958,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let json = schema.canonical_form(); assert_eq!( @@ -3936,8 +3970,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3433_preserve_schema_refs_in_json() -> TestResult { + #[tokio::test] + async fn test_avro_3433_preserve_schema_refs_in_json() -> TestResult { let schema = r#" { "name": "test.test", @@ -3952,7 +3986,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let expected = r#"{"name":"test.test","type":"record","fields":[{"name":"bar","type":{"name":"test.foo","type":"record","fields":[{"name":"id","type":"long"}]}},{"name":"baz","type":"test.foo"}]}"#; assert_eq!(schema.canonical_form(), expected); @@ -3960,8 +3994,8 @@ mod schema { Ok(()) } - #[test] - fn test_read_namespace_from_name() -> TestResult { + #[tokio::test] + async fn test_read_namespace_from_name() -> TestResult { let schema = r#" { "name": "space.name", @@ -3975,7 +4009,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; if let Schema::Record(RecordSchema { name, .. }) = schema { assert_eq!(name.name, "name"); assert_eq!(name.namespace, Some("space".to_string())); @@ -3986,8 +4020,8 @@ mod schema { Ok(()) } - #[test] - fn test_namespace_from_name_has_priority_over_from_field() -> TestResult { + #[tokio::test] + async fn test_namespace_from_name_has_priority_over_from_field() -> TestResult { let schema = r#" { "name": "space1.name", @@ -4002,7 +4036,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; if let Schema::Record(RecordSchema { name, .. }) = schema { assert_eq!(name.namespace, Some("space1".to_string())); } else { @@ -4012,8 +4046,8 @@ mod schema { Ok(()) } - #[test] - fn test_namespace_from_field() -> TestResult { + #[tokio::test] + async fn test_namespace_from_field() -> TestResult { let schema = r#" { "name": "name", @@ -4028,7 +4062,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; if let Schema::Record(RecordSchema { name, .. }) = schema { assert_eq!(name.namespace, Some("space2".to_string())); } else { @@ -4038,9 +4072,9 @@ mod schema { Ok(()) } - #[test] + #[tokio::test] /// Zero-length namespace is considered as no-namespace. - fn test_namespace_from_name_with_empty_value() -> TestResult { + async fn test_namespace_from_name_with_empty_value() -> TestResult { let name = Name::new(".name")?; assert_eq!(name.name, "name"); assert_eq!(name.namespace, None); @@ -4048,26 +4082,26 @@ mod schema { Ok(()) } - #[test] + #[tokio::test] /// Whitespace is not allowed in the name. - fn test_name_with_whitespace_value() { + async fn test_name_with_whitespace_value() { match Name::new(" ").map_err(Error::into_details) { Err(Details::InvalidSchemaName(_, _)) => {} _ => panic!("Expected an Details::InvalidSchemaName!"), } } - #[test] + #[tokio::test] /// The name must be non-empty. - fn test_name_with_no_name_part() { + async fn test_name_with_no_name_part() { match Name::new("space.").map_err(Error::into_details) { Err(Details::InvalidSchemaName(_, _)) => {} _ => panic!("Expected an Details::InvalidSchemaName!"), } } - #[test] - fn avro_3448_test_proper_resolution_inner_record_inherited_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_record_inherited_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4097,7 +4131,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_record_name"] { @@ -4107,8 +4141,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_record_qualified_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_record_qualified_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4138,7 +4172,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_record_name"] { @@ -4148,8 +4182,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_enum_inherited_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_enum_inherited_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4174,7 +4208,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_enum_name"] { @@ -4184,8 +4218,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_enum_qualified_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_enum_qualified_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4210,7 +4244,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_enum_name"] { @@ -4220,8 +4254,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_fixed_inherited_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_fixed_inherited_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4246,7 +4280,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_fixed_name"] { @@ -4256,8 +4290,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_fixed_qualified_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_fixed_qualified_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4282,7 +4316,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_fixed_name"] { @@ -4292,8 +4326,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_record_inner_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_record_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4324,7 +4358,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "inner_space.inner_record_name"] { @@ -4334,8 +4368,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_enum_inner_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_enum_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4361,7 +4395,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "inner_space.inner_enum_name"] { @@ -4371,8 +4405,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_resolution_inner_fixed_inner_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_resolution_inner_fixed_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4398,7 +4432,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "inner_space.inner_fixed_name"] { @@ -4408,9 +4442,9 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_multi_level_resolution_inner_record_outer_namespace() -> TestResult - { + #[tokio::test] + async fn avro_3448_test_proper_multi_level_resolution_inner_record_outer_namespace() + -> TestResult { let schema = r#" { "name": "record_name", @@ -4452,7 +4486,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 3); for s in &[ @@ -4466,9 +4500,9 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_multi_level_resolution_inner_record_middle_namespace() -> TestResult - { + #[tokio::test] + async fn avro_3448_test_proper_multi_level_resolution_inner_record_middle_namespace() + -> TestResult { let schema = r#" { "name": "record_name", @@ -4511,7 +4545,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 3); for s in &[ @@ -4525,9 +4559,9 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_multi_level_resolution_inner_record_inner_namespace() -> TestResult - { + #[tokio::test] + async fn avro_3448_test_proper_multi_level_resolution_inner_record_inner_namespace() + -> TestResult { let schema = r#" { "name": "record_name", @@ -4571,7 +4605,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 3); for s in &[ @@ -4585,8 +4619,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_in_array_resolution_inherited_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_in_array_resolution_inherited_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4616,7 +4650,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.in_array_record"] { @@ -4626,8 +4660,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3448_test_proper_in_map_resolution_inherited_namespace() -> TestResult { + #[tokio::test] + async fn avro_3448_test_proper_in_map_resolution_inherited_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4657,7 +4691,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.in_map_record"] { @@ -4667,8 +4701,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3466_test_to_json_inner_enum_inner_namespace() -> TestResult { + #[tokio::test] + async fn avro_3466_test_to_json_inner_enum_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4694,7 +4728,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); // confirm we have expected 2 full-names @@ -4705,14 +4739,14 @@ mod schema { // convert Schema back to JSON string let schema_str = serde_json::to_string(&schema).expect("test failed"); - let _schema = Schema::parse_str(&schema_str).expect("test failed"); + let _schema = Schema::parse_str(&schema_str).await.expect("test failed"); assert_eq!(schema, _schema); Ok(()) } - #[test] - fn avro_3466_test_to_json_inner_fixed_inner_namespace() -> TestResult { + #[tokio::test] + async fn avro_3466_test_to_json_inner_fixed_inner_namespace() -> TestResult { let schema = r#" { "name": "record_name", @@ -4738,7 +4772,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); // confirm we have expected 2 full-names @@ -4749,7 +4783,7 @@ mod schema { // convert Schema back to JSON string let schema_str = serde_json::to_string(&schema).expect("test failed"); - let _schema = Schema::parse_str(&schema_str).expect("test failed"); + let _schema = Schema::parse_str(&schema_str).await.expect("test failed"); assert_eq!(schema, _schema); Ok(()) @@ -4769,8 +4803,8 @@ mod schema { } } - #[test] - fn avro_3512_alias_with_null_namespace_record() -> TestResult { + #[tokio::test] + async fn avro_3512_alias_with_null_namespace_record() -> TestResult { let schema = Schema::parse_str( r#" { @@ -4783,7 +4817,8 @@ mod schema { ] } "#, - )?; + ) + .await?; if let Schema::Record(RecordSchema { ref aliases, .. }) = schema { assert_avro_3512_aliases(aliases); @@ -4794,8 +4829,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3512_alias_with_null_namespace_enum() -> TestResult { + #[tokio::test] + async fn avro_3512_alias_with_null_namespace_enum() -> TestResult { let schema = Schema::parse_str( r#" { @@ -4808,7 +4843,8 @@ mod schema { ] } "#, - )?; + ) + .await?; if let Schema::Enum(EnumSchema { ref aliases, .. }) = schema { assert_avro_3512_aliases(aliases); @@ -4819,8 +4855,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3512_alias_with_null_namespace_fixed() -> TestResult { + #[tokio::test] + async fn avro_3512_alias_with_null_namespace_fixed() -> TestResult { let schema = Schema::parse_str( r#" { @@ -4831,7 +4867,8 @@ mod schema { "size" : 12 } "#, - )?; + ) + .await?; if let Schema::Fixed(FixedSchema { ref aliases, .. }) = schema { assert_avro_3512_aliases(aliases); @@ -4842,8 +4879,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3518_serialize_aliases_record() -> TestResult { + #[tokio::test] + async fn avro_3518_serialize_aliases_record() -> TestResult { let schema = Schema::parse_str( r#" { @@ -4862,7 +4899,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let value = serde_json::to_value(&schema)?; let serialized = serde_json::to_string(&value)?; @@ -4870,13 +4908,13 @@ mod schema { r#"{"aliases":["space.b","x.y","c"],"fields":[{"aliases":["time1","ns.time2"],"default":123,"name":"time","type":"long"}],"name":"a","namespace":"space","type":"record"}"#, &serialized ); - assert_eq!(schema, Schema::parse_str(&serialized)?); + assert_eq!(schema, Schema::parse_str(&serialized).await?); Ok(()) } - #[test] - fn avro_3518_serialize_aliases_enum() -> TestResult { + #[tokio::test] + async fn avro_3518_serialize_aliases_enum() -> TestResult { let schema = Schema::parse_str( r#" { @@ -4889,7 +4927,8 @@ mod schema { ] } "#, - )?; + ) + .await?; let value = serde_json::to_value(&schema)?; let serialized = serde_json::to_string(&value)?; @@ -4897,13 +4936,13 @@ mod schema { r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","symbols":["symbol1","symbol2"],"type":"enum"}"#, &serialized ); - assert_eq!(schema, Schema::parse_str(&serialized)?); + assert_eq!(schema, Schema::parse_str(&serialized).await?); Ok(()) } - #[test] - fn avro_3518_serialize_aliases_fixed() -> TestResult { + #[tokio::test] + async fn avro_3518_serialize_aliases_fixed() -> TestResult { let schema = Schema::parse_str( r#" { @@ -4914,7 +4953,8 @@ mod schema { "size" : 12 } "#, - )?; + ) + .await?; let value = serde_json::to_value(&schema)?; let serialized = serde_json::to_string(&value)?; @@ -4922,13 +4962,13 @@ mod schema { r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","size":12,"type":"fixed"}"#, &serialized ); - assert_eq!(schema, Schema::parse_str(&serialized)?); + assert_eq!(schema, Schema::parse_str(&serialized).await?); Ok(()) } - #[test] - fn avro_3130_parse_anonymous_union_type() -> TestResult { + #[tokio::test] + async fn avro_3130_parse_anonymous_union_type() -> TestResult { let schema_str = r#" { "type": "record", @@ -4948,7 +4988,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; if let Schema::Record(RecordSchema { name, fields, .. }) = schema { assert_eq!(name, Name::new("AccountEvent")?); @@ -4978,8 +5018,8 @@ mod schema { Ok(()) } - #[test] - fn avro_custom_attributes_schema_without_attributes() -> TestResult { + #[tokio::test] + async fn avro_custom_attributes_schema_without_attributes() -> TestResult { let schemata_str = [ r#" { @@ -5007,7 +5047,7 @@ mod schema { "#, ]; for schema_str in schemata_str.iter() { - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; assert_eq!(schema.custom_attributes(), Some(&Default::default())); } @@ -5024,8 +5064,8 @@ mod schema { } "#; - #[test] - fn avro_3609_custom_attributes_schema_with_attributes() -> TestResult { + #[tokio::test] + async fn avro_3609_custom_attributes_schema_with_attributes() -> TestResult { let schemata_str = [ r#" { @@ -5065,7 +5105,8 @@ mod schema { .to_owned() .replace("{{{}}}", CUSTOM_ATTRS_SUFFIX) .as_str(), - )?; + ) + .await?; assert_eq!( schema.custom_attributes(), @@ -5076,24 +5117,29 @@ mod schema { Ok(()) } - fn expected_custom_attributes() -> BTreeMap { - let mut expected_attributes: BTreeMap = Default::default(); - expected_attributes - .insert("string_key".to_string(), Value::String("value".to_string())); + fn expected_custom_attributes() -> BTreeMap { + let mut expected_attributes: BTreeMap = Default::default(); + expected_attributes.insert( + "string_key".to_string(), + serde_json::Value::String("value".to_string()), + ); expected_attributes.insert("number_key".to_string(), json!(1.23)); - expected_attributes.insert("null_key".to_string(), Value::Null); + expected_attributes.insert("null_key".to_string(), serde_json::Value::Null); expected_attributes.insert( "array_key".to_string(), - Value::Array(vec![json!(1), json!(2), json!(3)]), + serde_json::Value::Array(vec![json!(1), json!(2), json!(3)]), + ); + let mut object_value: HashMap = HashMap::new(); + object_value.insert( + "key".to_string(), + serde_json::Value::String("value".to_string()), ); - let mut object_value: HashMap = HashMap::new(); - object_value.insert("key".to_string(), Value::String("value".to_string())); expected_attributes.insert("object_key".to_string(), json!(object_value)); expected_attributes } - #[test] - fn avro_3609_custom_attributes_record_field_without_attributes() -> TestResult { + #[tokio::test] + async fn avro_3609_custom_attributes_record_field_without_attributes() -> TestResult { let schema_str = String::from( r#" { @@ -5112,7 +5158,8 @@ mod schema { ); let schema = - Schema::parse_str(schema_str.replace("{{{}}}", CUSTOM_ATTRS_SUFFIX).as_str())?; + Schema::parse_str(schema_str.replace("{{{}}}", CUSTOM_ATTRS_SUFFIX).as_str()) + .await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5128,8 +5175,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3625_null_is_first() -> TestResult { + #[tokio::test] + async fn avro_3625_null_is_first() -> TestResult { let schema_str = String::from( r#" { @@ -5142,7 +5189,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5150,7 +5197,7 @@ mod schema { assert_eq!(fields.len(), 1); let field = &fields[0]; assert_eq!(&field.name, "a"); - assert_eq!(&field.default, &Some(Value::Null)); + assert_eq!(&field.default, &Some(serde_json::Value::Null)); match &field.schema { Schema::Union(union) => { assert_eq!(union.variants().len(), 2); @@ -5167,8 +5214,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3625_null_is_last() -> TestResult { + #[tokio::test] + async fn avro_3625_null_is_last() -> TestResult { let schema_str = String::from( r#" { @@ -5181,7 +5228,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5205,8 +5252,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3625_null_is_the_middle() -> TestResult { + #[tokio::test] + async fn avro_3625_null_is_the_middle() -> TestResult { let schema_str = String::from( r#" { @@ -5219,7 +5266,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5244,8 +5291,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3649_default_notintfirst() -> TestResult { + #[tokio::test] + async fn avro_3649_default_notintfirst() -> TestResult { let schema_str = String::from( r#" { @@ -5258,7 +5305,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5282,8 +5329,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3709_parsing_of_record_field_aliases() -> TestResult { + #[tokio::test] + async fn avro_3709_parsing_of_record_field_aliases() -> TestResult { let schema = r#" { "name": "rec", @@ -5298,7 +5345,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; if let Schema::Record(RecordSchema { fields, .. }) = schema { let num_field = &fields[0]; assert_eq!(num_field.name, "num"); @@ -5310,8 +5357,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3735_parse_enum_namespace() -> TestResult { + #[tokio::test] + async fn avro_3735_parse_enum_namespace() -> TestResult { let schema = r#" { "type": "record", @@ -5366,7 +5413,7 @@ mod schema { pub bar_use: Bar, } - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let foo = Foo { bar_init: Bar::Bar0, @@ -5374,18 +5421,18 @@ mod schema { }; let avro_value = to_value(foo)?; - assert!(avro_value.validate(&schema)); + assert!(avro_value.validate(&schema).await); let mut writer = Writer::new(&schema, Vec::new()); // schema validation happens here - writer.append(avro_value)?; + writer.append(avro_value).await?; Ok(()) } - #[test] - fn avro_3755_deserialize() -> TestResult { + #[tokio::test] + async fn avro_3755_deserialize() -> TestResult { #[derive( Debug, PartialEq, @@ -5466,20 +5513,20 @@ mod schema { ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = Schema::parse_str(writer_schema).await?; let foo = Foo { bar_init: Bar::Bar0, bar_use: Bar::Bar1, }; let avro_value = to_value(foo)?; assert!( - avro_value.validate(&writer_schema), + avro_value.validate(&writer_schema).await, "value is valid for schema", ); - let datum = to_avro_datum(&writer_schema, avro_value)?; + let datum = to_avro_datum(&writer_schema, avro_value).await?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; - let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; + let reader_schema = Schema::parse_str(reader_schema).await?; + let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema)).await?; match deser_value { Value::Record(fields) => { assert_eq!(fields.len(), 2); @@ -5494,8 +5541,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3780_decimal_schema_type_with_fixed() -> TestResult { + #[tokio::test] + async fn test_avro_3780_decimal_schema_type_with_fixed() -> TestResult { let schema = json!( { "type": "record", @@ -5512,7 +5559,7 @@ mod schema { ] }); - let parse_result = Schema::parse(&schema); + let parse_result = Schema::parse(&schema).await; assert!( parse_result.is_ok(), "parse result must be ok, got: {parse_result:?}" @@ -5521,8 +5568,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3772_enum_default_wrong_type() -> TestResult { + #[tokio::test] + async fn test_avro_3772_enum_default_wrong_type() -> TestResult { let schema = r#" { "type": "record", @@ -5543,7 +5590,7 @@ mod schema { } "#; - match Schema::parse_str(schema) { + match Schema::parse_str(schema).await { Err(err) => { assert_eq!( err.to_string(), @@ -5555,8 +5602,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3812_handle_null_namespace_properly() -> TestResult { + #[tokio::test] + async fn test_avro_3812_handle_null_namespace_properly() -> TestResult { let schema_str = r#" { "namespace": "", @@ -5585,7 +5632,7 @@ mod schema { "#; let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"a","type":{"name":"my_enum","type":"enum","symbols":["a","b"]}},{"name":"b","type":{"name":"my_fixed","type":"fixed","size":10}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5598,8 +5645,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3818_inherit_enclosing_namespace() -> TestResult { + #[tokio::test] + async fn test_avro_3818_inherit_enclosing_namespace() -> TestResult { // Enclosing namespace is specified but inner namespaces are not. let schema_str = r#" { @@ -5627,7 +5674,7 @@ mod schema { "#; let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"my_ns.fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5661,7 +5708,7 @@ mod schema { "#; let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5693,7 +5740,7 @@ mod schema { "#; let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"f1.ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"f2.ns.fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5742,15 +5789,15 @@ mod schema { "#; let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.inner_record1","type":"record","fields":[{"name":"f1_1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}}]}},{"name":"f2","type":{"name":"inner_ns.inner_record2","type":"record","fields":[{"name":"f2_1","type":{"name":"inner_ns.enum2","type":"enum","symbols":["a"]}}]}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); Ok(()) } - #[test] - fn test_avro_3779_bigdecimal_schema() -> TestResult { + #[tokio::test] + async fn test_avro_3779_bigdecimal_schema() -> TestResult { let schema = json!( { "name": "decimal", @@ -5759,7 +5806,7 @@ mod schema { } ); - let parse_result = Schema::parse(&schema); + let parse_result = Schema::parse(&schema).await; assert!( parse_result.is_ok(), "parse result must be ok, got: {parse_result:?}" @@ -5772,8 +5819,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3820_deny_invalid_field_names() -> TestResult { + #[tokio::test] + async fn test_avro_3820_deny_invalid_field_names() -> TestResult { let schema_str = r#" { "name": "my_record", @@ -5798,14 +5845,17 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::FieldName(x)) if x == "f1.x" => Ok(()), other => Err(format!("Expected Details::FieldName, got {other:?}").into()), } } - #[test] - fn test_avro_3827_disallow_duplicate_field_names() -> TestResult { + #[tokio::test] + async fn test_avro_3827_disallow_duplicate_field_names() -> TestResult { let schema_str = r#" { "name": "my_schema", @@ -5830,7 +5880,10 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::FieldNameDuplicate(_)) => (), other => { return Err( @@ -5866,15 +5919,15 @@ mod schema { "#; let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"a","type":"record","fields":[{"name":"f1","type":{"name":"b","type":"record","fields":[]}}]}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); Ok(()) } - #[test] - fn test_avro_3830_null_namespace_in_fully_qualified_names() -> TestResult { + #[tokio::test] + async fn test_avro_3830_null_namespace_in_fully_qualified_names() -> TestResult { // Check whether all the named types don't refer to the namespace field // if their name starts with a dot. let schema_str = r#" @@ -5905,7 +5958,7 @@ mod schema { "#; let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5936,7 +5989,7 @@ mod schema { "#; let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5949,8 +6002,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3814_schema_resolution_failure() -> TestResult { + #[tokio::test] + async fn test_avro_3814_schema_resolution_failure() -> TestResult { // Define a reader schema: a nested record with an optional field. let reader_schema = json!( { @@ -6051,21 +6104,21 @@ mod schema { }; // Serialize using the writer schema. - let writer_schema = Schema::parse(&writer_schema)?; + let writer_schema = Schema::parse(&writer_schema).await?; let avro_value = to_value(s)?; assert!( - avro_value.validate(&writer_schema), + avro_value.validate(&writer_schema).await, "value is valid for schema", ); - let datum = to_avro_datum(&writer_schema, avro_value)?; + let datum = to_avro_datum(&writer_schema, avro_value).await?; // Now, attempt to deserialize using the reader schema. - let reader_schema = Schema::parse(&reader_schema)?; + let reader_schema = Schema::parse(&reader_schema).await?; let mut x = &datum[..]; // Deserialization should succeed and we should be able to resolve the schema. - let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; - assert!(deser_value.validate(&reader_schema)); + let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema)).await?; + assert!(deser_value.validate(&reader_schema).await); // Verify that we can read a field from the record. let d: MyRecordReader = from_value(&deser_value)?; @@ -6073,8 +6126,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3837_disallow_invalid_namespace() -> TestResult { + #[tokio::test] + async fn test_avro_3837_disallow_invalid_namespace() -> TestResult { // Valid namespace #1 (Single name portion) let schema_str = r#" { @@ -6086,7 +6139,7 @@ mod schema { "#; let expected = r#"{"name":"ns1.record1","type":"record","fields":[]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -6101,7 +6154,7 @@ mod schema { "#; let expected = r#"{"name":"ns1.foo.bar.enum1","type":"enum","symbols":["a"]}"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -6115,7 +6168,10 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::InvalidNamespace(_, _)) => (), other => { return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); @@ -6132,7 +6188,10 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::InvalidNamespace(_, _)) => (), other => { return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); @@ -6149,7 +6208,10 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::InvalidNamespace(_, _)) => (), other => { return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); @@ -6166,7 +6228,10 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::InvalidNamespace(_, _)) => (), other => { return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); @@ -6183,7 +6248,10 @@ mod schema { } "#; - match Schema::parse_str(schema_str).map_err(Error::into_details) { + match Schema::parse_str(schema_str) + .await + .map_err(Error::into_details) + { Err(Details::InvalidNamespace(_, _)) => (), other => { return Err(format!("Expected Details::InvalidNamespace, got {other:?}").into()); @@ -6193,8 +6261,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_simple_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_simple_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6215,7 +6283,7 @@ mod schema { r#""int""#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6226,8 +6294,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_nested_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_nested_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6258,7 +6326,7 @@ mod schema { .to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6269,8 +6337,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_enum_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_enum_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6295,7 +6363,7 @@ mod schema { r#"{"name":"ns.enum1","type":"enum","symbols":["a","b","c"]}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6306,8 +6374,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_fixed_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_fixed_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6332,7 +6400,7 @@ mod schema { r#"{"name":"ns.fixed1","type":"fixed","size":3}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6343,8 +6411,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_array_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_array_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6366,7 +6434,7 @@ mod schema { r#"{"type":"array","items":"int"}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6377,8 +6445,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_map_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_map_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6400,7 +6468,7 @@ mod schema { r#"{"type":"map","values":"string"}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6411,8 +6479,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_ref_record_field() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_ref_record_field() -> TestResult { let schema_str = r#" { "name": "record1", @@ -6445,7 +6513,7 @@ mod schema { r#""ns.record2""#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6456,8 +6524,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3851_validate_default_value_of_enum() -> TestResult { + #[tokio::test] + async fn test_avro_3851_validate_default_value_of_enum() -> TestResult { let schema_str = r#" { "name": "enum1", @@ -6468,7 +6536,7 @@ mod schema { } "#; let expected = Details::EnumDefaultWrongType(100.into()).to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6490,7 +6558,7 @@ mod schema { symbols: vec!["a".to_string(), "b".to_string(), "c".to_string()], } .to_string(); - let result = Schema::parse_str(schema_str); + let result = Schema::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6501,8 +6569,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3862_get_aliases() -> TestResult { + #[tokio::test] + async fn test_avro_3862_get_aliases() -> TestResult { // Test for Record let schema_str = r#" { @@ -6516,7 +6584,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let expected = vec![Alias::new("ns1.r1")?, Alias::new("ns2.r2")?]; match schema.aliases() { Some(aliases) => assert_eq!(aliases, &expected), @@ -6534,7 +6602,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match schema.aliases() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6550,7 +6618,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let expected = vec![Alias::new("ns1.en1")?, Alias::new("ns2.en2")?]; match schema.aliases() { Some(aliases) => assert_eq!(aliases, &expected), @@ -6565,7 +6633,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match schema.aliases() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6581,7 +6649,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let expected = vec![Alias::new("ns1.fx1")?, Alias::new("ns2.fx2")?]; match schema.aliases() { Some(aliases) => assert_eq!(aliases, &expected), @@ -6596,7 +6664,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match schema.aliases() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6612,8 +6680,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3862_get_doc() -> TestResult { + #[tokio::test] + async fn test_avro_3862_get_doc() -> TestResult { // Test for Record let schema_str = r#" { @@ -6626,7 +6694,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let expected = "Record Document"; match schema.doc() { Some(doc) => assert_eq!(doc, expected), @@ -6643,7 +6711,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match schema.doc() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6658,7 +6726,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let expected = "Enum Document"; match schema.doc() { Some(doc) => assert_eq!(doc, expected), @@ -6672,7 +6740,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match schema.doc() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6687,7 +6755,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let expected = "Fixed Document"; match schema.doc() { Some(doc) => assert_eq!(doc, expected), @@ -6701,7 +6769,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match schema.doc() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6717,17 +6785,20 @@ mod schema { Ok(()) } - #[test] - fn avro_3886_serialize_attributes() -> TestResult { + #[tokio::test] + async fn avro_3886_serialize_attributes() -> TestResult { let attributes = BTreeMap::from([ ("string_key".into(), "value".into()), ("number_key".into(), 1.23.into()), - ("null_key".into(), Value::Null), + ("null_key".into(), serde_json::Value::Null), ( "array_key".into(), - Value::Array(vec![1.into(), 2.into(), 3.into()]), + serde_json::Value::Array(vec![1.into(), 2.into(), 3.into()]), + ), + ( + "object_key".into(), + serde_json::Value::Object(serde_json::Map::default()), ), - ("object_key".into(), Value::Object(Map::default())), ]); // Test serialize enum attributes @@ -6780,8 +6851,8 @@ mod schema { /// A test cases showing that names and namespaces can be constructed /// entirely by underscores. - #[test] - fn test_avro_3897_funny_valid_names_and_namespaces() -> TestResult { + #[tokio::test] + async fn test_avro_3897_funny_valid_names_and_namespaces() -> TestResult { for funny_name in ["_", "_._", "__._", "_.__", "_._._"] { let name = Name::new(funny_name); assert!(name.is_ok()); @@ -6789,8 +6860,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3896_decimal_schema() -> TestResult { + #[tokio::test] + async fn test_avro_3896_decimal_schema() -> TestResult { // bytes decimal, represented as native logical type. let schema = json!( { @@ -6801,7 +6872,7 @@ mod schema { "precision": 9, "scale": 2 }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert!(matches!( parse_result, Schema::Decimal(DecimalSchema { @@ -6818,15 +6889,15 @@ mod schema { "name": "LongDecimal", "logicalType": "decimal" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; // assert!(matches!(parse_result, Schema::Long)); assert_eq!(parse_result, Schema::Long); Ok(()) } - #[test] - fn avro_3896_uuid_schema_for_string() -> TestResult { + #[tokio::test] + async fn avro_3896_uuid_schema_for_string() -> TestResult { // string uuid, represents as native logical type. let schema = json!( { @@ -6834,22 +6905,22 @@ mod schema { "name": "StringUUID", "logicalType": "uuid" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert_eq!(parse_result, Schema::Uuid); Ok(()) } - #[test] + #[tokio::test] #[serial(serde_is_human_readable)] - fn avro_rs_53_uuid_with_fixed() -> TestResult { + async fn avro_rs_53_uuid_with_fixed() -> TestResult { #[derive(Debug, Serialize, Deserialize)] struct Comment { id: Uuid, } impl AvroSchema for Comment { - fn get_schema() -> Schema { + async fn get_schema() -> Schema { Schema::parse_str( r#"{ "type" : "record", @@ -6865,6 +6936,7 @@ mod schema { } ] }"#, ) + .await .expect("Invalid Comment Avro schema") } } @@ -6875,22 +6947,24 @@ mod schema { let mut buffer = Vec::new(); // serialize the Uuid as String - util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let bytes = SpecificSingleObjectWriter::::with_capacity(64)? + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let bytes = SpecificSingleObjectWriter::::with_capacity(64) + .await? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 47); // serialize the Uuid as Bytes - util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); - let bytes = SpecificSingleObjectWriter::::with_capacity(64)? + crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); + let bytes = SpecificSingleObjectWriter::::with_capacity(64) + .await? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 27); Ok(()) } - #[test] - fn avro_3926_uuid_schema_for_fixed_with_size_16() -> TestResult { + #[tokio::test] + async fn avro_3926_uuid_schema_for_fixed_with_size_16() -> TestResult { let schema = json!( { "type": "fixed", @@ -6898,7 +6972,7 @@ mod schema { "size": 16, "logicalType": "uuid" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert_eq!(parse_result, Schema::Uuid); assert_not_logged( r#"Ignoring uuid logical type for a Fixed schema because its size (6) is not 16! Schema: Fixed(FixedSchema { name: Name { name: "FixedUUID", namespace: None }, aliases: None, doc: None, size: 6, attributes: {"logicalType": String("uuid")} })"#, @@ -6907,8 +6981,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3926_uuid_schema_for_fixed_with_size_different_than_16() -> TestResult { + #[tokio::test] + async fn avro_3926_uuid_schema_for_fixed_with_size_different_than_16() -> TestResult { let schema = json!( { "type": "fixed", @@ -6916,7 +6990,7 @@ mod schema { "size": 6, "logicalType": "uuid" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert_eq!( parse_result, @@ -6936,8 +7010,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3896_timestamp_millis_schema() -> TestResult { + #[tokio::test] + async fn test_avro_3896_timestamp_millis_schema() -> TestResult { // long timestamp-millis, represents as native logical type. let schema = json!( { @@ -6945,7 +7019,7 @@ mod schema { "name": "LongTimestampMillis", "logicalType": "timestamp-millis" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert_eq!(parse_result, Schema::TimestampMillis); // int timestamp-millis, represents as native complex type. @@ -6955,14 +7029,14 @@ mod schema { "name": "IntTimestampMillis", "logicalType": "timestamp-millis" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert_eq!(parse_result, Schema::Int); Ok(()) } - #[test] - fn test_avro_3896_custom_bytes_schema() -> TestResult { + #[tokio::test] + async fn test_avro_3896_custom_bytes_schema() -> TestResult { // log type, represents as complex type. let schema = json!( { @@ -6970,15 +7044,15 @@ mod schema { "name": "BytesLog", "logicalType": "custom" }); - let parse_result = Schema::parse(&schema)?; + let parse_result = Schema::parse(&schema).await?; assert_eq!(parse_result, Schema::Bytes); assert_eq!(parse_result.custom_attributes(), None); Ok(()) } - #[test] - fn test_avro_3899_parse_decimal_type() -> TestResult { + #[tokio::test] + async fn test_avro_3899_parse_decimal_type() -> TestResult { let schema = Schema::parse_str( r#"{ "name": "InvalidDecimal", @@ -6988,7 +7062,8 @@ mod schema { "precision": 2, "scale": 3 }"#, - )?; + ) + .await?; match schema { Schema::Fixed(fixed_schema) => { let attrs = fixed_schema.attributes; @@ -7013,7 +7088,8 @@ mod schema { "precision": 3, "scale": 2 }"#, - )?; + ) + .await?; match schema { Schema::Decimal(_) => { assert_not_logged( @@ -7026,8 +7102,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3920_serialize_record_with_custom_attributes() -> TestResult { + #[tokio::test] + async fn avro_3920_serialize_record_with_custom_attributes() -> TestResult { let expected = { let mut lookup = BTreeMap::new(); lookup.insert("value".to_owned(), 0); @@ -7059,13 +7135,13 @@ mod schema { r#"{"aliases":["LinkedLongs"],"custom-attribute":"value","fields":[{"field-id":1,"name":"value","type":"long"}],"name":"LongList","type":"record"}"#, &serialized ); - assert_eq!(expected, Schema::parse_str(&serialized)?); + assert_eq!(expected, Schema::parse_str(&serialized).await?); Ok(()) } - #[test] - fn test_avro_3925_serialize_decimal_inner_fixed() -> TestResult { + #[tokio::test] + async fn test_avro_3925_serialize_decimal_inner_fixed() -> TestResult { let schema = Schema::Decimal(DecimalSchema { precision: 36, scale: 10, @@ -7095,8 +7171,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3925_serialize_decimal_inner_bytes() -> TestResult { + #[tokio::test] + async fn test_avro_3925_serialize_decimal_inner_bytes() -> TestResult { let schema = Schema::Decimal(DecimalSchema { precision: 36, scale: 10, @@ -7117,8 +7193,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3925_serialize_decimal_inner_invalid() -> TestResult { + #[tokio::test] + async fn test_avro_3925_serialize_decimal_inner_invalid() -> TestResult { let schema = Schema::Decimal(DecimalSchema { precision: 36, scale: 10, @@ -7132,8 +7208,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3927_serialize_array_with_custom_attributes() -> TestResult { + #[tokio::test] + async fn test_avro_3927_serialize_array_with_custom_attributes() -> TestResult { let expected = Schema::array_with_attributes( Schema::Long, BTreeMap::from([("field-id".to_string(), "1".into())]), @@ -7145,7 +7221,7 @@ mod schema { r#"{"field-id":"1","items":"long","type":"array"}"#, &serialized ); - let actual_schema = Schema::parse_str(&serialized)?; + let actual_schema = Schema::parse_str(&serialized).await?; assert_eq!(expected, actual_schema); assert_eq!( expected.custom_attributes(), @@ -7155,8 +7231,8 @@ mod schema { Ok(()) } - #[test] - fn test_avro_3927_serialize_map_with_custom_attributes() -> TestResult { + #[tokio::test] + async fn test_avro_3927_serialize_map_with_custom_attributes() -> TestResult { let expected = Schema::map_with_attributes( Schema::Long, BTreeMap::from([("field-id".to_string(), "1".into())]), @@ -7168,7 +7244,7 @@ mod schema { r#"{"field-id":"1","type":"map","values":"long"}"#, &serialized ); - let actual_schema = Schema::parse_str(&serialized)?; + let actual_schema = Schema::parse_str(&serialized).await?; assert_eq!(expected, actual_schema); assert_eq!( expected.custom_attributes(), @@ -7178,8 +7254,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3928_parse_int_based_schema_with_default() -> TestResult { + #[tokio::test] + async fn avro_3928_parse_int_based_schema_with_default() -> TestResult { let schema = r#" { "type": "record", @@ -7191,7 +7267,7 @@ mod schema { } ] }"#; - match Schema::parse_str(schema)? { + match Schema::parse_str(schema).await? { Schema::Record(record_schema) => { assert_eq!(record_schema.fields.len(), 1); let field = record_schema.fields.first().unwrap(); @@ -7208,8 +7284,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3946_union_with_single_type() -> TestResult { + #[tokio::test] + async fn avro_3946_union_with_single_type() -> TestResult { let schema = r#" { "type": "record", @@ -7223,7 +7299,7 @@ mod schema { ] }"#; - let _ = Schema::parse_str(schema)?; + let _ = Schema::parse_str(schema).await?; assert_logged( "Union schema with just one member! Consider dropping the union! \ @@ -7234,8 +7310,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3946_union_without_any_types() -> TestResult { + #[tokio::test] + async fn avro_3946_union_without_any_types() -> TestResult { let schema = r#" { "type": "record", @@ -7249,7 +7325,7 @@ mod schema { ] }"#; - let _ = Schema::parse_str(schema)?; + let _ = Schema::parse_str(schema).await?; assert_logged( "Union schemas should have at least two members! \ @@ -7260,8 +7336,8 @@ mod schema { Ok(()) } - #[test] - fn avro_3965_fixed_schema_with_default_bigger_than_size() -> TestResult { + #[tokio::test] + async fn avro_3965_fixed_schema_with_default_bigger_than_size() -> TestResult { match Schema::parse_str( r#"{ "type": "fixed", @@ -7269,7 +7345,9 @@ mod schema { "size": 1, "default": "123456789" }"#, - ) { + ) + .await + { Ok(_schema) => panic!("Must fail!"), Err(err) => { assert_eq!( @@ -7282,8 +7360,8 @@ mod schema { Ok(()) } - #[test] - fn avro_4004_canonical_form_strip_logical_types() -> TestResult { + #[tokio::test] + async fn avro_4004_canonical_form_strip_logical_types() -> TestResult { let schema_str = r#" { "type": "record", @@ -7295,7 +7373,7 @@ mod schema { ] }"#; - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); let fp_rabin = schema.fingerprint::(); assert_eq!( @@ -7306,8 +7384,8 @@ mod schema { Ok(()) } - #[test] - fn avro_4055_should_fail_to_parse_invalid_schema() -> TestResult { + #[tokio::test] + async fn avro_4055_should_fail_to_parse_invalid_schema() -> TestResult { // This is invalid because the record type should be inside the type field. let invalid_schema_str = r#" { @@ -7329,7 +7407,7 @@ mod schema { ] }"#; - let schema = Schema::parse_str(invalid_schema_str); + let schema = Schema::parse_str(invalid_schema_str).await; assert!(schema.is_err()); assert_eq!( schema.unwrap_err().to_string(), @@ -7358,7 +7436,7 @@ mod schema { } ] }"#; - let schema = Schema::parse_str(valid_schema); + let schema = Schema::parse_str(valid_schema).await; assert!(schema.is_ok()); Ok(()) diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index b9128936..7949a20f 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -25,16 +25,18 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::codec::tokio => crate::codec::sync, crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, + crate::reader::tokio => crate::reader::sync, crate::types::tokio => crate::types::sync, crate::schema_equality::tokio => crate::schema_equality::sync, crate::util::tokio => crate::util::sync, crate::validator::tokio => crate::validator::sync, + crate::writer::tokio => crate::writer::sync, #[tokio::test] => #[test] ); } @@ -557,104 +559,131 @@ mod schema_compatibility { mod tests { use super::*; use crate::{ - Codec, Reader, Writer, - types::{Record, Value}, + codec::tokio::Codec, + reader::tokio::Reader, + types::tokio::{Record, Value}, + writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; use rstest::*; - fn int_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"int"}"#).unwrap() + async fn int_array_schema() -> Schema { + Schema::parse_str(r#"{"type":"array", "items":"int"}"#) + .await + .unwrap() } - fn long_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"long"}"#).unwrap() + async fn long_array_schema() -> Schema { + Schema::parse_str(r#"{"type":"array", "items":"long"}"#) + .await + .unwrap() } - fn string_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"string"}"#).unwrap() + async fn string_array_schema() -> Schema { + Schema::parse_str(r#"{"type":"array", "items":"string"}"#) + .await + .unwrap() } - fn int_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"int"}"#).unwrap() + async fn int_map_schema() -> Schema { + Schema::parse_str(r#"{"type":"map", "values":"int"}"#) + .await + .unwrap() } - fn long_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"long"}"#).unwrap() + async fn long_map_schema() -> Schema { + Schema::parse_str(r#"{"type":"map", "values":"long"}"#) + .await + .unwrap() } - fn string_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"string"}"#).unwrap() + async fn string_map_schema() -> Schema { + Schema::parse_str(r#"{"type":"map", "values":"string"}"#) + .await + .unwrap() } - fn enum1_ab_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B"]}"#).unwrap() + async fn enum1_ab_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B"]}"#) + .await + .unwrap() } - fn enum1_abc_schema() -> Schema { + async fn enum1_abc_schema() -> Schema { Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}"#) + .await .unwrap() } - fn enum1_bc_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["B","C"]}"#).unwrap() + async fn enum1_bc_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["B","C"]}"#) + .await + .unwrap() } - fn enum2_ab_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum2", "symbols":["A","B"]}"#).unwrap() + async fn enum2_ab_schema() -> Schema { + Schema::parse_str(r#"{"type":"enum", "name":"Enum2", "symbols":["A","B"]}"#) + .await + .unwrap() } - fn empty_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[]}"#).unwrap() + async fn empty_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[]}"#) + .await + .unwrap() } - fn empty_record2_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record2", "fields": []}"#).unwrap() + async fn empty_record2_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record2", "fields": []}"#) + .await + .unwrap() } - fn a_int_record1_schema() -> Schema { + async fn a_int_record1_schema() -> Schema { Schema::parse_str( r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}]}"#, ) + .await .unwrap() } - fn a_long_record1_schema() -> Schema { + async fn a_long_record1_schema() -> Schema { Schema::parse_str( r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"long"}]}"#, ) + .await .unwrap() } - fn a_int_b_int_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}"#).unwrap() + async fn a_int_b_int_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}"#).await.unwrap() } - fn a_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}"#).unwrap() + async fn a_dint_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}"#).await.unwrap() } - fn a_int_b_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}"#).unwrap() + async fn a_int_b_dint_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}"#).await.unwrap() } - fn a_dint_b_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}"#).unwrap() + async fn a_dint_b_dint_record1_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}"#).await.unwrap() } - fn nested_record() -> Schema { - Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}}]}"#).unwrap() + async fn nested_record() -> Schema { + Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}}]}"#).await.unwrap() } - fn nested_optional_record() -> Schema { - Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":["null",{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}],"default":null}]}"#).unwrap() + async fn nested_optional_record() -> Schema { + Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":["null",{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}],"default":null}]}"#).await.unwrap() } - fn int_list_record_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"List", "fields": [{"name": "head", "type": "int"},{"name": "tail", "type": "array", "items": "int"}]}"#).unwrap() + async fn int_list_record_schema() -> Schema { + Schema::parse_str(r#"{"type":"record", "name":"List", "fields": [{"name": "head", "type": "int"},{"name": "tail", "type": "array", "items": "int"}]}"#).await.unwrap() } - fn long_list_record_schema() -> Schema { + async fn long_list_record_schema() -> Schema { Schema::parse_str( r#" { @@ -664,56 +693,62 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - fn union_schema(schemas: Vec) -> Schema { + async fn union_schema(schemas: Vec) -> Schema { let schema_string = schemas .iter() .map(|s| s.canonical_form()) .collect::>() .join(","); - Schema::parse_str(&format!("[{schema_string}]")).unwrap() + Schema::parse_str(&format!("[{schema_string}]")) + .await + .unwrap() } - fn empty_union_schema() -> Schema { - union_schema(vec![]) + async fn empty_union_schema() -> Schema { + union_schema(vec![]).await } // unused // fn null_union_schema() -> Schema { union_schema(vec![Schema::Null]) } - fn int_union_schema() -> Schema { - union_schema(vec![Schema::Int]) + async fn int_union_schema() -> Schema { + union_schema(vec![Schema::Int]).await } - fn long_union_schema() -> Schema { - union_schema(vec![Schema::Long]) + async fn long_union_schema() -> Schema { + union_schema(vec![Schema::Long]).await } - fn string_union_schema() -> Schema { - union_schema(vec![Schema::String]) + async fn string_union_schema() -> Schema { + union_schema(vec![Schema::String]).await } - fn int_string_union_schema() -> Schema { - union_schema(vec![Schema::Int, Schema::String]) + async fn int_string_union_schema() -> Schema { + union_schema(vec![Schema::Int, Schema::String]).await } - fn string_int_union_schema() -> Schema { - union_schema(vec![Schema::String, Schema::Int]) + async fn string_int_union_schema() -> Schema { + union_schema(vec![Schema::String, Schema::Int]).await } - #[test] - fn test_broken() { + #[tokio::test] + async fn test_broken() { assert_eq!( CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&int_string_union_schema(), &int_union_schema()) - .unwrap_err() + SchemaCompatibility::can_read( + &int_string_union_schema().await, + &int_union_schema().await + ) + .unwrap_err() ) } - #[test] - fn test_incompatible_reader_writer_pairs() { + #[tokio::test] + async fn test_incompatible_reader_writer_pairs() { let incompatible_schemas = vec![ // null (Schema::Null, Schema::Int), @@ -748,25 +783,31 @@ mod schema_compatibility { (Schema::Date, Schema::Long), (Schema::TimeMillis, Schema::Long), // array and maps - (int_array_schema(), long_array_schema()), - (int_map_schema(), int_array_schema()), - (int_array_schema(), int_map_schema()), - (int_map_schema(), long_map_schema()), + (int_array_schema().await, long_array_schema().await), + (int_map_schema().await, int_array_schema().await), + (int_array_schema().await, int_map_schema().await), + (int_map_schema().await, long_map_schema().await), // enum - (enum1_ab_schema(), enum1_abc_schema()), - (enum1_bc_schema(), enum1_abc_schema()), - (enum1_ab_schema(), enum2_ab_schema()), - (Schema::Int, enum2_ab_schema()), - (enum2_ab_schema(), Schema::Int), + (enum1_ab_schema().await, enum1_abc_schema().await), + (enum1_bc_schema().await, enum1_abc_schema().await), + (enum1_ab_schema().await, enum2_ab_schema().await), + (Schema::Int, enum2_ab_schema().await), + (enum2_ab_schema().await, Schema::Int), //union - (int_union_schema(), int_string_union_schema()), - (string_union_schema(), int_string_union_schema()), + (int_union_schema().await, int_string_union_schema().await), + (string_union_schema().await, int_string_union_schema().await), //record - (empty_record2_schema(), empty_record1_schema()), - (a_int_record1_schema(), empty_record1_schema()), - (a_int_b_dint_record1_schema(), empty_record1_schema()), - (int_list_record_schema(), long_list_record_schema()), - (nested_record(), nested_optional_record()), + (empty_record2_schema().await, empty_record1_schema().await), + (a_int_record1_schema().await, empty_record1_schema().await), + ( + a_int_b_dint_record1_schema().await, + empty_record1_schema().await, + ), + ( + int_list_record_schema().await, + long_list_record_schema().await, + ), + (nested_record().await, nested_optional_record().await), ]; assert!( @@ -776,6 +817,7 @@ mod schema_compatibility { ); } + #[cfg_attr(feature = "tokio", tokio::test)] #[rstest] // Record type test #[case( @@ -844,16 +886,17 @@ mod schema_compatibility { r#"{"type": "array", "items": "int"}"#, r#"{"type": "array", "items": "long"}"# )] - fn test_avro_3950_match_schemas_ok( + async fn test_avro_3950_match_schemas_ok( #[case] writer_schema_str: &str, #[case] reader_schema_str: &str, ) { - let writer_schema = Schema::parse_str(writer_schema_str).unwrap(); - let reader_schema = Schema::parse_str(reader_schema_str).unwrap(); + let writer_schema = Schema::parse_str(writer_schema_str).await.unwrap(); + let reader_schema = Schema::parse_str(reader_schema_str).await.unwrap(); assert!(SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).is_ok()); } + #[cfg_attr(feature = "tokio", tokio::test)] #[rstest] // Record type test #[case( @@ -1024,13 +1067,13 @@ mod schema_compatibility { r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, CompatibilityError::Inconclusive(String::from("writers_schema")) )] - fn test_avro_3950_match_schemas_error( + async fn test_avro_3950_match_schemas_error( #[case] writer_schema_str: &str, #[case] reader_schema_str: &str, #[case] expected_error: CompatibilityError, ) { - let writer_schema = Schema::parse_str(writer_schema_str).unwrap(); - let reader_schema = Schema::parse_str(reader_schema_str).unwrap(); + let writer_schema = Schema::parse_str(writer_schema_str).await.unwrap(); + let reader_schema = Schema::parse_str(reader_schema_str).await.unwrap(); assert_eq!( expected_error, @@ -1038,8 +1081,8 @@ mod schema_compatibility { ) } - #[test] - fn test_compatible_reader_writer_pairs() { + #[tokio::test] + async fn test_compatible_reader_writer_pairs() { let compatible_schemas = vec![ (Schema::Null, Schema::Null), (Schema::Long, Schema::Int), @@ -1072,36 +1115,66 @@ mod schema_compatibility { (Schema::Long, Schema::LocalTimestampMillis), (Schema::Long, Schema::LocalTimestampMicros), (Schema::Long, Schema::LocalTimestampNanos), - (int_array_schema(), int_array_schema()), - (long_array_schema(), int_array_schema()), - (int_map_schema(), int_map_schema()), - (long_map_schema(), int_map_schema()), - (enum1_ab_schema(), enum1_ab_schema()), - (enum1_abc_schema(), enum1_ab_schema()), - (empty_union_schema(), empty_union_schema()), - (int_union_schema(), int_union_schema()), - (int_string_union_schema(), string_int_union_schema()), - (int_union_schema(), empty_union_schema()), - (long_union_schema(), int_union_schema()), - (int_union_schema(), Schema::Int), - (Schema::Int, int_union_schema()), - (empty_record1_schema(), empty_record1_schema()), - (empty_record1_schema(), a_int_record1_schema()), - (a_int_record1_schema(), a_int_record1_schema()), - (a_dint_record1_schema(), a_int_record1_schema()), - (a_dint_record1_schema(), a_dint_record1_schema()), - (a_int_record1_schema(), a_dint_record1_schema()), - (a_long_record1_schema(), a_int_record1_schema()), - (a_int_record1_schema(), a_int_b_int_record1_schema()), - (a_dint_record1_schema(), a_int_b_int_record1_schema()), - (a_int_b_dint_record1_schema(), a_int_record1_schema()), - (a_dint_b_dint_record1_schema(), empty_record1_schema()), - (a_dint_b_dint_record1_schema(), a_int_record1_schema()), - (a_int_b_int_record1_schema(), a_dint_b_dint_record1_schema()), - (int_list_record_schema(), int_list_record_schema()), - (long_list_record_schema(), long_list_record_schema()), - (long_list_record_schema(), int_list_record_schema()), - (nested_optional_record(), nested_record()), + (int_array_schema().await, int_array_schema().await), + (long_array_schema().await, int_array_schema().await), + (int_map_schema().await, int_map_schema().await), + (long_map_schema().await, int_map_schema().await), + (enum1_ab_schema().await, enum1_ab_schema().await), + (enum1_abc_schema().await, enum1_ab_schema().await), + (empty_union_schema().await, empty_union_schema().await), + (int_union_schema().await, int_union_schema().await), + ( + int_string_union_schema().await, + string_int_union_schema().await, + ), + (int_union_schema().await, empty_union_schema().await), + (long_union_schema().await, int_union_schema().await), + (int_union_schema().await, Schema::Int), + (Schema::Int, int_union_schema().await), + (empty_record1_schema().await, empty_record1_schema().await), + (empty_record1_schema().await, a_int_record1_schema().await), + (a_int_record1_schema().await, a_int_record1_schema().await), + (a_dint_record1_schema().await, a_int_record1_schema().await), + (a_dint_record1_schema().await, a_dint_record1_schema().await), + (a_int_record1_schema().await, a_dint_record1_schema().await), + (a_long_record1_schema().await, a_int_record1_schema().await), + ( + a_int_record1_schema().await, + a_int_b_int_record1_schema().await, + ), + ( + a_dint_record1_schema().await, + a_int_b_int_record1_schema().await, + ), + ( + a_int_b_dint_record1_schema().await, + a_int_record1_schema().await, + ), + ( + a_dint_b_dint_record1_schema().await, + empty_record1_schema().await, + ), + ( + a_dint_b_dint_record1_schema().await, + a_int_record1_schema().await, + ), + ( + a_int_b_int_record1_schema().await, + a_dint_b_dint_record1_schema().await, + ), + ( + int_list_record_schema().await, + int_list_record_schema().await, + ), + ( + long_list_record_schema().await, + long_list_record_schema().await, + ), + ( + long_list_record_schema().await, + int_list_record_schema().await, + ), + (nested_optional_record().await, nested_record().await), ]; assert!( @@ -1111,7 +1184,7 @@ mod schema_compatibility { ); } - fn writer_schema() -> Schema { + async fn writer_schema() -> Schema { Schema::parse_str( r#" {"type":"record", "name":"Record", "fields":[ @@ -1120,47 +1193,50 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - #[test] - fn test_missing_field() -> TestResult { + #[tokio::test] + async fn test_missing_field() -> TestResult { let reader_schema = Schema::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema,).is_ok()); + ) + .await?; + assert!(SchemaCompatibility::can_read(&writer_schema().await, &reader_schema,).is_ok()); assert_eq!( CompatibilityError::MissingDefaultValue(String::from("oldfield2")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + SchemaCompatibility::can_read(&reader_schema, &writer_schema().await).unwrap_err() ); Ok(()) } - #[test] - fn test_missing_second_field() -> TestResult { + #[tokio::test] + async fn test_missing_second_field() -> TestResult { let reader_schema = Schema::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield2", "type":"string"} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); + ) + .await?; + assert!(SchemaCompatibility::can_read(&writer_schema().await, &reader_schema).is_ok()); assert_eq!( CompatibilityError::MissingDefaultValue(String::from("oldfield1")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + SchemaCompatibility::can_read(&reader_schema, &writer_schema().await).unwrap_err() ); Ok(()) } - #[test] - fn test_all_fields() -> TestResult { + #[tokio::test] + async fn test_all_fields() -> TestResult { let reader_schema = Schema::parse_str( r#" {"type":"record", "name":"Record", "fields":[ @@ -1168,15 +1244,16 @@ mod schema_compatibility { {"name":"oldfield2", "type":"string"} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); - assert!(SchemaCompatibility::can_read(&reader_schema, &writer_schema()).is_ok()); + ) + .await?; + assert!(SchemaCompatibility::can_read(&writer_schema().await, &reader_schema).is_ok()); + assert!(SchemaCompatibility::can_read(&reader_schema, &writer_schema().await).is_ok()); Ok(()) } - #[test] - fn test_new_field_with_default() -> TestResult { + #[tokio::test] + async fn test_new_field_with_default() -> TestResult { let reader_schema = Schema::parse_str( r#" {"type":"record", "name":"Record", "fields":[ @@ -1184,18 +1261,19 @@ mod schema_compatibility { {"name":"newfield1", "type":"int", "default":42} ]} "#, - )?; - assert!(SchemaCompatibility::can_read(&writer_schema(), &reader_schema).is_ok()); + ) + .await?; + assert!(SchemaCompatibility::can_read(&writer_schema().await, &reader_schema).is_ok()); assert_eq!( CompatibilityError::MissingDefaultValue(String::from("oldfield2")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + SchemaCompatibility::can_read(&reader_schema, &writer_schema().await).unwrap_err() ); Ok(()) } - #[test] - fn test_new_field() -> TestResult { + #[tokio::test] + async fn test_new_field() -> TestResult { let reader_schema = Schema::parse_str( r#" {"type":"record", "name":"Record", "fields":[ @@ -1203,33 +1281,37 @@ mod schema_compatibility { {"name":"newfield1", "type":"int"} ]} "#, - )?; + ) + .await?; assert_eq!( CompatibilityError::MissingDefaultValue(String::from("newfield1")), - SchemaCompatibility::can_read(&writer_schema(), &reader_schema).unwrap_err() + SchemaCompatibility::can_read(&writer_schema().await, &reader_schema).unwrap_err() ); assert_eq!( CompatibilityError::MissingDefaultValue(String::from("oldfield2")), - SchemaCompatibility::can_read(&reader_schema, &writer_schema()).unwrap_err() + SchemaCompatibility::can_read(&reader_schema, &writer_schema().await).unwrap_err() ); Ok(()) } - #[test] - fn test_array_writer_schema() { - let valid_reader = string_array_schema(); - let invalid_reader = string_map_schema(); + #[tokio::test] + async fn test_array_writer_schema() { + let valid_reader = string_array_schema().await; + let invalid_reader = string_map_schema().await; - assert!(SchemaCompatibility::can_read(&string_array_schema(), &valid_reader).is_ok()); + assert!( + SchemaCompatibility::can_read(&string_array_schema().await, &valid_reader).is_ok() + ); assert_eq!( CompatibilityError::Inconclusive(String::from("writers_schema")), - SchemaCompatibility::can_read(&string_array_schema(), &invalid_reader).unwrap_err() + SchemaCompatibility::can_read(&string_array_schema().await, &invalid_reader) + .unwrap_err() ); } - #[test] - fn test_primitive_writer_schema() { + #[tokio::test] + async fn test_primitive_writer_schema() { let valid_reader = Schema::String; assert!(SchemaCompatibility::can_read(&Schema::String, &valid_reader).is_ok()); assert_eq!( @@ -1248,11 +1330,11 @@ mod schema_compatibility { ); } - #[test] - fn test_union_reader_writer_subset_incompatibility() { + #[tokio::test] + async fn test_union_reader_writer_subset_incompatibility() { // reader union schema must contain all writer union branches - let union_writer = union_schema(vec![Schema::Int, Schema::String]); - let union_reader = union_schema(vec![Schema::String]); + let union_writer = union_schema(vec![Schema::Int, Schema::String]).await; + let union_reader = union_schema(vec![Schema::String]).await; assert_eq!( CompatibilityError::MissingUnionElements, @@ -1261,15 +1343,16 @@ mod schema_compatibility { assert!(SchemaCompatibility::can_read(&union_reader, &union_writer).is_ok()); } - #[test] - fn test_incompatible_record_field() -> TestResult { + #[tokio::test] + async fn test_incompatible_record_field() -> TestResult { let string_schema = Schema::parse_str( r#" {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [ {"name":"field1", "type":"string"} ]} "#, - )?; + ) + .await?; let int_schema = Schema::parse_str( r#" @@ -1277,7 +1360,8 @@ mod schema_compatibility { {"name":"field1", "type":"int"} ]} "#, - )?; + ) + .await?; assert_eq!( CompatibilityError::FieldTypeMismatch( @@ -1297,15 +1381,17 @@ mod schema_compatibility { Ok(()) } - #[test] - fn test_enum_symbols() -> TestResult { + #[tokio::test] + async fn test_enum_symbols() -> TestResult { let enum_schema1 = Schema::parse_str( r#" {"type":"enum", "name":"MyEnum", "symbols":["A","B"]} "#, - )?; + ) + .await?; let enum_schema2 = - Schema::parse_str(r#"{"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}"#)?; + Schema::parse_str(r#"{"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}"#) + .await?; assert_eq!( CompatibilityError::MissingSymbols, SchemaCompatibility::can_read(&enum_schema2, &enum_schema1).unwrap_err() @@ -1315,7 +1401,7 @@ mod schema_compatibility { Ok(()) } - fn point_2d_schema() -> Schema { + async fn point_2d_schema() -> Schema { Schema::parse_str( r#" {"type":"record", "name":"Point2D", "fields":[ @@ -1324,10 +1410,11 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - fn point_2d_fullname_schema() -> Schema { + async fn point_2d_fullname_schema() -> Schema { Schema::parse_str( r#" {"type":"record", "name":"Point", "namespace":"written", "fields":[ @@ -1336,10 +1423,11 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - fn point_3d_no_default_schema() -> Schema { + async fn point_3d_no_default_schema() -> Schema { Schema::parse_str( r#" {"type":"record", "name":"Point", "fields":[ @@ -1349,10 +1437,11 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - fn point_3d_schema() -> Schema { + async fn point_3d_schema() -> Schema { Schema::parse_str( r#" {"type":"record", "name":"Point3D", "fields":[ @@ -1362,10 +1451,11 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - fn point_3d_match_name_schema() -> Schema { + async fn point_3d_match_name_schema() -> Schema { Schema::parse_str( r#" {"type":"record", "name":"Point", "fields":[ @@ -1375,85 +1465,92 @@ mod schema_compatibility { ]} "#, ) + .await .unwrap() } - #[test] - fn test_union_resolution_no_structure_match() { + #[tokio::test] + async fn test_union_resolution_no_structure_match() { // short name match, but no structure match - let read_schema = union_schema(vec![Schema::Null, point_3d_no_default_schema()]); + let read_schema = + union_schema(vec![Schema::Null, point_3d_no_default_schema().await]).await; assert_eq!( CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + SchemaCompatibility::can_read(&point_2d_fullname_schema().await, &read_schema) .unwrap_err() ); } - #[test] - fn test_union_resolution_first_structure_match_2d() { + #[tokio::test] + async fn test_union_resolution_first_structure_match_2d() { // multiple structure matches with no name matches let read_schema = union_schema(vec![ Schema::Null, - point_3d_no_default_schema(), - point_2d_schema(), - point_3d_schema(), - ]); + point_3d_no_default_schema().await, + point_2d_schema().await, + point_3d_schema().await, + ]) + .await; assert_eq!( CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + SchemaCompatibility::can_read(&point_2d_fullname_schema().await, &read_schema) .unwrap_err() ); } - #[test] - fn test_union_resolution_first_structure_match_3d() { + #[tokio::test] + async fn test_union_resolution_first_structure_match_3d() { // multiple structure matches with no name matches let read_schema = union_schema(vec![ Schema::Null, - point_3d_no_default_schema(), - point_3d_schema(), - point_2d_schema(), - ]); + point_3d_no_default_schema().await, + point_3d_schema().await, + point_2d_schema().await, + ]) + .await; assert_eq!( CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + SchemaCompatibility::can_read(&point_2d_fullname_schema().await, &read_schema) .unwrap_err() ); } - #[test] - fn test_union_resolution_named_structure_match() { + #[tokio::test] + async fn test_union_resolution_named_structure_match() { // multiple structure matches with a short name match let read_schema = union_schema(vec![ Schema::Null, - point_2d_schema(), - point_3d_match_name_schema(), - point_3d_schema(), - ]); + point_2d_schema().await, + point_3d_match_name_schema().await, + point_3d_schema().await, + ]) + .await; assert_eq!( CompatibilityError::MissingUnionElements, - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema) + SchemaCompatibility::can_read(&point_2d_fullname_schema().await, &read_schema) .unwrap_err() ); } - #[test] - fn test_union_resolution_full_name_match() { + #[tokio::test] + async fn test_union_resolution_full_name_match() { // there is a full name match that should be chosen let read_schema = union_schema(vec![ Schema::Null, - point_2d_schema(), - point_3d_match_name_schema(), - point_3d_schema(), - point_2d_fullname_schema(), - ]); + point_2d_schema().await, + point_3d_match_name_schema().await, + point_3d_schema().await, + point_2d_fullname_schema().await, + ]) + .await; assert!( - SchemaCompatibility::can_read(&point_2d_fullname_schema(), &read_schema).is_ok() + SchemaCompatibility::can_read(&point_2d_fullname_schema().await, &read_schema) + .is_ok() ); } - #[test] - fn test_avro_3772_enum_default() -> TestResult { + #[tokio::test] + async fn test_avro_3772_enum_default() -> TestResult { let writer_raw_schema = r#" { "type": "record", @@ -1493,16 +1590,16 @@ mod schema_compatibility { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema)?; - let reader_schema = Schema::parse_str(reader_raw_schema)?; + let writer_schema = Schema::parse_str(writer_raw_schema).await?; + let reader_schema = Schema::parse_str(reader_raw_schema).await?; let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); record.put("b", "foo"); record.put("c", "clubs"); - writer.append(record).unwrap(); + writer.append(record).await.unwrap(); let input = writer.into_inner()?; - let mut reader = Reader::with_schema(&reader_schema, &input[..])?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; assert_eq!( reader.next().unwrap().unwrap(), Value::Record(vec![ @@ -1516,8 +1613,8 @@ mod schema_compatibility { Ok(()) } - #[test] - fn test_avro_3772_enum_default_less_symbols() -> TestResult { + #[tokio::test] + async fn test_avro_3772_enum_default_less_symbols() -> TestResult { let writer_raw_schema = r#" { "type": "record", @@ -1557,16 +1654,16 @@ mod schema_compatibility { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema)?; - let reader_schema = Schema::parse_str(reader_raw_schema)?; + let writer_schema = Schema::parse_str(writer_raw_schema).await?; + let reader_schema = Schema::parse_str(reader_raw_schema).await?; let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); record.put("b", "foo"); record.put("c", "hearts"); - writer.append(record).unwrap(); + writer.append(record).await.unwrap(); let input = writer.into_inner()?; - let mut reader = Reader::with_schema(&reader_schema, &input[..])?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; assert_eq!( reader.next().unwrap().unwrap(), Value::Record(vec![ @@ -1580,8 +1677,8 @@ mod schema_compatibility { Ok(()) } - #[test] - fn avro_3894_take_aliases_into_account_when_serializing_for_schema_compatibility() + #[tokio::test] + async fn avro_3894_take_aliases_into_account_when_serializing_for_schema_compatibility() -> TestResult { let schema_v1 = Schema::parse_str( r#" @@ -1594,7 +1691,8 @@ mod schema_compatibility { {"type": "long", "name": "date"} ] }"#, - )?; + ) + .await?; let schema_v2 = Schema::parse_str( r#" @@ -1607,15 +1705,16 @@ mod schema_compatibility { {"type": "long", "name": "date", "aliases" : [ "time" ]} ] }"#, - )?; + ) + .await?; assert!(SchemaCompatibility::mutual_read(&schema_v1, &schema_v2).is_ok()); Ok(()) } - #[test] - fn avro_3917_take_aliases_into_account_for_schema_compatibility() -> TestResult { + #[tokio::test] + async fn avro_3917_take_aliases_into_account_for_schema_compatibility() -> TestResult { let schema_v1 = Schema::parse_str( r#" { @@ -1627,7 +1726,8 @@ mod schema_compatibility { {"type": "long", "name": "date", "aliases" : [ "time" ]} ] }"#, - )?; + ) + .await?; let schema_v2 = Schema::parse_str( r#" @@ -1640,7 +1740,8 @@ mod schema_compatibility { {"type": "long", "name": "time"} ] }"#, - )?; + ) + .await?; assert!(SchemaCompatibility::can_read(&schema_v2, &schema_v1).is_ok()); assert_eq!( @@ -1651,8 +1752,8 @@ mod schema_compatibility { Ok(()) } - #[test] - fn test_avro_3898_record_schemas_match_by_unqualified_name() -> TestResult { + #[tokio::test] + async fn test_avro_3898_record_schemas_match_by_unqualified_name() -> TestResult { let schemas = [ // Record schemas ( @@ -1667,7 +1768,8 @@ mod schema_compatibility { { "name": "max", "type": "int", "default": 0 } ] }"#, - )?, + ) + .await?, Schema::parse_str( r#"{ "type": "record", @@ -1680,7 +1782,8 @@ mod schema_compatibility { { "name": "average", "type": "int", "default": 0} ] }"#, - )?, + ) + .await?, ), // Enum schemas ( @@ -1690,7 +1793,8 @@ mod schema_compatibility { "name": "Suit", "symbols": ["diamonds", "spades", "clubs"] }"#, - )?, + ) + .await?, Schema::parse_str( r#"{ "type": "enum", @@ -1698,7 +1802,8 @@ mod schema_compatibility { "namespace": "my.namespace", "symbols": ["diamonds", "spades", "clubs", "hearts"] }"#, - )?, + ) + .await?, ), // Fixed schemas ( @@ -1708,7 +1813,8 @@ mod schema_compatibility { "name": "EmployeeId", "size": 16 }"#, - )?, + ) + .await?, Schema::parse_str( r#"{ "type": "fixed", @@ -1716,7 +1822,8 @@ mod schema_compatibility { "namespace": "my.namespace", "size": 16 }"#, - )?, + ) + .await?, ), ]; @@ -1727,8 +1834,8 @@ mod schema_compatibility { Ok(()) } - #[test] - fn test_can_read_compatibility_errors() -> TestResult { + #[tokio::test] + async fn test_can_read_compatibility_errors() -> TestResult { let schemas = [ ( Schema::parse_str( @@ -1740,7 +1847,7 @@ mod schema_compatibility { {"name": "success", "type": {"type": "map", "values": "int"}} ] }"#, - )?, + ).await?, Schema::parse_str( r#"{ "type": "record", @@ -1750,7 +1857,7 @@ mod schema_compatibility { {"name": "success", "type": ["null", {"type": "map", "values": "int"}], "default": null} ] }"#, - )?, + ).await?, "Incompatible schemata! Field 'success' in reader schema does not match the type in the writer schema", ), ( @@ -1762,7 +1869,7 @@ mod schema_compatibility { {"name": "max_values", "type": {"type": "array", "items": "int"}} ] }"#, - )?, + ).await?, Schema::parse_str( r#"{ "type": "record", @@ -1771,7 +1878,7 @@ mod schema_compatibility { {"name": "max_values", "type": ["null", {"type": "array", "items": "int"}], "default": null} ] }"#, - )?, + ).await?, "Incompatible schemata! Field 'max_values' in reader schema does not match the type in the writer schema", ), ]; @@ -1789,8 +1896,8 @@ mod schema_compatibility { Ok(()) } - #[test] - fn avro_3974_can_read_schema_references() -> TestResult { + #[tokio::test] + async fn avro_3974_can_read_schema_references() -> TestResult { let schema_strs = vec![ r#"{ "type": "record", @@ -1818,7 +1925,7 @@ mod schema_compatibility { "#, ]; - let schemas = Schema::parse_list(schema_strs).unwrap(); + let schemas = Schema::parse_list(schema_strs).await.unwrap(); SchemaCompatibility::can_read(&schemas[1], &schemas[1])?; Ok(()) diff --git a/avro/src/schema_equality.rs b/avro/src/schema_equality.rs index 02bd42b4..803733da 100644 --- a/avro/src/schema_equality.rs +++ b/avro/src/schema_equality.rs @@ -272,7 +272,7 @@ mod schema_equality { #[allow(non_snake_case)] mod tests { use super::*; - use crate::schema::{Name, RecordFieldOrder}; + use crate::schema::tokio::{Name, RecordFieldOrder}; use apache_avro_test_helper::TestResult; use serde_json::Value; use std::collections::BTreeMap; diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 73cb5e43..991004d0 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -513,7 +513,7 @@ mod ser { #[cfg(test)] mod tests { use super::*; - use crate::Decimal; + use crate::decimal::tokio::Decimal; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 04a73573..2aad1bfa 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -1849,8 +1849,8 @@ mod bigdecimal { mod tests { use super::*; use crate::{ - Days, Duration, Millis, Months, decimal::Decimal, error::Details, - schema::ResolvedSchema, + Days, Duration, Millis, Months, decimal::tokio::Decimal, error::tokio::Details, + schema::tokio::ResolvedSchema, }; use apache_avro_test_helper::TestResult; use bigdecimal::BigDecimal; @@ -2032,8 +2032,8 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_record() -> TestResult { + #[tokio::test] + async fn test_serialize_record() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "record", @@ -2043,7 +2043,7 @@ mod bigdecimal { {"name": "intField", "type": "int"} ] }"#, - )?; + ).await?; #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -2084,15 +2084,15 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_empty_record() -> TestResult { + #[tokio::test] + async fn test_serialize_empty_record() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "record", "name": "EmptyRecord", "fields": [] }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2136,15 +2136,15 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_enum() -> TestResult { + #[tokio::test] + async fn test_serialize_enum() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "enum", "name": "Suit", "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] }"#, - )?; + ).await?; #[derive(Serialize)] enum Suit { @@ -2184,14 +2184,14 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_array() -> TestResult { + #[tokio::test] + async fn test_serialize_array() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "array", "items": "long" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2222,14 +2222,14 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_map() -> TestResult { + #[tokio::test] + async fn test_serialize_map() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "map", "values": "long" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2268,13 +2268,13 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_nullable_union() -> TestResult { + #[tokio::test] + async fn test_serialize_nullable_union() -> TestResult { let schema = Schema::parse_str( r#"{ "type": ["null", "long"] }"#, - )?; + ).await?; #[derive(Serialize)] enum NullableLong { @@ -2316,13 +2316,13 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_union() -> TestResult { + #[tokio::test] + async fn test_serialize_union() -> TestResult { let schema = Schema::parse_str( r#"{ "type": ["null", "long", "string"] }"#, - )?; + ).await?; #[derive(Serialize)] enum LongOrString { @@ -2367,15 +2367,15 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_fixed() -> TestResult { + #[tokio::test] + async fn test_serialize_fixed() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "fixed", "size": 8, "name": "LongVal" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2443,8 +2443,8 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_decimal_bytes() -> TestResult { + #[tokio::test] + async fn test_serialize_decimal_bytes() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "bytes", @@ -2452,7 +2452,7 @@ mod bigdecimal { "precision": 16, "scale": 2 }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2480,8 +2480,8 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_decimal_fixed() -> TestResult { + #[tokio::test] + async fn test_serialize_decimal_fixed() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "fixed", @@ -2491,7 +2491,7 @@ mod bigdecimal { "precision": 16, "scale": 2 }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2519,15 +2519,15 @@ mod bigdecimal { Ok(()) } - #[test] + #[tokio::test] #[serial(serde_is_human_readable)] - fn test_serialize_bigdecimal() -> TestResult { + async fn test_serialize_bigdecimal() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "bytes", "logicalType": "big-decimal" }"#, - )?; + ).await?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); @@ -2543,15 +2543,15 @@ mod bigdecimal { Ok(()) } - #[test] + #[tokio::test] #[serial(serde_is_human_readable)] - fn test_serialize_uuid() -> TestResult { + async fn test_serialize_uuid() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "string", "logicalType": "uuid" }"#, - )?; + ).await?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); @@ -2588,14 +2588,14 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_date() -> TestResult { + #[tokio::test] + async fn test_serialize_date() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "int", "logicalType": "date" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2632,14 +2632,14 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_time_millis() -> TestResult { + #[tokio::test] + async fn test_serialize_time_millis() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "int", "logicalType": "time-millis" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2676,14 +2676,14 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_time_micros() -> TestResult { + #[tokio::test] + async fn test_serialize_time_micros() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "long", "logicalType": "time-micros" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2723,15 +2723,15 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_timestamp() -> TestResult { + #[tokio::test] + async fn test_serialize_timestamp() -> TestResult { for precision in ["millis", "micros", "nanos"] { let schema = Schema::parse_str(&format!( r#"{{ "type": "long", "logicalType": "timestamp-{precision}" }}"# - ))?; + )).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2781,8 +2781,8 @@ mod bigdecimal { Ok(()) } - #[test] - fn test_serialize_duration() -> TestResult { + #[tokio::test] + async fn test_serialize_duration() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "fixed", @@ -2790,7 +2790,7 @@ mod bigdecimal { "name": "duration", "logicalType": "duration" }"#, - )?; + ).await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2826,9 +2826,9 @@ mod bigdecimal { Ok(()) } - #[test] + #[tokio::test] #[serial(serde_is_human_readable)] // for BigDecimal and Uuid - fn test_serialize_recursive_record() -> TestResult { + async fn test_serialize_recursive_record() -> TestResult { let schema = Schema::parse_str( r#"{ "type": "record", @@ -2841,7 +2841,7 @@ mod bigdecimal { {"name": "innerRecord", "type": ["null", "TestRecord"]} ] }"#, - )?; + ).await?; #[derive(Serialize)] #[serde(rename_all = "camelCase")] diff --git a/avro/src/types.rs b/avro/src/types.rs index af716a0f..f071ffe6 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -33,6 +33,7 @@ crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, crate::schema_equality::tokio => crate::schema_equality::sync, + crate::ser::tokio => crate::ser::sync, crate::util::tokio => crate::util::sync, crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] @@ -771,16 +772,13 @@ mod types { .. }) => self.resolve_enum(symbols, default, field_default), Schema::Array(ref inner) => { - Box::pin(self.resolve_array(&inner.items, names, enclosing_namespace)) - .await + Box::pin(self.resolve_array(&inner.items, names, enclosing_namespace)).await } Schema::Map(ref inner) => { - Box::pin(self.resolve_map(&inner.types, names, enclosing_namespace)) - .await + Box::pin(self.resolve_map(&inner.types, names, enclosing_namespace)).await } Schema::Record(RecordSchema { ref fields, .. }) => { - Box::pin(self.resolve_record(fields, names, enclosing_namespace)) - .await + Box::pin(self.resolve_record(fields, names, enclosing_namespace)).await } Schema::Decimal(DecimalSchema { scale, @@ -1154,9 +1152,13 @@ mod types { Value::Array(items) => { let mut resolved_values = Vec::with_capacity(items.len()); for item in items.into_iter() { - let resolved = Box::pin(item - .resolve_internal(schema, names, enclosing_namespace, &None)) - .await?; + let resolved = Box::pin(item.resolve_internal( + schema, + names, + enclosing_namespace, + &None, + )) + .await?; resolved_values.push(resolved); } Ok(Value::Array(resolved_values)) @@ -1302,6 +1304,7 @@ mod types { duration::{Days, Millis, Months}, error::tokio::Details, schema::tokio::RecordFieldOrder, + ser::tokio::Serializer, }; use apache_avro_test_helper::{ TestResult, @@ -1342,7 +1345,8 @@ mod types { } ] }"#, - ).await?; + ) + .await?; let value = Value::Record(vec![( "outer_field_1".into(), Value::Record(vec![ @@ -1357,7 +1361,7 @@ mod types { ]), )]); - assert!(value.validate(&schema)); + assert!(value.validate(&schema).await); Ok(()) } @@ -1497,8 +1501,9 @@ mod types { ]; for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { - let err_message = - value.validate_internal::(&schema, &HashMap::default(), &None).await; + let err_message = value + .validate_internal::(&schema, &HashMap::default(), &None) + .await; assert_eq!(valid, err_message.is_none()); if !valid { let full_err_message = format!( @@ -1523,9 +1528,9 @@ mod types { attributes: Default::default(), }); - assert!(Value::Fixed(4, vec![0, 0, 0, 0]).validate(&schema)); + assert!(Value::Fixed(4, vec![0, 0, 0, 0]).validate(&schema).await); let value = Value::Fixed(5, vec![0, 0, 0, 0, 0]); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1534,9 +1539,9 @@ mod types { .as_str(), ); - assert!(Value::Bytes(vec![0, 0, 0, 0]).validate(&schema)); + assert!(Value::Bytes(vec![0, 0, 0, 0]).validate(&schema).await); let value = Value::Bytes(vec![0, 0, 0, 0, 0]); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1564,11 +1569,11 @@ mod types { attributes: Default::default(), }); - assert!(Value::Enum(0, "spades".to_string()).validate(&schema)); - assert!(Value::String("spades".to_string()).validate(&schema)); + assert!(Value::Enum(0, "spades".to_string()).validate(&schema).await); + assert!(Value::String("spades".to_string()).validate(&schema).await); let value = Value::Enum(1, "spades".to_string()); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1578,7 +1583,7 @@ mod types { ); let value = Value::Enum(1000, "spades".to_string()); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1588,7 +1593,7 @@ mod types { ); let value = Value::String("lorem".to_string()); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1691,19 +1696,20 @@ mod types { ("b".to_string(), Value::String("foo".to_string())), ]) .validate(&schema) + .await ); let value = Value::Record(vec![ ("b".to_string(), Value::String("foo".to_string())), ("a".to_string(), Value::Long(42i64)), ]); - assert!(value.validate(&schema)); + assert!(value.validate(&schema).await); let value = Value::Record(vec![ ("a".to_string(), Value::Boolean(false)), ("b".to_string(), Value::String("foo".to_string())), ]); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( r#"Invalid value: Record([("a", Boolean(false)), ("b", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Unsupported value-schema combination! Value: Boolean(false), schema: Long"#, ); @@ -1712,7 +1718,7 @@ mod types { ("a".to_string(), Value::Long(42i64)), ("c".to_string(), Value::String("foo".to_string())), ]); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( r#"Invalid value: Record([("a", Long(42)), ("c", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Could not find matching type in union"#, ); @@ -1724,7 +1730,7 @@ mod types { ("a".to_string(), Value::Long(42i64)), ("d".to_string(), Value::String("foo".to_string())), ]); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( r#"Invalid value: Record([("a", Long(42)), ("d", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: There is no schema field for field 'd'"#, ); @@ -1735,7 +1741,7 @@ mod types { ("c".to_string(), Value::Null), ("d".to_string(), Value::Null), ]); - assert!(!value.validate(&schema)); + assert!(!value.validate(&schema).await); assert_logged( r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")), ("c", Null), ("d", Null)]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: The value's records length (4) is greater than the schema's (3 fields)"#, ); @@ -1750,6 +1756,7 @@ mod types { .collect() ) .validate(&schema) + .await ); assert!( @@ -1759,6 +1766,7 @@ mod types { .collect() ) .validate(&schema) + .await ); assert_logged( r#"Invalid value: Map({"d": Long(123)}) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Field with name '"a"' is not a member of the map items @@ -1776,6 +1784,7 @@ Field with name '"b"' is not a member of the map items"#, ])) ) .validate(&union_schema) + .await ); assert!( @@ -1791,6 +1800,7 @@ Field with name '"b"' is not a member of the map items"#, )) ) .validate(&union_schema) + .await ); Ok(()) @@ -2077,7 +2087,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let value = Value::Record(vec![( "event".to_string(), @@ -2114,11 +2125,11 @@ Field with name '"b"' is not a member of the map items"#, ); assert_eq!( serde_json::Value::try_from(Value::Float(1.0))?, - serde_json::Value::Number(Number::from_f64(1.0).unwrap()) + serde_json::Value::Number(serde_json::Number::from_f64(1.0).unwrap()) ); assert_eq!( serde_json::Value::try_from(Value::Double(1.0))?, - serde_json::Value::Number(Number::from_f64(1.0).unwrap()) + serde_json::Value::Number(serde_json::Number::from_f64(1.0).unwrap()) ); assert_eq!( serde_json::Value::try_from(Value::Bytes(vec![1, 2, 3]))?, @@ -2300,7 +2311,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -2344,7 +2356,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -2391,7 +2404,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -2442,7 +2456,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![( @@ -2487,7 +2502,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -2531,7 +2547,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); @@ -2595,7 +2612,7 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -2687,7 +2704,7 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -2780,7 +2797,7 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( @@ -2854,7 +2871,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let inner_value_right = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value_wrong1 = Value::Record(vec![("z".into(), Value::Null)]); @@ -2871,11 +2889,11 @@ Field with name '"b"' is not a member of the map items"#, ]); assert!( - !outer1.validate(&schema), + !outer1.validate(&schema).await, "field b record is invalid against the schema" ); // this should pass, but doesn't assert!( - !outer2.validate(&schema), + !outer2.validate(&schema).await, "field b record is invalid against the schema" ); // this should pass, but doesn't @@ -2933,7 +2951,8 @@ Field with name '"b"' is not a member of the map items"#, } ] }"#, - )?; + ) + .await?; let test_inner = TestInner { z: 3 }; let test_outer1 = TestRefSchemaStruct1 { @@ -2957,15 +2976,15 @@ Field with name '"b"' is not a member of the map items"#, let test_outer3: Value = test_outer3.serialize(&mut ser)?; assert!( - !test_outer1.validate(&schema), + !test_outer1.validate(&schema).await, "field b record is invalid against the schema" ); assert!( - !test_outer2.validate(&schema), + !test_outer2.validate(&schema).await, "field b record is invalid against the schema" ); assert!( - !test_outer3.validate(&schema), + !test_outer3.validate(&schema).await, "field b record is invalid against the schema" ); @@ -2973,7 +2992,7 @@ Field with name '"b"' is not a member of the map items"#, } async fn avro_3674_with_or_without_namespace(with_namespace: bool) -> TestResult { - use crate::ser::Serializer; + use crate::ser::tokio::Serializer; use serde::Serialize; let schema_str = r#" @@ -3016,7 +3035,7 @@ Field with name '"b"' is not a member of the map items"#, }, ); - let schema = Schema::parse_str(&schema_str)?; + let schema = Schema::parse_str(&schema_str).await?; #[derive(Serialize)] enum EnumType { @@ -3046,7 +3065,10 @@ Field with name '"b"' is not a member of the map items"#, let mut ser = Serializer::default(); let test_value: Value = msg.serialize(&mut ser)?; - assert!(test_value.validate(&schema), "test_value should validate"); + assert!( + test_value.validate(&schema).await, + "test_value should validate" + ); assert!( test_value.resolve(&schema).await.is_ok(), "test_value should resolve" @@ -3066,7 +3088,6 @@ Field with name '"b"' is not a member of the map items"#, } async fn avro_3688_schema_resolution_panic(set_field_b: bool) -> TestResult { - use crate::ser::Serializer; use serde::{Deserialize, Serialize}; let schema_str = r#"{ @@ -3112,7 +3133,7 @@ Field with name '"b"' is not a member of the map items"#, field_b: Option, } - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; let msg = Message { field_a: Some(Inner { @@ -3129,7 +3150,10 @@ Field with name '"b"' is not a member of the map items"#, let mut ser = Serializer::default(); let test_value: Value = msg.serialize(&mut ser)?; - assert!(test_value.validate(&schema), "test_value should validate"); + assert!( + test_value.validate(&schema).await, + "test_value should validate" + ); assert!( test_value.resolve(&schema).await.is_ok(), "test_value should resolve" @@ -3164,7 +3188,7 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::from(value); - let schemas = Schema::parse_list([main_schema, referenced_schema])?; + let schemas = Schema::parse_list([main_schema, referenced_schema]).await?; let main_schema = schemas.first().unwrap(); let schemata: Vec<_> = schemas.iter().skip(1).collect(); @@ -3207,7 +3231,8 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::from(value); - let schemata = Schema::parse_list([referenced_enum, referenced_record, main_schema])?; + let schemata = + Schema::parse_list([referenced_enum, referenced_record, main_schema]).await?; let main_schema = schemata.last().unwrap(); let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); @@ -3222,7 +3247,9 @@ Field with name '"b"' is not a member of the map items"#, ); assert!( - resolve_result?.validate_schemata(schemata.iter().collect()), + resolve_result? + .validate_schemata(schemata.iter().collect()) + .await, "result of validation with schemata should be true" ); @@ -3236,7 +3263,7 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::Decimal(Decimal::from( BigInt::from(12345678u32).to_signed_bytes_be(), )); - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let resolve_result = avro_value.resolve(&schema).await; assert!( resolve_result.is_ok(), @@ -3252,7 +3279,7 @@ Field with name '"b"' is not a member of the map items"#, r#"{"name": "bigDecimalSchema", "logicalType": "big-decimal", "type": "bytes" }"#; let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let resolve_result: AvroResult = avro_value.resolve(&schema).await; assert!( resolve_result.is_ok(), @@ -3359,7 +3386,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "double"}"#)?; + let schema = Schema::parse_str(r#"{"type": "double"}"#).await?; let value = Value::String("unknown".to_owned()); match value.resolve(&schema).await.map_err(Error::into_details) { Err(err @ Details::GetDouble(_)) => { @@ -3377,7 +3404,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "float"}"#)?; + let schema = Schema::parse_str(r#"{"type": "float"}"#).await?; let value = Value::String("unknown".to_owned()); match value.resolve(&schema).await.map_err(Error::into_details) { Err(err @ Details::GetFloat(_)) => { @@ -3529,7 +3556,7 @@ Field with name '"b"' is not a member of the map items"#, ]; for (schema_str, value, expected_error) in data { - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; match value.resolve(&schema).await { Err(error) => { assert_eq!(format!("{error}"), expected_error); @@ -3562,7 +3589,7 @@ Field with name '"b"' is not a member of the map items"#, } "#; - let schema = Schema::parse_str(schema)?; + let schema = Schema::parse_str(schema).await?; let mut record = Record::new(&schema).unwrap(); record.put("foo", "hello"); record.put("bar", 123_i64); diff --git a/avro/src/validator.rs b/avro/src/validator.rs index 553e3cdf..1f194ee7 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -270,7 +270,7 @@ mod validator { #[cfg(test)] mod tests { use super::*; - use crate::schema::Name; + use crate::schema::tokio::Name; use apache_avro_test_helper::TestResult; #[test] diff --git a/avro/src/writer.rs b/avro/src/writer.rs index a475fde9..e37c72e1 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -33,6 +33,7 @@ crate::headers::tokio => crate::headers::sync, crate::schema::tokio => crate::schema::sync, crate::ser_schema::tokio => crate::ser_schema::sync, + crate::reader::tokio => crate::reader::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, crate::schema_equality::tokio => crate::schema_equality::sync, @@ -650,8 +651,8 @@ mod writer { where T: AvroSchema, { - pub fn with_capacity(buffer_cap: usize) -> AvroResult> { - let schema = T::get_schema(); + pub async fn with_capacity(buffer_cap: usize) -> AvroResult> { + let schema = T::get_schema().await; Ok(SpecificSingleObjectWriter { inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, schema, @@ -832,20 +833,20 @@ mod writer { use super::*; use crate::{ - Reader, - decimal::Decimal, + decimal::tokio::Decimal, duration::{Days, Duration, Millis, Months}, - headers::GlueSchemaUuidHeader, + headers::tokio::GlueSchemaUuidHeader, rabin::Rabin, - schema::{DecimalSchema, FixedSchema, Name}, - types::Record, - util::zig_i64, + reader::tokio::{Reader, from_avro_datum}, + schema::tokio::{DecimalSchema, FixedSchema, Name}, + types::tokio::Record, + util::tokio::zig_i64, }; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; use uuid::Uuid; - use crate::{codec::DeflateSettings, error::Details}; + use crate::{codec::tokio::DeflateSettings, error::tokio::Details}; use apache_avro_test_helper::TestResult; const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); @@ -870,9 +871,9 @@ mod writer { const UNION_SCHEMA: &str = r#"["null", "long"]"#; - #[test] - fn test_to_avro_datum() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_to_avro_datum() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut record = Record::new(&schema).unwrap(); record.put("a", 27i64); record.put("b", "foo"); @@ -882,20 +883,20 @@ mod writer { zig_i64(3, &mut expected)?; expected.extend([b'f', b'o', b'o']); - assert_eq!(to_avro_datum(&schema, record)?, expected); + assert_eq!(to_avro_datum(&schema, record).await?, expected); Ok(()) } - #[test] - fn avro_rs_193_write_avro_datum_ref() -> TestResult { + #[tokio::test] + async fn avro_rs_193_write_avro_datum_ref() -> TestResult { #[derive(Serialize)] struct TestStruct { a: i64, b: String, } - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let mut writer: Vec = Vec::new(); let data = TestStruct { a: 27, @@ -915,9 +916,9 @@ mod writer { Ok(()) } - #[test] - fn avro_rs_220_flush_write_header() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn avro_rs_220_flush_write_header() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; // By default flush should write the header even if nothing was added yet let mut writer = Writer::new(&schema, Vec::new()); @@ -938,34 +939,34 @@ mod writer { Ok(()) } - #[test] - fn test_union_not_null() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA)?; + #[tokio::test] + async fn test_union_not_null() -> TestResult { + let schema = Schema::parse_str(UNION_SCHEMA).await?; let union = Value::Union(1, Box::new(Value::Long(3))); let mut expected = Vec::new(); zig_i64(1, &mut expected)?; zig_i64(3, &mut expected)?; - assert_eq!(to_avro_datum(&schema, union)?, expected); + assert_eq!(to_avro_datum(&schema, union).await?, expected); Ok(()) } - #[test] - fn test_union_null() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA)?; + #[tokio::test] + async fn test_union_null() -> TestResult { + let schema = Schema::parse_str(UNION_SCHEMA).await?; let union = Value::Union(0, Box::new(Value::Null)); let mut expected = Vec::new(); zig_i64(0, &mut expected)?; - assert_eq!(to_avro_datum(&schema, union)?, expected); + assert_eq!(to_avro_datum(&schema, union).await?, expected); Ok(()) } - fn logical_type_test + Clone>( + async fn logical_type_test + Clone>( schema_str: &'static str, expected_schema: &Schema, @@ -974,22 +975,22 @@ mod writer { raw_schema: &Schema, raw_value: T, ) -> TestResult { - let schema = Schema::parse_str(schema_str)?; + let schema = Schema::parse_str(schema_str).await?; assert_eq!(&schema, expected_schema); // The serialized format should be the same as the schema. - let ser = to_avro_datum(&schema, value.clone())?; - let raw_ser = to_avro_datum(raw_schema, raw_value)?; + let ser = to_avro_datum(&schema, value.clone()).await?; + let raw_ser = to_avro_datum(raw_schema, raw_value).await?; assert_eq!(ser, raw_ser); // Should deserialize from the schema into the logical type. let mut r = ser.as_slice(); - let de = crate::from_avro_datum(&schema, &mut r, None)?; + let de = from_avro_datum(&schema, &mut r, None).await?; assert_eq!(de, value); Ok(()) } - #[test] - fn date() -> TestResult { + #[tokio::test] + async fn date() -> TestResult { logical_type_test( r#"{"type": "int", "logicalType": "date"}"#, &Schema::Date, @@ -997,10 +998,11 @@ mod writer { &Schema::Int, 1_i32, ) + .await } - #[test] - fn time_millis() -> TestResult { + #[tokio::test] + async fn time_millis() -> TestResult { logical_type_test( r#"{"type": "int", "logicalType": "time-millis"}"#, &Schema::TimeMillis, @@ -1008,10 +1010,11 @@ mod writer { &Schema::Int, 1_i32, ) + .await } - #[test] - fn time_micros() -> TestResult { + #[tokio::test] + async fn time_micros() -> TestResult { logical_type_test( r#"{"type": "long", "logicalType": "time-micros"}"#, &Schema::TimeMicros, @@ -1019,10 +1022,11 @@ mod writer { &Schema::Long, 1_i64, ) + .await } - #[test] - fn timestamp_millis() -> TestResult { + #[tokio::test] + async fn timestamp_millis() -> TestResult { logical_type_test( r#"{"type": "long", "logicalType": "timestamp-millis"}"#, &Schema::TimestampMillis, @@ -1030,10 +1034,11 @@ mod writer { &Schema::Long, 1_i64, ) + .await } - #[test] - fn timestamp_micros() -> TestResult { + #[tokio::test] + async fn timestamp_micros() -> TestResult { logical_type_test( r#"{"type": "long", "logicalType": "timestamp-micros"}"#, &Schema::TimestampMicros, @@ -1041,10 +1046,11 @@ mod writer { &Schema::Long, 1_i64, ) + .await } - #[test] - fn decimal_fixed() -> TestResult { + #[tokio::test] + async fn decimal_fixed() -> TestResult { let size = 30; let inner = Schema::Fixed(FixedSchema { name: Name::new("decimal")?, @@ -1065,11 +1071,11 @@ mod writer { Value::Decimal(Decimal::from(value.clone())), &inner, Value::Fixed(size, value), - ) + ).await } - #[test] - fn decimal_bytes() -> TestResult { + #[tokio::test] + async fn decimal_bytes() -> TestResult { let inner = Schema::Bytes; let value = vec![0u8; 10]; logical_type_test( @@ -1083,10 +1089,11 @@ mod writer { &inner, value, ) + .await } - #[test] - fn duration() -> TestResult { + #[tokio::test] + async fn duration() -> TestResult { let inner = Schema::Fixed(FixedSchema { name: Name::new("duration")?, aliases: None, @@ -1106,20 +1113,20 @@ mod writer { value, &inner, Value::Fixed(12, vec![0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0]), - ) + ).await } - #[test] - fn test_writer_append() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_writer_append() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut record = Record::new(&schema).unwrap(); record.put("a", 27i64); record.put("b", "foo"); - let n1 = writer.append(record.clone())?; - let n2 = writer.append(record.clone())?; + let n1 = writer.append(record.clone()).await?; + let n2 = writer.append(record.clone()).await?; let n3 = writer.flush()?; let result = writer.into_inner()?; @@ -1143,9 +1150,9 @@ mod writer { Ok(()) } - #[test] - fn test_writer_extend() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_writer_extend() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut record = Record::new(&schema).unwrap(); @@ -1154,7 +1161,7 @@ mod writer { let record_copy = record.clone(); let records = vec![record, record_copy]; - let n1 = writer.extend(records)?; + let n1 = writer.extend(records).await?; let n2 = writer.flush()?; let result = writer.into_inner()?; @@ -1184,9 +1191,9 @@ mod writer { b: String, } - #[test] - fn test_writer_append_ser() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_writer_append_ser() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let record = TestSerdeSerialize { @@ -1217,9 +1224,9 @@ mod writer { Ok(()) } - #[test] - fn test_writer_extend_ser() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_writer_extend_ser() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let record = TestSerdeSerialize { @@ -1270,13 +1277,13 @@ mod writer { .build() } - fn check_writer(mut writer: Writer<'_, Vec>, schema: &Schema) -> TestResult { + async fn check_writer(mut writer: Writer<'_, Vec>, schema: &Schema) -> TestResult { let mut record = Record::new(schema).unwrap(); record.put("a", 27i64); record.put("b", "foo"); - let n1 = writer.append(record.clone())?; - let n2 = writer.append(record.clone())?; + let n1 = writer.append(record.clone()).await?; + let n2 = writer.append(record.clone()).await?; let n3 = writer.flush()?; let result = writer.into_inner()?; @@ -1301,22 +1308,22 @@ mod writer { Ok(()) } - #[test] - fn test_writer_with_codec() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_writer_with_codec() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let writer = make_writer_with_codec(&schema); - check_writer(writer, &schema) + check_writer(writer, &schema).await } - #[test] - fn test_writer_with_builder() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_writer_with_builder() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let writer = make_writer_with_builder(&schema); - check_writer(writer, &schema) + check_writer(writer, &schema).await } - #[test] - fn test_logical_writer() -> TestResult { + #[tokio::test] + async fn test_logical_writer() -> TestResult { const LOGICAL_TYPE_SCHEMA: &str = r#" { "type": "record", @@ -1336,7 +1343,7 @@ mod writer { } "#; let codec = Codec::Deflate(DeflateSettings::default()); - let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA)?; + let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA).await?; let mut writer = Writer::builder() .schema(&schema) .codec(codec) @@ -1352,8 +1359,8 @@ mod writer { let mut record2 = Record::new(&schema).unwrap(); record2.put("a", Value::Union(0, Box::new(Value::Null))); - let n1 = writer.append(record1)?; - let n2 = writer.append(record2)?; + let n1 = writer.append(record1).await?; + let n2 = writer.append(record2).await?; let n3 = writer.flush()?; let result = writer.into_inner()?; @@ -1380,9 +1387,9 @@ mod writer { Ok(()) } - #[test] - fn test_avro_3405_writer_add_metadata_success() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_avro_3405_writer_add_metadata_success() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); writer.add_user_metadata("stringKey".to_string(), String::from("stringValue"))?; @@ -1394,8 +1401,8 @@ mod writer { record.put("a", 27i64); record.put("b", "foo"); - writer.append(record.clone())?; - writer.append(record.clone())?; + writer.append(record.clone()).await?; + writer.append(record.clone()).await?; writer.flush()?; let result = writer.into_inner()?; @@ -1404,14 +1411,14 @@ mod writer { Ok(()) } - #[test] - fn test_avro_3881_metadata_empty_body() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_avro_3881_metadata_empty_body() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); writer.add_user_metadata("a".to_string(), "b")?; let result = writer.into_inner()?; - let reader = Reader::with_schema(&schema, &result[..])?; + let reader = Reader::with_schema(&schema, &result[..]).await?; let mut expected = HashMap::new(); expected.insert("a".to_string(), vec![b'b']); assert_eq!(reader.user_metadata(), &expected); @@ -1420,15 +1427,15 @@ mod writer { Ok(()) } - #[test] - fn test_avro_3405_writer_add_metadata_failure() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_avro_3405_writer_add_metadata_failure() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut record = Record::new(&schema).unwrap(); record.put("a", 27i64); record.put("b", "foo"); - writer.append(record.clone())?; + writer.append(record.clone()).await?; match writer .add_user_metadata("stringKey".to_string(), String::from("value2")) @@ -1446,9 +1453,9 @@ mod writer { Ok(()) } - #[test] - fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let key = "avro.stringKey".to_string(); @@ -1477,9 +1484,9 @@ mod writer { Ok(()) } - #[test] - fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult { - let schema = Schema::parse_str(SCHEMA)?; + #[tokio::test] + async fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult { + let schema = Schema::parse_str(SCHEMA).await?; let mut user_meta_data: HashMap = HashMap::new(); user_meta_data.insert( @@ -1508,7 +1515,7 @@ mod writer { } impl AvroSchema for TestSingleObjectWriter { - fn get_schema() -> Schema { + async fn get_schema() -> Schema { let schema = r#" { "type":"record", @@ -1532,7 +1539,7 @@ mod writer { ] } "#; - Schema::parse_str(schema).unwrap() + Schema::parse_str(schema).await.unwrap() } } @@ -1549,8 +1556,8 @@ mod writer { } } - #[test] - fn test_single_object_writer() -> TestResult { + #[tokio::test] + async fn test_single_object_writer() -> TestResult { let mut buf: Vec = Vec::new(); let obj = TestSingleObjectWriter { a: 300, @@ -1565,6 +1572,7 @@ mod writer { let value = obj.into(); let written_bytes = writer .write_value_ref(&value, &mut buf) + .await .expect("Error serializing properly"); assert!(buf.len() > 10, "no bytes written"); @@ -1589,8 +1597,8 @@ mod writer { Ok(()) } - #[test] - fn test_single_object_writer_with_header_builder() -> TestResult { + #[tokio::test] + async fn test_single_object_writer_with_header_builder() -> TestResult { let mut buf: Vec = Vec::new(); let obj = TestSingleObjectWriter { a: 300, @@ -1608,6 +1616,7 @@ mod writer { let value = obj.into(); writer .write_value_ref(&value, &mut buf) + .await .expect("Error serializing properly"); assert_eq!(buf[0], 0x03); @@ -1616,8 +1625,8 @@ mod writer { Ok(()) } - #[test] - fn test_writer_parity() -> TestResult { + #[tokio::test] + async fn test_writer_parity() -> TestResult { let obj1 = TestSingleObjectWriter { a: 300, b: 34.555, @@ -1641,9 +1650,11 @@ mod writer { .expect("Serialization expected"); specific_writer .write_value(obj1.clone(), &mut buf2) + .await .expect("Serialization expected"); generic_writer .write_value(obj1.into(), &mut buf3) + .await .expect("Serialization expected"); assert_eq!(buf1, buf2); assert_eq!(buf1, buf3); @@ -1651,8 +1662,8 @@ mod writer { Ok(()) } - #[test] - fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { + #[tokio::test] + async fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { const SCHEMA: &str = r#" { "type": "record", @@ -1674,7 +1685,7 @@ mod writer { time: Some(1234567890), }; - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let bytes = writer.append_ser(conf)?; @@ -1684,8 +1695,8 @@ mod writer { Ok(()) } - #[test] - fn avro_4014_validation_returns_a_detailed_error() -> TestResult { + #[tokio::test] + async fn avro_4014_validation_returns_a_detailed_error() -> TestResult { const SCHEMA: &str = r#" { "type": "record", @@ -1707,7 +1718,7 @@ mod writer { time: Some(12345678.90), }; - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); match writer.append_ser(conf) { @@ -1722,8 +1733,8 @@ mod writer { Ok(()) } - #[test] - fn avro_4063_flush_applies_to_inner_writer() -> TestResult { + #[tokio::test] + async fn avro_4063_flush_applies_to_inner_writer() -> TestResult { const SCHEMA: &str = r#" { "type": "record", @@ -1757,14 +1768,14 @@ mod writer { let buffered_writer = std::io::BufWriter::new(shared_buffer.clone()); - let schema = Schema::parse_str(SCHEMA)?; + let schema = Schema::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, buffered_writer); let mut record = Record::new(writer.schema()).unwrap(); record.put("exampleField", "value"); - writer.append(record)?; + writer.append(record).await?; writer.flush()?; assert_eq!( From 3525a6b273a349f0aa42825194ee6b0d08a3ac4b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 6 Aug 2025 16:45:18 +0300 Subject: [PATCH 14/47] All IT tests and examples use `sync` feature for now Signed-off-by: Martin Tzvetanov Grigorov --- avro/Cargo.toml | 100 ++++++++++++++++++ avro/examples/benchmark.rs | 4 +- avro/examples/generate_interop_data.rs | 4 +- .../test_interop_single_object_encoding.rs | 2 +- avro/src/bigdecimal.rs | 14 ++- avro/src/reader.rs | 4 +- avro/src/schema.rs | 1 + avro/src/types.rs | 2 +- avro/src/writer.rs | 13 +-- avro/tests/avro-3787.rs | 4 +- avro/tests/codecs.rs | 2 +- avro/tests/io.rs | 2 +- avro/tests/schema.rs | 7 +- avro/tests/shared.rs | 2 +- avro/tests/to_from_avro_datum_schemata.rs | 2 +- avro/tests/validators.rs | 4 +- 16 files changed, 136 insertions(+), 31 deletions(-) diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 5577fadc..c5ed1848 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -104,3 +104,103 @@ rstest = { default-features = false, version = "0.26.1" } [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] + +[[example]] +name = "benchmark" +path = "examples/benchmark.rs" +required-features = ["sync"] + +[[example]] +name = "generate_interop_data" +path = "examples/generate_interop_data.rs" +required-features = ["sync"] + +[[example]] +name = "specific_single_object" +path = "examples/specific_single_object.rs" +required-features = ["sync"] + +[[example]] +name = "test_interop_data" +path = "examples/test_interop_data.rs" +required-features = ["sync"] + +[[example]] +name = "test_interop_single_object_encoding" +path = "examples/test_interop_single_object_encoding.rs" +required-features = ["sync"] + +[[example]] +name = "to_value" +path = "examples/to_value.rs" +required-features = ["sync"] + +[[test]] +name = "append_to_existing" +path = "tests/append_to_existing.rs" +required-features = ["sync"] + +[[test]] +name = "avro-3786" +path = "tests/avro-3786.rs" +required-features = ["sync"] + +[[test]] +name = "avro-3787" +path = "tests/avro-3787.rs" +required-features = ["sync"] + +[[test]] +name = "avro-rs-219" +path = "tests/avro-rs-219.rs" +required-features = ["sync"] + +[[test]] +name = "avro-rs-226" +path = "tests/avro-rs-226.rs" +required-features = ["sync"] + +[[test]] +name = "big_decimal" +path = "tests/big_decimal.rs" +required-features = ["sync"] + +[[test]] +name = "codecs" +path = "tests/codecs.rs" +required-features = ["sync"] + +[[test]] +name = "io" +path = "tests/io.rs" +required-features = ["sync"] + +[[test]] +name = "schema" +path = "tests/schema.rs" +required-features = ["sync"] + +[[test]] +name = "shared" +path = "tests/shared.rs" +required-features = ["sync"] + +[[test]] +name = "to_from_avro_datum_schemata" +path = "tests/to_from_avro_datum_schemata.rs" +required-features = ["sync"] + +[[test]] +name = "union_schema" +path = "tests/union_schema.rs" +required-features = ["sync"] + +[[test]] +name = "uuids" +path = "tests/uuids.rs" +required-features = ["sync"] + +[[test]] +name = "validatrs" +path = "tests/validators.rs" +required-features = ["sync"] diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index b604ab45..68bd0fb7 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -17,8 +17,8 @@ use apache_avro::{ Reader, Writer, - schema::Schema, - types::{Record, Value}, + Schema, + types::sync::{Record, Value}, }; use apache_avro_test_helper::TestResult; use std::{ diff --git a/avro/examples/generate_interop_data.rs b/avro/examples/generate_interop_data.rs index 7b3c369b..6d5558ee 100644 --- a/avro/examples/generate_interop_data.rs +++ b/avro/examples/generate_interop_data.rs @@ -17,8 +17,8 @@ use apache_avro::{ Codec, Writer, - schema::Schema, - types::{Record, Value}, + Schema, + types::sync::{Record, Value}, }; use std::{ collections::HashMap, diff --git a/avro/examples/test_interop_single_object_encoding.rs b/avro/examples/test_interop_single_object_encoding.rs index 61b1e6f9..432be81c 100644 --- a/avro/examples/test_interop_single_object_encoding.rs +++ b/avro/examples/test_interop_single_object_encoding.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{schema::AvroSchema, types::Value}; +use apache_avro::{AvroSchema, types::sync::Value}; use std::error::Error; struct InteropMessage; diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 38420c2f..5eabb420 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -106,11 +106,17 @@ mod bigdecimal { use bigdecimal::{One, Zero}; use pretty_assertions::assert_eq; use std::{ - fs::File, - io::BufReader, ops::{Div, Mul}, str::FromStr, }; + #[synca::cfg(sync)] + use std::fs::File; + #[synca::cfg(tokio)] + use tokio::fs::File; + #[synca::cfg(sync)] + use std::io::BufReader; + #[synca::cfg(tokio)] + use tokio::io::BufReader; #[tokio::test] async fn test_avro_3779_bigdecimal_serial() -> TestResult { @@ -217,8 +223,8 @@ mod bigdecimal { async fn test_avro_3779_from_java_file() -> TestResult { // Open file generated with Java code to ensure compatibility // with Java big decimal logical type. - let file: File = File::open("./tests/bigdec.avro")?; - let mut reader = Reader::new(BufReader::new(&file)).await?; + let file = File::open("./tests/bigdec.avro").await?; + let mut reader = Reader::new(BufReader::new(file)).await?; let next_element = reader.next().await; assert!(next_element.is_some()); let value = next_element.unwrap()?; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 08ae037c..4b420f2d 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -932,9 +932,7 @@ mod reader { c: Vec, } - #[synca::cfg(tokio)] - use async_trait::async_trait; - #[cfg_attr(feature = "tokio", async_trait)] + #[cfg_attr(feature = "tokio", async_trait::async_trait)] impl AvroSchema for TestSingleObjectReader { async fn get_schema() -> Schema { let schema = r#" diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 1ebfce85..5dae3d5d 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -6919,6 +6919,7 @@ mod schema { id: Uuid, } + #[cfg_attr(feature = "tokio", async_trait::async_trait)] impl AvroSchema for Comment { async fn get_schema() -> Schema { Schema::parse_str( diff --git a/avro/src/types.rs b/avro/src/types.rs index f071ffe6..d09de72f 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -1617,7 +1617,7 @@ mod types { }); let value = Value::Enum(0, "spades".to_string()); - assert!(!value.validate(&other_schema)); + assert!(!value.validate(&other_schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", diff --git a/avro/src/writer.rs b/avro/src/writer.rs index e37c72e1..b084ba17 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -1514,6 +1514,7 @@ mod writer { c: Vec, } + #[cfg_attr(feature = "tokio", async_trait::async_trait)] impl AvroSchema for TestSingleObjectWriter { async fn get_schema() -> Schema { let schema = r#" @@ -1565,7 +1566,7 @@ mod writer { c: vec!["cat".into(), "dog".into()], }; let mut writer = GenericSingleObjectWriter::new_with_capacity( - &TestSingleObjectWriter::get_schema(), + &TestSingleObjectWriter::get_schema().await, 1024, ) .expect("Should resolve schema"); @@ -1581,14 +1582,14 @@ mod writer { assert_eq!(buf[1], 0x01); assert_eq!( &buf[2..10], - &TestSingleObjectWriter::get_schema() + &TestSingleObjectWriter::get_schema().await .fingerprint::() .bytes[..] ); let mut msg_binary = Vec::new(); encode( &value, - &TestSingleObjectWriter::get_schema(), + &TestSingleObjectWriter::get_schema().await, &mut msg_binary, ) .expect("encode should have failed by here as a dependency of any writing"); @@ -1608,7 +1609,7 @@ mod writer { let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder( - &TestSingleObjectWriter::get_schema(), + &TestSingleObjectWriter::get_schema().await, 1024, header_builder, ) @@ -1638,12 +1639,12 @@ mod writer { let mut buf3: Vec = Vec::new(); let mut generic_writer = GenericSingleObjectWriter::new_with_capacity( - &TestSingleObjectWriter::get_schema(), + &TestSingleObjectWriter::get_schema().await, 1024, ) .expect("Should resolve schema"); let mut specific_writer = - SpecificSingleObjectWriter::::with_capacity(1024) + SpecificSingleObjectWriter::::with_capacity(1024).await .expect("Resolved should pass"); specific_writer .write(obj1.clone(), &mut buf1) diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index 827765cb..878fb5ca 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::schema::tokio::Schema; -use apache_avro::types::tokio::Value; +use apache_avro::Schema; +use apache_avro::types::sync::Value; use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; diff --git a/avro/tests/codecs.rs b/avro/tests/codecs.rs index f58f7706..866eaf86 100644 --- a/avro/tests/codecs.rs +++ b/avro/tests/codecs.rs @@ -17,7 +17,7 @@ use apache_avro::{ Codec, DeflateSettings, Reader, Schema, Writer, - types::{Record, Value}, + types::sync::{Record, Value}, }; use apache_avro_test_helper::TestResult; use miniz_oxide::deflate::CompressionLevel; diff --git a/avro/tests/io.rs b/avro/tests/io.rs index ef7dc2db..f29d8d9e 100644 --- a/avro/tests/io.rs +++ b/avro/tests/io.rs @@ -16,7 +16,7 @@ // under the License. //! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py -use apache_avro::{Error, Schema, error::Details, from_avro_datum, to_avro_datum, types::Value}; +use apache_avro::{Error, Schema, error::sync::Details, from_avro_datum, to_avro_datum, types::sync::Value}; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use std::{io::Cursor, sync::OnceLock}; diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index a911ad33..cb49b270 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -22,11 +22,11 @@ use std::{ use apache_avro::{ Codec, Reader, Writer, - error::tokio::{Details, Error}, + error::sync::{Details, Error}, from_avro_datum, from_value, - schema::tokio::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, + schema::sync::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, to_avro_datum, to_value, - types::tokio::{Record, Value}, + types::sync::{Record, Value}, }; use apache_avro_test_helper::{ TestResult, @@ -885,7 +885,6 @@ fn avro_old_issue_47() -> TestResult { #[test] fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_type() -> TestResult { - use apache_avro::{from_avro_datum, to_avro_datum, types::tokio::Value}; use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] diff --git a/avro/tests/shared.rs b/avro/tests/shared.rs index c9397dfe..c7954a8d 100644 --- a/avro/tests/shared.rs +++ b/avro/tests/shared.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, Reader, Schema, Writer, types::Value}; +use apache_avro::{Codec, Reader, Schema, Writer, types::sync::Value}; use apache_avro_test_helper::TestResult; use std::{ fmt, diff --git a/avro/tests/to_from_avro_datum_schemata.rs b/avro/tests/to_from_avro_datum_schemata.rs index c300c0fe..7ab6f217 100644 --- a/avro/tests/to_from_avro_datum_schemata.rs +++ b/avro/tests/to_from_avro_datum_schemata.rs @@ -17,7 +17,7 @@ use apache_avro::{ Codec, Reader, Schema, Writer, from_avro_datum_reader_schemata, from_avro_datum_schemata, - to_avro_datum_schemata, types::Value, + to_avro_datum_schemata, types::sync::Value, }; use apache_avro_test_helper::{TestResult, init}; diff --git a/avro/tests/validators.rs b/avro/tests/validators.rs index ab372486..cd24d24d 100644 --- a/avro/tests/validators.rs +++ b/avro/tests/validators.rs @@ -17,8 +17,8 @@ use apache_avro::{ AvroResult, - schema::Namespace, - validator::{ + schema::sync::Namespace, + validator::sync::{ EnumSymbolNameValidator, RecordFieldNameValidator, SchemaNameValidator, SchemaNamespaceValidator, set_enum_symbol_name_validator, set_record_field_name_validator, set_schema_name_validator, set_schema_namespace_validator, From 784ddcaf37f9fd0ea97b94efd53f91e44060260d Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 8 Aug 2025 09:52:14 +0300 Subject: [PATCH 15/47] Fix the imports for append_to_existing.rs IT test Signed-off-by: Martin Tzvetanov Grigorov --- avro/tests/append_to_existing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avro/tests/append_to_existing.rs b/avro/tests/append_to_existing.rs index e9af4b30..e4993b8c 100644 --- a/avro/tests/append_to_existing.rs +++ b/avro/tests/append_to_existing.rs @@ -17,7 +17,7 @@ use apache_avro::{ AvroResult, Reader, Schema, Writer, read_marker, - types::{Record, Value}, + types::sync::{Record, Value}, }; use apache_avro_test_helper::TestResult; From a5e46eb489e3b03a97106153321e9e8c9694a2df Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 8 Aug 2025 15:06:33 +0300 Subject: [PATCH 16/47] WIP: Implement futures::Stream for Reader It is not finished - somehow two lines of code lead rustc to run forever Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 10 ++-- avro/src/lib.rs | 10 ++-- avro/src/reader.rs | 85 +++++++++++++++++++++----------- avro/src/schema_compatibility.rs | 10 ++-- avro/src/writer.rs | 12 +++-- 5 files changed, 83 insertions(+), 44 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 5eabb420..a685b6f0 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -104,17 +104,19 @@ mod bigdecimal { }; use apache_avro_test_helper::TestResult; use bigdecimal::{One, Zero}; + #[synca::cfg(tokio)] + use futures::StreamExt; use pretty_assertions::assert_eq; + #[synca::cfg(sync)] + use std::fs::File; + #[synca::cfg(sync)] + use std::io::BufReader; use std::{ ops::{Div, Mul}, str::FromStr, }; - #[synca::cfg(sync)] - use std::fs::File; #[synca::cfg(tokio)] use tokio::fs::File; - #[synca::cfg(sync)] - use std::io::BufReader; #[synca::cfg(tokio)] use tokio::io::BufReader; diff --git a/avro/src/lib.rs b/avro/src/lib.rs index f829bb30..f13b699c 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -961,9 +961,9 @@ pub type AvroResult = Result; #[synca::synca( #[cfg(feature = "tokio")] - pub mod tokio_tests { }, + pub mod tokio { }, #[cfg(feature = "sync")] - pub mod sync_tests { + pub mod sync { sync!(); replace!( crate::codec::tokio => crate::codec::sync, @@ -984,6 +984,8 @@ mod tests { types::tokio::{Record, Value}, writer::tokio::Writer, }; + // #[synca::cfg(tokio)] + use futures::StreamExt; use pretty_assertions::assert_eq; //TODO: move where it fits better @@ -1072,14 +1074,14 @@ mod tests { let input = writer.into_inner().unwrap(); let mut reader = Reader::with_schema(&schema, &input[..]).await.unwrap(); assert_eq!( - reader.next().unwrap().unwrap(), + reader.next().await.unwrap().unwrap(), Value::Record(vec![ ("a".to_string(), Value::Long(27)), ("b".to_string(), Value::String("foo".to_string())), ("c".to_string(), Value::Enum(2, "clubs".to_string())), ]) ); - assert!(reader.next().is_none()); + assert!(reader.next().await.is_none()); } //TODO: move where it fits better diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 4b420f2d..99e77290 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -360,7 +360,7 @@ mod reader { /// Main interface for reading Avro formatted values. /// - /// To be used as an iterator: + /// To be used as an iterator/Stream: /// /// ```no_run /// use apache_avro::Reader; @@ -380,6 +380,8 @@ mod reader { reader_schema: Option<&'a Schema>, errored: bool, should_resolve_schema: bool, + #[synca::cfg(tokio)] + pending_future: Option> + Send>>>, } impl<'a, R> Reader<'a, R> @@ -397,6 +399,8 @@ mod reader { reader_schema: None, errored: false, should_resolve_schema: false, + #[synca::cfg(tokio)] + pending_future: None, }; Ok(reader) } @@ -412,6 +416,8 @@ mod reader { reader_schema: Some(schema), errored: false, should_resolve_schema: false, + #[synca::cfg(tokio)] + pending_future: None, }; // Check if the reader and writer schemas disagree. reader.should_resolve_schema = reader.writer_schema() != schema; @@ -433,6 +439,8 @@ mod reader { reader_schema: Some(schema), errored: false, should_resolve_schema: false, + #[synca::cfg(tokio)] + pending_future: None, }; // Check if the reader and writer schemas disagree. reader.should_resolve_schema = reader.writer_schema() != schema; @@ -469,27 +477,6 @@ mod reader { } } - // #[cfg(feature = "tokio")] - // impl Stream for Reader<'_, R> { - // type Item = AvroResult; - // fn poll_next( - // mut self: Pin<&mut Self>, - // _cx: &mut futures::task::Context<'_>, - // ) -> Poll> { - // // to prevent keep on reading after the first error occurs - // if self.errored { - // return Poll::Ready(None); - // }; - // match self.read_next().await { - // Ok(opt) => Poll::Ready(opt.map(Ok)), - // Err(e) => { - // self.errored = true; - // Poll::Ready(Some(Err(e))) - // } - // } - // } - // } - #[synca::cfg(sync)] impl Iterator for Reader<'_, R> { type Item = AvroResult; @@ -509,6 +496,44 @@ mod reader { } } + #[synca::cfg(tokio)] + use std::pin::Pin; + #[synca::cfg(tokio)] + use std::task::{Context, Poll}; + + #[synca::cfg(tokio)] + impl futures::Stream for Reader<'_, R> { + type Item = AvroResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // to prevent keep on reading after the first error occurs + if self.errored { + return Poll::Ready(None); + }; + if let Some(mut future) = self.pending_future.take() { + match future.as_mut().poll(cx) { + Poll::Ready(result) => { + // Future completed, return the result + if result.is_err() { + self.errored = true; + } + Poll::Ready(Some(result)) + } + Poll::Pending => { + // Restore the pending future + self.pending_future = Some(future); + Poll::Pending + } + } + } else { + // TODO: mgrigorov: This breaks rustc !!! + // let future = self.read_next(); + // self.pending_future = Some(future); + Poll::Pending + } + } + } + /// Decode a `Value` encoded in Avro format given its `Schema` and anything implementing `io::Read` /// to read from. /// @@ -682,6 +707,8 @@ mod reader { types::tokio::Record, }; use apache_avro_test_helper::TestResult; + #[synca::cfg(tokio)] + use futures::StreamExt; use pretty_assertions::assert_eq; use serde::Deserialize; use std::io::Cursor; @@ -824,7 +851,7 @@ mod reader { #[tokio::test] async fn test_reader_iterator() -> TestResult { let schema = Schema::parse_str(SCHEMA).await?; - let reader = Reader::with_schema(&schema, ENCODED).await?; + let mut reader = Reader::with_schema(&schema, ENCODED).await?; let mut record1 = Record::new(&schema).unwrap(); record1.put("a", 27i64); @@ -836,8 +863,10 @@ mod reader { let expected = [record1.into(), record2.into()]; - for (i, value) in reader.enumerate() { + let mut i = 0; + while let Some(value) = reader.next().await { assert_eq!(value?, expected[i]); + i += 1; } Ok(()) @@ -864,8 +893,8 @@ mod reader { .into_iter() .rev() .collect::>(); - let reader = Reader::with_schema(&schema, &invalid[..]).await?; - for value in reader { + let mut reader = Reader::with_schema(&schema, &invalid[..]).await?; + while let Some(value) = reader.next().await { assert!(value.is_err()); } @@ -883,8 +912,8 @@ mod reader { #[tokio::test] async fn test_reader_only_header() -> TestResult { let invalid = ENCODED.iter().copied().take(165).collect::>(); - let reader = Reader::new(&invalid[..]).await?; - for value in reader { + let mut reader = Reader::new(&invalid[..]).await?; + while let Some(value) = reader.next().await { assert!(value.is_err()); } diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index 7949a20f..658c9762 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -565,6 +565,8 @@ mod schema_compatibility { writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; + #[synca::cfg(tokio)] + use futures::StreamExt; use rstest::*; async fn int_array_schema() -> Schema { @@ -1601,14 +1603,14 @@ mod schema_compatibility { let input = writer.into_inner()?; let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; assert_eq!( - reader.next().unwrap().unwrap(), + reader.next().await.unwrap().unwrap(), Value::Record(vec![ ("a".to_string(), Value::Long(27)), ("b".to_string(), Value::String("foo".to_string())), ("c".to_string(), Value::Enum(1, "spades".to_string())), ]) ); - assert!(reader.next().is_none()); + assert!(reader.next().await.is_none()); Ok(()) } @@ -1665,14 +1667,14 @@ mod schema_compatibility { let input = writer.into_inner()?; let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; assert_eq!( - reader.next().unwrap().unwrap(), + reader.next().await.unwrap().unwrap(), Value::Record(vec![ ("a".to_string(), Value::Long(27)), ("b".to_string(), Value::String("foo".to_string())), ("c".to_string(), Value::Enum(0, "hearts".to_string())), ]) ); - assert!(reader.next().is_none()); + assert!(reader.next().await.is_none()); Ok(()) } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index b084ba17..e8690a77 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -842,6 +842,8 @@ mod writer { types::tokio::Record, util::tokio::zig_i64, }; + #[synca::cfg(tokio)] + use futures::StreamExt; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -1418,11 +1420,11 @@ mod writer { writer.add_user_metadata("a".to_string(), "b")?; let result = writer.into_inner()?; - let reader = Reader::with_schema(&schema, &result[..]).await?; + let mut reader = Reader::with_schema(&schema, &result[..]).await?; let mut expected = HashMap::new(); expected.insert("a".to_string(), vec![b'b']); assert_eq!(reader.user_metadata(), &expected); - assert_eq!(reader.into_iter().count(), 0); + assert!(reader.next().await.is_none()); Ok(()) } @@ -1582,7 +1584,8 @@ mod writer { assert_eq!(buf[1], 0x01); assert_eq!( &buf[2..10], - &TestSingleObjectWriter::get_schema().await + &TestSingleObjectWriter::get_schema() + .await .fingerprint::() .bytes[..] ); @@ -1644,7 +1647,8 @@ mod writer { ) .expect("Should resolve schema"); let mut specific_writer = - SpecificSingleObjectWriter::::with_capacity(1024).await + SpecificSingleObjectWriter::::with_capacity(1024) + .await .expect("Resolved should pass"); specific_writer .write(obj1.clone(), &mut buf1) From 6524b40b7125b8b5dd975c9a484f86303f8ae714 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 8 Aug 2025 15:25:50 +0300 Subject: [PATCH 17/47] WIP: First steps to fix avro_derive Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/lib.rs | 2 +- avro/src/schema.rs | 7 +++---- avro_derive/Cargo.toml | 15 +++++++++++++-- avro_derive/tests/derive.rs | 21 +++++++++++---------- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/avro/src/lib.rs b/avro/src/lib.rs index f13b699c..895fe20e 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -984,7 +984,7 @@ mod tests { types::tokio::{Record, Value}, writer::tokio::Writer, }; - // #[synca::cfg(tokio)] + #[synca::cfg(tokio)] use futures::StreamExt; use pretty_assertions::assert_eq; diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 5dae3d5d..26496cab 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -2556,9 +2556,7 @@ mod schema { /// through `derive` feature. Do not implement directly! /// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait /// through a blanket implementation. - #[synca::cfg(tokio)] - use async_trait::async_trait; - #[cfg_attr(feature = "tokio", async_trait)] + #[cfg_attr(feature = "tokio", async_trait::async_trait)] pub trait AvroSchema { async fn get_schema() -> Schema; } @@ -2623,11 +2621,12 @@ mod schema { ) -> Schema; } + #[cfg_attr(feature = "tokio", async_trait::async_trait)] impl AvroSchema for T where T: AvroSchemaComponent, { - fn get_schema() -> Schema { + async fn get_schema() -> Schema { T::get_schema_in_ctxt(&mut HashMap::default(), &None) } } diff --git a/avro_derive/Cargo.toml b/avro_derive/Cargo.toml index 157faab7..d595273b 100644 --- a/avro_derive/Cargo.toml +++ b/avro_derive/Cargo.toml @@ -36,13 +36,24 @@ darling = { default-features = false, version = "0.21.1" } proc-macro2 = { default-features = false, version = "1.0.96" } quote = { default-features = false, version = "1.0.40" } serde_json = { workspace = true } -syn = { default-features = false, version = "2.0.104", features = ["full", "fold"] } +syn = { default-features = false, version = "2.0.104", features = [ + "full", + "fold", +] } [dev-dependencies] -apache-avro = { default-features = false, path = "../avro", features = ["derive"] } +apache-avro = { default-features = false, path = "../avro", features = [ + "derive", +] } proptest = { default-features = false, version = "1.7.0", features = ["std"] } serde = { workspace = true } [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docsrs"] + + +[[test]] +name = "derive" +path = "tests/derive.rs" +required-features = ["sync"] diff --git a/avro_derive/tests/derive.rs b/avro_derive/tests/derive.rs index 82c377b1..6ac95570 100644 --- a/avro_derive/tests/derive.rs +++ b/avro_derive/tests/derive.rs @@ -15,18 +15,19 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{ - Reader, Schema, Writer, from_value, - schema::{AvroSchema, derive::AvroSchemaComponent}, -}; -use apache_avro_derive::*; -use proptest::prelude::*; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use std::collections::HashMap; - #[cfg(test)] mod test_derive { - use apache_avro::schema::{Alias, EnumSchema, RecordSchema}; + + use apache_avro::{ + AsyncReader, AsyncSchema, AsyncWriter, AvroSchema, async_from_value, + derive::AvroSchemaComponent, + }; + use apache_avro_derive::*; + use proptest::prelude::*; + use serde::{Deserialize, Serialize, de::DeserializeOwned}; + use std::collections::HashMap; + + use apache_avro::schema::tokio::{Alias, EnumSchema, RecordSchema}; use std::{borrow::Cow, sync::Mutex}; use super::*; From 37adaf6c486065346f86169b0ede21072bfe2593 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 8 Aug 2025 16:36:34 +0300 Subject: [PATCH 18/47] Get rid of Borror for ResolvedSchema This way we need more cloning but otherwise Rustc complains that it cannot resolve whether &schema::tokio::Schema is Send Signed-off-by: Martin Tzvetanov Grigorov --- avro/examples/test_interop_data.rs | 2 +- avro/src/schema.rs | 39 ++++++++------------ avro/src/ser_schema.rs | 59 +++++++++++++++++++----------- avro/src/types.rs | 35 +++++++++--------- avro/src/writer.rs | 2 +- avro/tests/avro-3786.rs | 4 +- avro_derive/Cargo.toml | 6 --- 7 files changed, 75 insertions(+), 72 deletions(-) diff --git a/avro/examples/test_interop_data.rs b/avro/examples/test_interop_data.rs index 7833b4f2..a39583e7 100644 --- a/avro/examples/test_interop_data.rs +++ b/avro/examples/test_interop_data.rs @@ -70,7 +70,7 @@ fn main() -> Result<(), Box> { } } -fn test_user_metadata( +fn test_user_metadata( reader: &Reader>, expected_user_metadata: &HashMap>, ) { diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 26496cab..1a5c0b7f 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -65,7 +65,6 @@ mod schema { ser::{SerializeMap, SerializeSeq}, }; use std::{ - borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, hash::Hash, @@ -272,7 +271,7 @@ mod schema { /// Represents Schema lookup within a schema env pub(crate) type Names = HashMap; /// Represents Schema lookup within a schema - pub type NamesRef<'a> = HashMap; + // pub type NamesRef<'a> = HashMap; /// Represents the namespace for Named Schema pub type Namespace = Option; @@ -440,7 +439,7 @@ mod schema { #[derive(Debug)] pub struct ResolvedSchema<'s> { - names_ref: NamesRef<'s>, + names_ref: Names, schemata: Vec<&'s Schema>, } @@ -448,9 +447,8 @@ mod schema { type Error = Error; fn try_from(schema: &'s Schema) -> AvroResult { - let names = HashMap::new(); let mut rs = ResolvedSchema { - names_ref: names, + names_ref: Default::default(), schemata: vec![schema], }; rs.resolve(rs.get_schemata(), &None, None)?; @@ -462,9 +460,8 @@ mod schema { type Error = Error; fn try_from(schemata: Vec<&'s Schema>) -> AvroResult { - let names = HashMap::new(); let mut rs = ResolvedSchema { - names_ref: names, + names_ref: Default::default(), schemata, }; rs.resolve(rs.get_schemata(), &None, None)?; @@ -477,7 +474,7 @@ mod schema { self.schemata.clone() } - pub fn get_names(&self) -> &NamesRef<'s> { + pub fn get_names(&self) -> &Names { &self.names_ref } @@ -487,7 +484,7 @@ mod schema { pub fn new_with_known_schemata<'n>( schemata_to_resolve: Vec<&'s Schema>, enclosing_namespace: &Namespace, - known_schemata: &'n NamesRef<'n>, + known_schemata: &Names, ) -> AvroResult { let names = HashMap::new(); let mut rs = ResolvedSchema { @@ -502,7 +499,7 @@ mod schema { &mut self, schemata: Vec<&'s Schema>, enclosing_namespace: &Namespace, - known_schemata: Option<&'n NamesRef<'n>>, + known_schemata: Option<&Names>, ) -> AvroResult<()> { for schema in schemata { match schema { @@ -522,7 +519,7 @@ mod schema { let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); if self .names_ref - .insert(fully_qualified_name.clone(), schema) + .insert(fully_qualified_name.clone(), schema.clone()) .is_some() { return Err( @@ -534,7 +531,7 @@ mod schema { let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); if self .names_ref - .insert(fully_qualified_name.clone(), schema) + .insert(fully_qualified_name.clone(), schema.clone()) .is_some() { return Err( @@ -993,10 +990,10 @@ mod schema { /// /// Extra arguments: /// - `known_schemata` - mapping between `Name` and `Schema` - if passed, additional external schemas would be used to resolve references. - pub async fn find_schema_with_known_schemata + Debug>( + pub async fn find_schema_with_known_schemata( &self, value: &Value, - known_schemata: Option<&HashMap>, + known_schemata: &HashMap, enclosing_namespace: &Namespace, ) -> Option<(usize, &Schema)> { let schema_kind = SchemaKind::from(value); @@ -1007,14 +1004,10 @@ mod schema { // slow path (required for matching logical or named types) // first collect what schemas we already know - let mut collected_names: HashMap = known_schemata - .map(|names| { - names - .iter() - .map(|(name, schema)| (name.clone(), schema.borrow())) - .collect() - }) - .unwrap_or_default(); + let mut collected_names: HashMap = known_schemata + .iter() + .map(|(name, schema)| (name.clone(), schema.clone())) + .collect(); let mut i: usize = 0; for schema in self.schemas.iter() { @@ -1027,7 +1020,7 @@ mod schema { let resolved_names = resolved_schema.names_ref; // extend known schemas with just resolved names - collected_names.extend(resolved_names); + collected_names.extend(resolved_names.clone()); let namespace = &schema.namespace().or_else(|| enclosing_namespace.clone()); if value diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 2aad1bfa..2e0e79e5 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -46,7 +46,7 @@ mod bigdecimal { bigdecimal::tokio::big_decimal_as_bytes, encode::tokio::{encode_int, encode_long}, error::tokio::{Details, Error}, - schema::tokio::{Name, NamesRef, Namespace, RecordField, RecordSchema, Schema}, + schema::tokio::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, }; use bigdecimal::BigDecimal; use serde::{Serialize, ser}; @@ -455,7 +455,7 @@ mod bigdecimal { pub struct SchemaAwareWriteSerializer<'s, W: Write> { writer: &'s mut W, root_schema: &'s Schema, - names: &'s NamesRef<'s>, + names: &'s Names, enclosing_namespace: Namespace, } @@ -472,7 +472,7 @@ mod bigdecimal { pub fn new( writer: &'s mut W, schema: &'s Schema, - names: &'s NamesRef<'s>, + names: &'s Names, enclosing_namespace: Namespace, ) -> SchemaAwareWriteSerializer<'s, W> { SchemaAwareWriteSerializer { @@ -492,7 +492,7 @@ mod bigdecimal { }), }; - let ref_schema = self.names.get(full_name.as_ref()).copied(); + let ref_schema = self.names.get(full_name.as_ref()).clone(); ref_schema .ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) @@ -2043,7 +2043,8 @@ mod bigdecimal { {"name": "intField", "type": "int"} ] }"#, - ).await?; + ) + .await?; #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -2092,7 +2093,8 @@ mod bigdecimal { "name": "EmptyRecord", "fields": [] }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2144,7 +2146,8 @@ mod bigdecimal { "name": "Suit", "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] }"#, - ).await?; + ) + .await?; #[derive(Serialize)] enum Suit { @@ -2191,7 +2194,8 @@ mod bigdecimal { "type": "array", "items": "long" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2229,7 +2233,8 @@ mod bigdecimal { "type": "map", "values": "long" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2274,7 +2279,8 @@ mod bigdecimal { r#"{ "type": ["null", "long"] }"#, - ).await?; + ) + .await?; #[derive(Serialize)] enum NullableLong { @@ -2322,7 +2328,8 @@ mod bigdecimal { r#"{ "type": ["null", "long", "string"] }"#, - ).await?; + ) + .await?; #[derive(Serialize)] enum LongOrString { @@ -2375,7 +2382,8 @@ mod bigdecimal { "size": 8, "name": "LongVal" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2452,7 +2460,8 @@ mod bigdecimal { "precision": 16, "scale": 2 }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2491,7 +2500,8 @@ mod bigdecimal { "precision": 16, "scale": 2 }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2527,7 +2537,8 @@ mod bigdecimal { "type": "bytes", "logicalType": "big-decimal" }"#, - ).await?; + ) + .await?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); @@ -2551,7 +2562,8 @@ mod bigdecimal { "type": "string", "logicalType": "uuid" }"#, - ).await?; + ) + .await?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); @@ -2595,7 +2607,8 @@ mod bigdecimal { "type": "int", "logicalType": "date" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2639,7 +2652,8 @@ mod bigdecimal { "type": "int", "logicalType": "time-millis" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2683,7 +2697,8 @@ mod bigdecimal { "type": "long", "logicalType": "time-micros" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2731,7 +2746,8 @@ mod bigdecimal { "type": "long", "logicalType": "timestamp-{precision}" }}"# - )).await?; + )) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2790,7 +2806,8 @@ mod bigdecimal { "name": "duration", "logicalType": "duration" }"#, - ).await?; + ) + .await?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); diff --git a/avro/src/types.rs b/avro/src/types.rs index d09de72f..42431e85 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -60,7 +60,6 @@ mod types { use bigdecimal::BigDecimal; use log::{debug, error}; use std::{ - borrow::Borrow, collections::{BTreeMap, HashMap}, fmt::Debug, hash::BuildHasher, @@ -436,17 +435,17 @@ mod types { } /// Validates the value against the provided schema. - pub(crate) async fn validate_internal + Debug>( + pub(crate) async fn validate_internal( &self, schema: &Schema, - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, ) -> Option { match (self, schema) { (_, Schema::Ref { name }) => { let name = name.fully_qualified_name(enclosing_namespace); if let Some(s) = names.get(&name) { - Box::pin(self.validate_internal(s.borrow(), names, &name.namespace)).await + Box::pin(self.validate_internal(s, names, &name.namespace)).await } else { Some(format!( "Unresolved schema reference: '{:?}'. Parsed names: {:?}", @@ -553,7 +552,7 @@ mod types { } (v, Schema::Union(inner)) => { match inner - .find_schema_with_known_schemata(v, Some(names), enclosing_namespace) + .find_schema_with_known_schemata(v, names, enclosing_namespace) .await { Some(_) => None, @@ -717,10 +716,10 @@ mod types { .await } - pub(crate) async fn resolve_internal + Debug>( + pub(crate) async fn resolve_internal( mut self, schema: &Schema, - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, ) -> AvroResult { @@ -742,7 +741,7 @@ mod types { if let Some(resolved) = names.get(&name) { debug!("Resolved {name:?}"); Box::pin(self.resolve_internal( - resolved.borrow(), + resolved, names, &name.namespace, field_default, @@ -1112,10 +1111,10 @@ mod types { } } - async fn resolve_union + Debug>( + async fn resolve_union( self, schema: &UnionSchema, - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, ) -> Result { @@ -1126,7 +1125,7 @@ mod types { v => v, }; let (i, inner) = schema - .find_schema_with_known_schemata(&v, Some(names), enclosing_namespace) + .find_schema_with_known_schemata(&v, names, enclosing_namespace) .await .ok_or_else(|| Details::FindUnionVariant { schema: schema.clone(), @@ -1142,10 +1141,10 @@ mod types { )) } - async fn resolve_array + Debug>( + async fn resolve_array( self, schema: &Schema, - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, ) -> Result { match self { @@ -1171,10 +1170,10 @@ mod types { } } - async fn resolve_map + Debug>( + async fn resolve_map( self, schema: &Schema, - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, ) -> Result { match self { @@ -1211,10 +1210,10 @@ mod types { } } - async fn resolve_record + Debug>( + async fn resolve_record( self, fields: &[RecordField], - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, ) -> Result { let mut items = match self { @@ -1502,7 +1501,7 @@ mod types { for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { let err_message = value - .validate_internal::(&schema, &HashMap::default(), &None) + .validate_internal(&schema, &HashMap::default(), &None) .await; assert_eq!(valid, err_message.is_none()); if !valid { diff --git a/avro/src/writer.rs b/avro/src/writer.rs index e8690a77..abf155c7 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -786,7 +786,7 @@ mod writer { data: &T, writer: &mut W, ) -> AvroResult { - let names: HashMap = HashMap::new(); + let names: HashMap = HashMap::new(); let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, &names, None); let bytes_written = data.serialize(&mut serializer)?; Ok(bytes_written) diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index e6bbeaa0..3b159ec6 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::schema::tokio::Schema; -use apache_avro::types::tokio::Value; +use apache_avro::schema::sync::Schema; +use apache_avro::types::sync::Value; use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; diff --git a/avro_derive/Cargo.toml b/avro_derive/Cargo.toml index d595273b..082e6eed 100644 --- a/avro_derive/Cargo.toml +++ b/avro_derive/Cargo.toml @@ -51,9 +51,3 @@ serde = { workspace = true } [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docsrs"] - - -[[test]] -name = "derive" -path = "tests/derive.rs" -required-features = ["sync"] From 184a322edf70903a3d326cab11ef7dd8eee60e3e Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 11 Aug 2025 16:19:06 +0300 Subject: [PATCH 19/47] WIP: Move back the pure data structures out of Synca control Introduce XyzExt structs for the Synca managed functionality, e.g. SchemaExt with an `impl` that contains all async methods (SchemaExt::parse_str(...) -> AvroResult) Signed-off-by: Martin Tzvetanov Grigorov --- avro/benches/serde.rs | 10 +- avro/benches/single.rs | 6 +- avro/examples/benchmark.rs | 11 +- avro/examples/generate_interop_data.rs | 7 +- .../test_interop_single_object_encoding.rs | 4 +- avro/src/bigdecimal.rs | 33 +- avro/src/bytes.rs | 7 +- avro/src/codec.rs | 7 +- avro/src/de.rs | 13 +- avro/src/decimal.rs | 309 +- avro/src/decode.rs | 33 +- avro/src/encode.rs | 186 +- avro/src/error.rs | 972 +++--- avro/src/headers.rs | 13 +- avro/src/lib.rs | 78 +- avro/src/reader.rs | 38 +- avro/src/schema.rs | 2864 +++++++++-------- avro/src/schema_compatibility.rs | 137 +- avro/src/schema_equality.rs | 1017 +++--- avro/src/ser.rs | 9 +- avro/src/ser_schema.rs | 48 +- avro/src/types.rs | 646 ++-- avro/src/util.rs | 94 +- avro/src/validator.rs | 565 ++-- avro/src/writer.rs | 92 +- avro/tests/append_to_existing.rs | 6 +- avro/tests/avro-3786.rs | 28 +- avro/tests/avro-3787.rs | 10 +- avro/tests/codecs.rs | 4 +- avro/tests/io.rs | 26 +- avro/tests/schema.rs | 156 +- avro/tests/shared.rs | 4 +- avro/tests/to_from_avro_datum_schemata.rs | 8 +- avro/tests/union_schema.rs | 20 +- avro/tests/validators.rs | 6 +- avro_derive/tests/derive.rs | 58 +- wasm-demo/tests/demos.rs | 2 +- 37 files changed, 3776 insertions(+), 3751 deletions(-) diff --git a/avro/benches/serde.rs b/avro/benches/serde.rs index 36484c36..aa714fd1 100644 --- a/avro/benches/serde.rs +++ b/avro/benches/serde.rs @@ -163,7 +163,7 @@ const RAW_ADDRESS_SCHEMA: &str = r#" "#; fn make_small_record() -> anyhow::Result<(Schema, Value)> { - let small_schema = Schema::parse_str(RAW_SMALL_SCHEMA)?; + let small_schema = SchemaExt::parse_str(RAW_SMALL_SCHEMA)?; let small_record = { let mut small_record = Record::new(&small_schema).unwrap(); small_record.put("field", "foo"); @@ -173,7 +173,7 @@ fn make_small_record() -> anyhow::Result<(Schema, Value)> { } fn make_small_record_ser() -> anyhow::Result<(Schema, SmallRecord)> { - let small_schema = Schema::parse_str(RAW_SMALL_SCHEMA)?; + let small_schema = SchemaExt::parse_str(RAW_SMALL_SCHEMA)?; let small_record = SmallRecord { field: String::from("foo"), }; @@ -181,8 +181,8 @@ fn make_small_record_ser() -> anyhow::Result<(Schema, SmallRecord)> { } fn make_big_record() -> anyhow::Result<(Schema, Value)> { - let big_schema = Schema::parse_str(RAW_BIG_SCHEMA)?; - let address_schema = Schema::parse_str(RAW_ADDRESS_SCHEMA)?; + let big_schema = SchemaExt::parse_str(RAW_BIG_SCHEMA)?; + let address_schema = SchemaExt::parse_str(RAW_ADDRESS_SCHEMA)?; let mut address = Record::new(&address_schema).unwrap(); address.put("street", "street"); address.put("city", "city"); @@ -213,7 +213,7 @@ fn make_big_record() -> anyhow::Result<(Schema, Value)> { } fn make_big_record_ser() -> anyhow::Result<(Schema, BigRecord)> { - let big_schema = Schema::parse_str(RAW_BIG_SCHEMA)?; + let big_schema = SchemaExt::parse_str(RAW_BIG_SCHEMA)?; let big_record = BigRecord { username: Some(String::from("username")), age: 10, diff --git a/avro/benches/single.rs b/avro/benches/single.rs index 48a06ca6..f95caf82 100644 --- a/avro/benches/single.rs +++ b/avro/benches/single.rs @@ -138,7 +138,7 @@ const RAW_ADDRESS_SCHEMA: &str = r#" "#; fn make_small_record() -> anyhow::Result<(Schema, Value)> { - let small_schema = Schema::parse_str(RAW_SMALL_SCHEMA)?; + let small_schema = SchemaExt::parse_str(RAW_SMALL_SCHEMA)?; let small_record = { let mut small_record = Record::new(&small_schema).unwrap(); small_record.put("field", "foo"); @@ -149,8 +149,8 @@ fn make_small_record() -> anyhow::Result<(Schema, Value)> { } fn make_big_record() -> anyhow::Result<(Schema, Value)> { - let big_schema = Schema::parse_str(RAW_BIG_SCHEMA)?; - let address_schema = Schema::parse_str(RAW_ADDRESS_SCHEMA)?; + let big_schema = SchemaExt::parse_str(RAW_BIG_SCHEMA)?; + let address_schema = SchemaExt::parse_str(RAW_ADDRESS_SCHEMA)?; let mut address = Record::new(&address_schema).unwrap(); address.put("street", "street"); address.put("city", "city"); diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index 68bd0fb7..d5d38ff9 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -16,9 +16,8 @@ // under the License. use apache_avro::{ - Reader, Writer, - Schema, - types::sync::{Record, Value}, + Reader, Schema, Writer, + types::{Record, Value}, }; use apache_avro_test_helper::TestResult; use std::{ @@ -110,8 +109,8 @@ fn main() -> TestResult { {"namespace": "my.example", "type": "record", "name": "userInfo", "fields": [{"default": "NONE", "type": "string", "name": "username"}, {"default": -1, "type": "int", "name": "age"}, {"default": "NONE", "type": "string", "name": "phone"}, {"default": "NONE", "type": "string", "name": "housenum"}, {"default": {}, "type": {"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}, "name": "address"}]} "#; - let small_schema = Schema::parse_str(raw_small_schema)?; - let big_schema = Schema::parse_str(raw_big_schema)?; + let small_schema = SchemaExt::parse_str(raw_small_schema)?; + let big_schema = SchemaExt::parse_str(raw_big_schema)?; println!("{small_schema:?}"); println!("{big_schema:?}"); @@ -121,7 +120,7 @@ fn main() -> TestResult { let small_record = small_record.into(); let raw_address_schema = r#"{"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}"#; - let address_schema = Schema::parse_str(raw_address_schema).unwrap(); + let address_schema = SchemaExt::parse_str(raw_address_schema).unwrap(); let mut address = Record::new(&address_schema).unwrap(); address.put("street", "street"); address.put("city", "city"); diff --git a/avro/examples/generate_interop_data.rs b/avro/examples/generate_interop_data.rs index 6d5558ee..57ef327a 100644 --- a/avro/examples/generate_interop_data.rs +++ b/avro/examples/generate_interop_data.rs @@ -16,9 +16,8 @@ // under the License. use apache_avro::{ - Codec, Writer, - Schema, - types::sync::{Record, Value}, + Codec, Schema, Writer, + types::{Record, Value}, }; use std::{ collections::HashMap, @@ -82,7 +81,7 @@ fn main() -> Result<(), Box> { &interop_root_folder )) .expect("Unable to read the interop Avro schema"); - let schema = Schema::parse_str(schema_str.as_str())?; + let schema = SchemaExt::parse_str(schema_str.as_str())?; let data_folder = format!("{interop_root_folder}/build/interop/data"); std::fs::create_dir_all(&data_folder)?; diff --git a/avro/examples/test_interop_single_object_encoding.rs b/avro/examples/test_interop_single_object_encoding.rs index 432be81c..48339329 100644 --- a/avro/examples/test_interop_single_object_encoding.rs +++ b/avro/examples/test_interop_single_object_encoding.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{AvroSchema, types::sync::Value}; +use apache_avro::{AvroSchema, types::Value}; use std::error::Error; struct InteropMessage; @@ -28,7 +28,7 @@ impl AvroSchema for InteropMessage { let schema = std::fs::read_to_string(format!("{resource_folder}/test_schema.avsc")) .expect("File should exist with schema inside"); - apache_avro::Schema::parse_str(schema.as_str()) + apache_avro::SchemaExt::parse_str(schema.as_str()) .expect("File should exist with schema inside") } } diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index a685b6f0..1ab48739 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +pub use bigdecimal::BigDecimal; + #[synca::synca( #[cfg(feature = "tokio")] pub mod tokio { }, @@ -24,7 +26,6 @@ replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, crate::codec::tokio => crate::codec::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -38,22 +39,17 @@ } )] mod bigdecimal { + use crate::AvroResult; use crate::{ decode::tokio::{decode_len, decode_long}, encode::tokio::{encode_bytes, encode_long}, - error::tokio::Details, - types::tokio::Value, + error::Details, + types::Value, }; - pub use bigdecimal::BigDecimal; + use bigdecimal::BigDecimal; use num_bigint::BigInt; - use std::io::Read; - - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; - pub(crate) fn big_decimal_as_bytes(decimal: &BigDecimal) -> AvroResult> { + pub(crate) async fn big_decimal_as_bytes(decimal: &BigDecimal) -> AvroResult> { let mut buffer: Vec = Vec::new(); let (big_int, exponent): (BigInt, i64) = decimal.as_bigint_and_exponent(); let big_endian_value: Vec = big_int.to_signed_bytes_be(); @@ -63,7 +59,7 @@ mod bigdecimal { Ok(buffer) } - pub(crate) fn serialize_big_decimal(decimal: &BigDecimal) -> AvroResult> { + pub(crate) async fn serialize_big_decimal(decimal: &BigDecimal) -> AvroResult> { // encode big decimal, without global size let buffer = big_decimal_as_bytes(decimal)?; @@ -81,6 +77,13 @@ mod bigdecimal { Err(err) => return Err(Details::BigDecimalLen(Box::new(err)).into()), }; + #[synca::cfg(sync)] + use std::io::Read; + #[synca::cfg(tokio)] + use tokio::io::AsyncRead; + #[synca::cfg(tokio)] + use tokio::io::AsyncReadExt; + bytes .read_exact(&mut big_decimal_buffer[..]) .map_err(Details::ReadDouble)?; @@ -99,8 +102,8 @@ mod bigdecimal { mod tests { use super::*; use crate::{ - codec::tokio::Codec, error::tokio::Error, reader::tokio::Reader, schema::tokio::Schema, - types::tokio::Record, writer::tokio::Writer, + codec::tokio::Codec, error::Error, reader::tokio::Reader, schema::Schema, + schema::tokio::SchemaExt, types::Record, writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; use bigdecimal::{One, Zero}; @@ -182,7 +185,7 @@ mod bigdecimal { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; // build record with big decimal value let mut record = Record::new(&schema).unwrap(); diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 933a8ae6..0e9af644 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -304,9 +304,10 @@ pub mod serde_avro_slice_opt { mod tests { use super::*; use crate::de::tokio::from_value; - use crate::schema::tokio::Schema; + use crate::schema::Schema; + use crate::schema::tokio::SchemaExt; use crate::ser::tokio::to_value; - use crate::types::tokio::Value; + use crate::types::Value; use apache_avro_test_helper::TestResult; use serde::{Deserialize, Serialize}; @@ -339,7 +340,7 @@ mod tests { slice_field_opt: Some(&[1, 2, 3]), }; let value: Value = to_value(test).unwrap(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", diff --git a/avro/src/codec.rs b/avro/src/codec.rs index d346b715..0b5e7234 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -25,7 +25,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -37,11 +36,9 @@ } )] mod codec { - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] + use crate::AvroResult; - use crate::{error::tokio::Details, error::tokio::Error, types::tokio::Value}; + use crate::{error::Details, error::Error, types::Value}; use strum_macros::{EnumIter, EnumString, IntoStaticStr}; /// Settings for the `Deflate` codec. diff --git a/avro/src/de.rs b/avro/src/de.rs index 9f678bef..ec5f3347 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -25,7 +25,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -41,9 +40,9 @@ )] mod de { + use crate::schema::tokio::SchemaExt; use crate::{ - bytes::DE_BYTES_BORROWED, error::tokio::Details, error::tokio::Error, - schema::tokio::SchemaKind, types::tokio::Value, + bytes::DE_BYTES_BORROWED, error::Details, error::Error, schema::SchemaKind, types::Value, }; use serde::{ Deserialize, @@ -811,8 +810,8 @@ mod de { use apache_avro_test_helper::TestResult; - use crate::decimal::tokio::Decimal; - use crate::schema::tokio::Schema; + use crate::decimal::Decimal; + use crate::schema::Schema; use super::*; @@ -842,7 +841,7 @@ mod de { } "#; - let schema = Schema::parse_str(schema_content).await?; + let schema = SchemaExt::parse_str(schema_content).await?; let data = StringEnum { source: "SOZU".to_string(), }; @@ -883,7 +882,7 @@ mod de { } "#; - let schema = Schema::parse_str(schema_content).await?; + let schema = SchemaExt::parse_str(schema_content).await?; let data = StringEnum { source: "WRONG_ITEM".to_string(), }; diff --git a/avro/src/decimal.rs b/avro/src/decimal.rs index 94f699ca..9f96a7f4 100644 --- a/avro/src/decimal.rs +++ b/avro/src/decimal.rs @@ -15,205 +15,180 @@ // specific language governing permissions and limitations // under the License. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - #[tokio::test] => #[test] - ); - } -)] -mod decimal { - - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; - use crate::{error::tokio::Details, error::tokio::Error}; - use num_bigint::{BigInt, Sign}; - use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; - - #[derive(Debug, Clone, Eq)] - pub struct Decimal { - value: BigInt, - len: usize, - } +use crate::AvroResult; +use crate::error::{Details, Error}; +use num_bigint::{BigInt, Sign}; +use serde::{Deserialize, Serialize, Serializer, de::SeqAccess}; + +#[derive(Debug, Clone, Eq)] +pub struct Decimal { + value: BigInt, + len: usize, +} - impl Serialize for Decimal { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self.to_vec() { - Ok(ref bytes) => serializer.serialize_bytes(bytes), - Err(e) => Err(serde::ser::Error::custom(e)), - } +impl Serialize for Decimal { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.to_vec() { + Ok(ref bytes) => serializer.serialize_bytes(bytes), + Err(e) => Err(serde::ser::Error::custom(e)), } } - impl<'de> Deserialize<'de> for Decimal { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct DecimalVisitor; - impl<'de> serde::de::Visitor<'de> for DecimalVisitor { - type Value = Decimal; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a byte slice or seq of bytes") - } +} +impl<'de> Deserialize<'de> for Decimal { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct DecimalVisitor; + impl<'de> serde::de::Visitor<'de> for DecimalVisitor { + type Value = Decimal; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a byte slice or seq of bytes") + } - fn visit_bytes(self, v: &[u8]) -> Result - where - E: serde::de::Error, - { - Ok(Decimal::from(v)) - } - fn visit_seq(self, mut seq: A) -> Result - where - A: SeqAccess<'de>, - { - let mut bytes = Vec::new(); - while let Some(value) = seq.next_element::()? { - bytes.push(value); - } - - Ok(Decimal::from(bytes)) + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + Ok(Decimal::from(v)) + } + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut bytes = Vec::new(); + while let Some(value) = seq.next_element::()? { + bytes.push(value); } + + Ok(Decimal::from(bytes)) } - deserializer.deserialize_bytes(DecimalVisitor) } + deserializer.deserialize_bytes(DecimalVisitor) } +} - // We only care about value equality, not byte length. Can two equal `BigInt`s have two different - // byte lengths? - impl PartialEq for Decimal { - fn eq(&self, other: &Self) -> bool { - self.value == other.value - } +// We only care about value equality, not byte length. Can two equal `BigInt`s have two different +// byte lengths? +impl PartialEq for Decimal { + fn eq(&self, other: &Self) -> bool { + self.value == other.value } +} - impl Decimal { - pub(crate) fn len(&self) -> usize { - self.len - } +impl Decimal { + pub(crate) fn len(&self) -> usize { + self.len + } - pub(crate) fn to_vec(&self) -> AvroResult> { - self.to_sign_extended_bytes_with_len(self.len) - } + pub(crate) fn to_vec(&self) -> AvroResult> { + self.to_sign_extended_bytes_with_len(self.len) + } - pub(crate) fn to_sign_extended_bytes_with_len(&self, len: usize) -> AvroResult> { - let sign_byte = 0xFF * u8::from(self.value.sign() == Sign::Minus); - let mut decimal_bytes = vec![sign_byte; len]; - let raw_bytes = self.value.to_signed_bytes_be(); - let num_raw_bytes = raw_bytes.len(); - let start_byte_index = len.checked_sub(num_raw_bytes).ok_or(Details::SignExtend { - requested: len, - needed: num_raw_bytes, - })?; - decimal_bytes[start_byte_index..].copy_from_slice(&raw_bytes); - Ok(decimal_bytes) - } + pub(crate) fn to_sign_extended_bytes_with_len(&self, len: usize) -> AvroResult> { + let sign_byte = 0xFF * u8::from(self.value.sign() == Sign::Minus); + let mut decimal_bytes = vec![sign_byte; len]; + let raw_bytes = self.value.to_signed_bytes_be(); + let num_raw_bytes = raw_bytes.len(); + let start_byte_index = len.checked_sub(num_raw_bytes).ok_or(Details::SignExtend { + requested: len, + needed: num_raw_bytes, + })?; + decimal_bytes[start_byte_index..].copy_from_slice(&raw_bytes); + Ok(decimal_bytes) } +} - impl From for BigInt { - fn from(decimal: Decimal) -> Self { - decimal.value - } +impl From for BigInt { + fn from(decimal: Decimal) -> Self { + decimal.value } +} - /// Gets the internal byte array representation of a referenced decimal. - /// Usage: - /// ``` - /// use apache_avro::Decimal; - /// use std::convert::TryFrom; - /// - /// let decimal = Decimal::from(vec![1, 24]); - /// let maybe_bytes = >::try_from(&decimal); - /// ``` - impl std::convert::TryFrom<&Decimal> for Vec { - type Error = Error; - - fn try_from(decimal: &Decimal) -> Result { - decimal.to_vec() - } +/// Gets the internal byte array representation of a referenced decimal. +/// Usage: +/// ``` +/// use apache_avro::Decimal; +/// use std::convert::TryFrom; +/// +/// let decimal = Decimal::from(vec![1, 24]); +/// let maybe_bytes = >::try_from(&decimal); +/// ``` +impl std::convert::TryFrom<&Decimal> for Vec { + type Error = Error; + + fn try_from(decimal: &Decimal) -> Result { + decimal.to_vec() } +} - /// Gets the internal byte array representation of an owned decimal. - /// Usage: - /// ``` - /// use apache_avro::Decimal; - /// use std::convert::TryFrom; - /// - /// let decimal = Decimal::from(vec![1, 24]); - /// let maybe_bytes = >::try_from(decimal); - /// ``` - impl std::convert::TryFrom for Vec { - type Error = Error; - - fn try_from(decimal: Decimal) -> Result { - decimal.to_vec() - } +/// Gets the internal byte array representation of an owned decimal. +/// Usage: +/// ``` +/// use apache_avro::Decimal; +/// use std::convert::TryFrom; +/// +/// let decimal = Decimal::from(vec![1, 24]); +/// let maybe_bytes = >::try_from(decimal); +/// ``` +impl std::convert::TryFrom for Vec { + type Error = Error; + + fn try_from(decimal: Decimal) -> Result { + decimal.to_vec() } +} - impl> From for Decimal { - fn from(bytes: T) -> Self { - let bytes_ref = bytes.as_ref(); - Self { - value: BigInt::from_signed_bytes_be(bytes_ref), - len: bytes_ref.len(), - } +impl> From for Decimal { + fn from(bytes: T) -> Self { + let bytes_ref = bytes.as_ref(); + Self { + value: BigInt::from_signed_bytes_be(bytes_ref), + len: bytes_ref.len(), } } +} - #[cfg(test)] - mod tests { - use super::*; - use apache_avro_test_helper::TestResult; - use pretty_assertions::assert_eq; +#[cfg(test)] +mod tests { + use super::*; + use apache_avro_test_helper::TestResult; + use pretty_assertions::assert_eq; - #[test] - fn test_decimal_from_bytes_from_ref_decimal() -> TestResult { - let input = vec![1, 24]; - let d = Decimal::from(&input); + #[test] + fn test_decimal_from_bytes_from_ref_decimal() -> TestResult { + let input = vec![1, 24]; + let d = Decimal::from(&input); - let output = >::try_from(&d)?; - assert_eq!(output, input); + let output = >::try_from(&d)?; + assert_eq!(output, input); - Ok(()) - } + Ok(()) + } - #[test] - fn test_decimal_from_bytes_from_owned_decimal() -> TestResult { - let input = vec![1, 24]; - let d = Decimal::from(&input); + #[test] + fn test_decimal_from_bytes_from_owned_decimal() -> TestResult { + let input = vec![1, 24]; + let d = Decimal::from(&input); - let output = >::try_from(d)?; - assert_eq!(output, input); + let output = >::try_from(d)?; + assert_eq!(output, input); - Ok(()) - } + Ok(()) + } - #[test] - fn avro_3949_decimal_serde() -> TestResult { - let decimal = Decimal::from(&[1, 2, 3]); + #[test] + fn avro_3949_decimal_serde() -> TestResult { + let decimal = Decimal::from(&[1, 2, 3]); - let ser = serde_json::to_string(&decimal)?; - let de = serde_json::from_str(&ser)?; - std::assert_eq!(decimal, de); + let ser = serde_json::to_string(&decimal)?; + let de = serde_json::from_str(&ser)?; + std::assert_eq!(decimal, de); - Ok(()) - } + Ok(()) } } diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 78381bae..4750c562 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -23,7 +23,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -42,26 +41,24 @@ mod decode { #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] use crate::AvroResult; + use crate::Uuid; use crate::util::tokio::{safe_len, zag_i32, zag_i64}; use crate::{ bigdecimal::tokio::deserialize_big_decimal, - decimal::tokio::Decimal, + decimal::Decimal, duration::Duration, encode::tokio::encode_long, - error::tokio::Details, - error::tokio::Error, - schema::tokio::{ + error::Details, + error::Error, + schema::tokio::SchemaExt, + schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, }, - types::tokio::Value, + types::Value, }; use std::{borrow::Borrow, collections::HashMap, io::ErrorKind, str::FromStr}; - use uuid::Uuid; #[inline] pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { @@ -465,11 +462,11 @@ mod decode { #[allow(clippy::expect_fun_call)] mod tests { use crate::{ - decimal::tokio::Decimal, + decimal::Decimal, decode::tokio::decode, encode::tokio::{encode, tests::success}, - schema::tokio::{DecimalSchema, FixedSchema, Name, Schema}, - types::tokio::Value, + schema::{DecimalSchema, FixedSchema, Name, Schema}, + types::Value, }; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; @@ -581,7 +578,7 @@ mod decode { #[tokio::test] async fn test_avro_3448_recursive_definition_decode_union() -> TestResult { // if encoding fails in this test check the corresponding test in encode - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -645,7 +642,7 @@ mod decode { #[tokio::test] async fn test_avro_3448_recursive_definition_decode_array() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -696,7 +693,7 @@ mod decode { #[tokio::test] async fn test_avro_3448_recursive_definition_decode_map() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -793,7 +790,7 @@ mod decode { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -910,7 +907,7 @@ mod decode { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 7b7d699a..d44a660e 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -23,7 +23,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -38,34 +37,42 @@ mod encode { use crate::util::tokio::{zig_i32, zig_i64}; - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] use crate::AvroResult; use crate::{ bigdecimal::tokio::serialize_big_decimal, - error::tokio::Details, - schema::tokio::{ + error::Details, + schema::tokio::SchemaExt, + schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, SchemaKind, UnionSchema, }, - types::tokio::{Value, ValueKind}, + types::{Value, ValueKind}, }; + #[synca::cfg(sync)] + use std::io::Write as AvroWrite; + #[synca::cfg(tokio)] + use tokio::io::AsyncWrite as AvroWrite; + #[cfg(feature = "tokio")] + use tokio::io::AsyncWriteExt; use log::error; - use std::{borrow::Borrow, collections::HashMap, io::Write}; + use std::{borrow::Borrow, collections::HashMap}; /// Encode a `Value` into avro format. /// /// **NOTE** This will not perform schema validation. The value is assumed to /// be valid with regards to the schema. Schema are needed only to guide the /// encoding for complex type values. - pub fn encode(value: &Value, schema: &Schema, writer: &mut W) -> AvroResult { + pub async fn encode( + value: &Value, + schema: &Schema, + writer: &mut W, + ) -> AvroResult { let rs = ResolvedSchema::try_from(schema)?; - encode_internal(value, schema, rs.get_names(), &None, writer) + encode_internal(value, schema, rs.get_names(), &None, writer).await } - pub(crate) fn encode_bytes + ?Sized, W: Write>( + pub(crate) async fn encode_bytes + ?Sized, W: AvroWrite>( s: &B, mut writer: W, ) -> AvroResult { @@ -73,18 +80,19 @@ mod encode { encode_long(bytes.len() as i64, &mut writer)?; writer .write(bytes) + .await .map_err(|e| Details::WriteBytes(e).into()) } - pub(crate) fn encode_long(i: i64, writer: W) -> AvroResult { - zig_i64(i, writer) + pub(crate) async fn encode_long(i: i64, writer: W) -> AvroResult { + zig_i64(i, writer).await } - pub(crate) fn encode_int(i: i32, writer: W) -> AvroResult { - zig_i32(i, writer) + pub(crate) async fn encode_int(i: i32, writer: W) -> AvroResult { + zig_i32(i, writer).await } - pub(crate) fn encode_internal>( + pub(crate) async fn encode_internal>( value: &Value, schema: &Schema, names: &HashMap, @@ -96,7 +104,14 @@ mod encode { let resolved = names .get(&fully_qualified_name) .ok_or(Details::SchemaResolutionError(fully_qualified_name))?; - return encode_internal(value, resolved.borrow(), names, enclosing_namespace, writer); + return Box::pin(encode_internal( + value, + resolved.borrow(), + names, + enclosing_namespace, + writer, + )) + .await; } match value { @@ -108,7 +123,7 @@ mod encode { supported_schema: vec![SchemaKind::Null, SchemaKind::Union], } .into()), - Some(p) => encode_long(p as i64, writer), + Some(p) => encode_long(p as i64, writer).await, } } else { Ok(0) @@ -118,7 +133,7 @@ mod encode { .write(&[u8::from(*b)]) .map_err(|e| Details::WriteBytes(e).into()), // Pattern | Pattern here to signify that these _must_ have the same encoding. - Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => encode_int(*i, writer), + Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => encode_int(*i, writer).await, Value::Long(i) | Value::TimestampMillis(i) | Value::TimestampMicros(i) @@ -126,7 +141,7 @@ mod encode { | Value::LocalTimestampMillis(i) | Value::LocalTimestampMicros(i) | Value::LocalTimestampNanos(i) - | Value::TimeMicros(i) => encode_long(*i, writer), + | Value::TimeMicros(i) => encode_long(*i, writer).await, Value::Float(x) => writer .write(&x.to_le_bytes()) .map_err(|e| Details::WriteBytes(e).into()), @@ -161,19 +176,22 @@ mod encode { .map_err(|e| Details::WriteBytes(e).into()) } Value::Uuid(uuid) => match *schema { - Schema::Uuid | Schema::String => encode_bytes( - // we need the call .to_string() to properly convert ASCII to UTF-8 - #[allow(clippy::unnecessary_to_owned)] - &uuid.to_string(), - writer, - ), + Schema::Uuid | Schema::String => { + encode_bytes( + // we need the call .to_string() to properly convert ASCII to UTF-8 + #[allow(clippy::unnecessary_to_owned)] + &uuid.to_string(), + writer, + ) + .await + } Schema::Fixed(FixedSchema { size, .. }) => { if size != 16 { return Err(Details::ConvertFixedToUuid(size).into()); } let bytes = uuid.as_bytes(); - encode_bytes(bytes, writer) + encode_bytes(bytes, writer).await } _ => Err(Details::EncodeValueAsSchemaError { value_kind: ValueKind::Uuid, @@ -188,7 +206,7 @@ mod encode { .map_err(|e| Details::WriteBytes(e).into()) } Value::Bytes(bytes) => match *schema { - Schema::Bytes => encode_bytes(bytes, writer), + Schema::Bytes => encode_bytes(bytes, writer).await, Schema::Fixed { .. } => writer .write(bytes.as_slice()) .map_err(|e| Details::WriteBytes(e).into()), @@ -199,10 +217,10 @@ mod encode { .into()), }, Value::String(s) => match *schema { - Schema::String | Schema::Uuid => encode_bytes(s, writer), + Schema::String | Schema::Uuid => encode_bytes(s, writer).await, Schema::Enum(EnumSchema { ref symbols, .. }) => { if let Some(index) = symbols.iter().position(|item| item == s) { - encode_int(index as i32, writer) + encode_int(index as i32, writer).await } else { error!("Invalid symbol string {:?}.", &s[..]); Err(Details::GetEnumSymbol(s.clone()).into()) @@ -217,15 +235,22 @@ mod encode { Value::Fixed(_, bytes) => writer .write(bytes.as_slice()) .map_err(|e| Details::WriteBytes(e).into()), - Value::Enum(i, _) => encode_int(*i as i32, writer), + Value::Enum(i, _) => encode_int(*i as i32, writer).await, Value::Union(idx, item) => { if let Schema::Union(ref inner) = *schema { let inner_schema = inner .schemas .get(*idx as usize) .expect("Invalid Union validation occurred"); - encode_long(*idx as i64, &mut *writer)?; - encode_internal(item, inner_schema, names, enclosing_namespace, &mut *writer) + encode_long(*idx as i64, &mut *writer).await?; + Box::pin(encode_internal( + item, + inner_schema, + names, + enclosing_namespace, + &mut *writer, + )) + .await } else { error!("invalid schema type for Union: {schema:?}"); Err(Details::EncodeValueAsSchemaError { @@ -238,19 +263,21 @@ mod encode { Value::Array(items) => { if let Schema::Array(ref inner) = *schema { if !items.is_empty() { - encode_long(items.len() as i64, &mut *writer)?; + Box::pin(encode_long(items.len() as i64, &mut *writer)).await?; for item in items.iter() { - encode_internal( + Box::pin(encode_internal( item, &inner.items, names, enclosing_namespace, &mut *writer, - )?; + )) + .await?; } } writer .write(&[0u8]) + .await .map_err(|e| Details::WriteBytes(e).into()) } else { error!("invalid schema type for Array: {schema:?}"); @@ -264,20 +291,22 @@ mod encode { Value::Map(items) => { if let Schema::Map(ref inner) = *schema { if !items.is_empty() { - encode_long(items.len() as i64, &mut *writer)?; + Box::pin(encode_long(items.len() as i64, &mut *writer)).await?; for (key, value) in items { - encode_bytes(key, &mut *writer)?; - encode_internal( + Box::pin(encode_bytes(key, &mut *writer)).await?; + Box::pin(encode_internal( value, &inner.types, names, enclosing_namespace, &mut *writer, - )?; + )) + .await?; } } writer .write(&[0u8]) + .await .map_err(|e| Details::WriteBytes(e).into()) } else { error!("invalid schema type for Map: {schema:?}"); @@ -314,13 +343,14 @@ mod encode { }); if let Some(value) = value_opt { - written_bytes += encode_internal( + written_bytes += Box::pin(encode_internal( value, &schema_field.schema, names, &record_namespace, writer, - )?; + )) + .await?; } else { return Err(Details::NoEntryInLookupTable( name.clone(), @@ -345,6 +375,7 @@ mod encode { Ok(_) => { return writer .write(union_buffer.as_slice()) + .await .map_err(|e| Details::WriteBytes(e).into()); } Err(_) => { @@ -369,9 +400,9 @@ mod encode { } } - pub fn encode_to_vec(value: &Value, schema: &Schema) -> AvroResult> { + pub async fn encode_to_vec(value: &Value, schema: &Schema) -> AvroResult> { let mut buffer = Vec::new(); - encode(value, schema, &mut buffer)?; + encode(value, schema, &mut buffer).await?; Ok(buffer) } @@ -379,7 +410,7 @@ mod encode { #[allow(clippy::expect_fun_call)] pub(crate) mod tests { use super::*; - use crate::error::tokio::{Details, Error}; + use crate::error::{Details, Error}; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use uuid::Uuid; @@ -400,6 +431,7 @@ mod encode { &Schema::array(Schema::Int), &mut buf, ) + .await .expect(&success(&Value::Array(empty), &Schema::array(Schema::Int))); assert_eq!(vec![0u8], buf); } @@ -413,6 +445,7 @@ mod encode { &Schema::map(Schema::Int), &mut buf, ) + .await .expect(&success(&Value::Map(empty), &Schema::map(Schema::Int))); assert_eq!(vec![0u8], buf); } @@ -420,7 +453,7 @@ mod encode { #[tokio::test] async fn test_avro_3433_recursive_definition_encode_record() -> TestResult { let mut buf = Vec::new(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -443,13 +476,16 @@ mod encode { } ] }"#, - ).await?; + ) + .await?; let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); let outer_value = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); Ok(()) } @@ -457,7 +493,7 @@ mod encode { #[tokio::test] async fn test_avro_3433_recursive_definition_encode_array() -> TestResult { let mut buf = Vec::new(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -498,7 +534,9 @@ mod encode { Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), ), ]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); Ok(()) } @@ -506,7 +544,7 @@ mod encode { #[tokio::test] async fn test_avro_3433_recursive_definition_encode_map() -> TestResult { let mut buf = Vec::new(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -544,7 +582,9 @@ mod encode { Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), ), ]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); Ok(()) } @@ -552,7 +592,7 @@ mod encode { #[tokio::test] async fn test_avro_3433_recursive_definition_encode_record_wrapper() -> TestResult { let mut buf = Vec::new(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -592,7 +632,9 @@ mod encode { )]); let outer_value = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); Ok(()) } @@ -600,7 +642,7 @@ mod encode { #[tokio::test] async fn test_avro_3433_recursive_definition_encode_map_and_array() -> TestResult { let mut buf = Vec::new(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -641,7 +683,9 @@ mod encode { ), ("b".into(), Value::Array(vec![inner_value1])), ]); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); assert!(!buf.is_empty()); Ok(()) } @@ -649,7 +693,7 @@ mod encode { #[tokio::test] async fn test_avro_3433_recursive_definition_encode_union() -> TestResult { let mut buf = Vec::new(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -681,7 +725,9 @@ mod encode { ("a".into(), Value::Union(1, Box::new(inner_value1))), ("b".into(), inner_value2.clone()), ]); - encode(&outer_value1, &schema, &mut buf).expect(&success(&outer_value1, &schema)); + encode(&outer_value1, &schema, &mut buf) + .await + .expect(&success(&outer_value1, &schema)); assert!(!buf.is_empty()); buf.drain(..); @@ -689,7 +735,9 @@ mod encode { ("a".into(), Value::Union(0, Box::new(Value::Null))), ("b".into(), inner_value2), ]); - encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value1, &schema)); + encode(&outer_value2, &schema, &mut buf) + .await + .expect(&success(&outer_value1, &schema)); assert!(!buf.is_empty()); Ok(()) } @@ -737,7 +785,7 @@ mod encode { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -771,14 +819,17 @@ mod encode { let mut buf = Vec::new(); encode(&outer_record_variation_1, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_1, &schema)); assert!(!buf.is_empty()); buf.drain(..); encode(&outer_record_variation_2, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_2, &schema)); assert!(!buf.is_empty()); buf.drain(..); encode(&outer_record_variation_3, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_3, &schema)); assert!(!buf.is_empty()); Ok(()) @@ -828,7 +879,7 @@ mod encode { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -862,14 +913,17 @@ mod encode { let mut buf = Vec::new(); encode(&outer_record_variation_1, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_1, &schema)); assert!(!buf.is_empty()); buf.drain(..); encode(&outer_record_variation_2, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_2, &schema)); assert!(!buf.is_empty()); buf.drain(..); encode(&outer_record_variation_3, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_3, &schema)); assert!(!buf.is_empty()); Ok(()) @@ -920,7 +974,7 @@ mod encode { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -954,14 +1008,17 @@ mod encode { let mut buf = Vec::new(); encode(&outer_record_variation_1, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_1, &schema)); assert!(!buf.is_empty()); buf.drain(..); encode(&outer_record_variation_2, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_2, &schema)); assert!(!buf.is_empty()); buf.drain(..); encode(&outer_record_variation_3, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_3, &schema)); assert!(!buf.is_empty()); Ok(()) @@ -990,7 +1047,10 @@ mod encode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - match encode(&value, &schema, &mut buffer).map_err(Error::into_details) { + match encode(&value, &schema, &mut buffer) + .map_err(Error::into_details) + .await + { Err(Details::ConvertFixedToUuid(actual)) => { assert_eq!(actual, 15); } diff --git a/avro/src/error.rs b/avro/src/error.rs index b01c967e..8999ec35 100644 --- a/avro/src/error.rs +++ b/avro/src/error.rs @@ -15,671 +15,667 @@ // specific language governing permissions and limitations // under the License. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - #[tokio::test] => #[test] - ); - } -)] -mod error { - use crate::{ - schema::tokio::{Name, Schema, SchemaKind, UnionSchema}, - types::tokio::{Value, ValueKind}, - }; - use std::{error::Error as _, fmt}; - - /// Errors encounterd by Avro. - /// - /// To inspect the details of the error use [`details`](Self::details) or [`into_details`](Self::into_details) - /// to get a [`Details`] which contains more precise error information. - /// - /// See [`Details`] for all possible errors. - #[derive(thiserror::Error, Debug)] - #[repr(transparent)] - #[error(transparent)] - pub struct Error { - details: Box
, - } +// #[synca::synca( +// #[cfg(feature = "tokio")] +// pub mod tokio { }, +// #[cfg(feature = "sync")] +// pub mod sync { +// sync!(); +// replace!( +// crate::bigdecimal::tokio => crate::bigdecimal::sync, +// crate::decimal::tokio => crate::decimal::sync, +// crate::decode::tokio => crate::decode::sync, +// crate::encode::tokio => crate::encode::sync, +// crate::error::tokio => crate::error::sync, +// crate::schema::tokio => crate::schema::sync, +// crate::util::tokio => crate::util::sync, +// crate::types::tokio => crate::types::sync, +// #[tokio::test] => #[test] +// ); +// } +// )] +// mod error { +use crate::{ + schema::{Name, Schema, SchemaKind, UnionSchema}, + types::{Value, ValueKind}, +}; +use std::{error::Error as _, fmt}; + +/// Errors encounterd by Avro. +/// +/// To inspect the details of the error use [`details`](Self::details) or [`into_details`](Self::into_details) +/// to get a [`Details`] which contains more precise error information. +/// +/// See [`Details`] for all possible errors. +#[derive(thiserror::Error, Debug)] +#[repr(transparent)] +#[error(transparent)] +pub struct Error { + details: Box
, +} - impl Error { - pub fn new(details: Details) -> Self { - Self { - details: Box::new(details), - } +impl Error { + pub fn new(details: Details) -> Self { + Self { + details: Box::new(details), } + } - pub fn details(&self) -> &Details { - &self.details - } + pub fn details(&self) -> &Details { + &self.details + } - pub fn into_details(self) -> Details { - *self.details - } + pub fn into_details(self) -> Details { + *self.details } +} - impl From
for Error { - fn from(details: Details) -> Self { - Self::new(details) - } +impl From
for Error { + fn from(details: Details) -> Self { + Self::new(details) } +} - impl serde::ser::Error for Error { - fn custom(msg: T) -> Self { - Self::new(
::custom(msg)) - } +impl serde::ser::Error for Error { + fn custom(msg: T) -> Self { + Self::new(
::custom(msg)) } +} - impl serde::de::Error for Error { - fn custom(msg: T) -> Self { - Self::new(
::custom(msg)) - } +impl serde::de::Error for Error { + fn custom(msg: T) -> Self { + Self::new(
::custom(msg)) } +} - #[derive(thiserror::Error)] - pub enum Details { - #[error("Bad Snappy CRC32; expected {expected:x} but got {actual:x}")] - SnappyCrc32 { expected: u32, actual: u32 }, +#[derive(thiserror::Error)] +pub enum Details { + #[error("Bad Snappy CRC32; expected {expected:x} but got {actual:x}")] + SnappyCrc32 { expected: u32, actual: u32 }, - #[error("Invalid u8 for bool: {0}")] - BoolValue(u8), + #[error("Invalid u8 for bool: {0}")] + BoolValue(u8), - #[error("Not a fixed value, required for decimal with fixed schema: {0:?}")] - FixedValue(Value), + #[error("Not a fixed value, required for decimal with fixed schema: {0:?}")] + FixedValue(Value), - #[error("Not a bytes value, required for decimal with bytes schema: {0:?}")] - BytesValue(Value), + #[error("Not a bytes value, required for decimal with bytes schema: {0:?}")] + BytesValue(Value), - #[error("Not a string value, required for uuid: {0:?}")] - GetUuidFromStringValue(Value), + #[error("Not a string value, required for uuid: {0:?}")] + GetUuidFromStringValue(Value), - #[error("Two schemas with the same fullname were given: {0:?}")] - NameCollision(String), + #[error("Two schemas with the same fullname were given: {0:?}")] + NameCollision(String), - #[error("Not a fixed or bytes type, required for decimal schema, got: {0:?}")] - ResolveDecimalSchema(SchemaKind), + #[error("Not a fixed or bytes type, required for decimal schema, got: {0:?}")] + ResolveDecimalSchema(SchemaKind), - #[error("Invalid utf-8 string")] - ConvertToUtf8(#[source] std::string::FromUtf8Error), + #[error("Invalid utf-8 string")] + ConvertToUtf8(#[source] std::string::FromUtf8Error), - #[error("Invalid utf-8 string")] - ConvertToUtf8Error(#[source] std::str::Utf8Error), + #[error("Invalid utf-8 string")] + ConvertToUtf8Error(#[source] std::str::Utf8Error), - /// Describes errors happened while validating Avro data. - #[error("Value does not match schema")] - Validation, + /// Describes errors happened while validating Avro data. + #[error("Value does not match schema")] + Validation, - /// Describes errors happened while validating Avro data. - #[error("Value {value:?} does not match schema {schema:?}: Reason: {reason}")] - ValidationWithReason { - value: Value, - schema: Schema, - reason: String, - }, + /// Describes errors happened while validating Avro data. + #[error("Value {value:?} does not match schema {schema:?}: Reason: {reason}")] + ValidationWithReason { + value: Value, + schema: Schema, + reason: String, + }, - #[error("Unable to allocate {desired} bytes (maximum allowed: {maximum})")] - MemoryAllocation { desired: usize, maximum: usize }, + #[error("Unable to allocate {desired} bytes (maximum allowed: {maximum})")] + MemoryAllocation { desired: usize, maximum: usize }, - /// Describe a specific error happening with decimal representation - #[error( - "Number of bytes requested for decimal sign extension {requested} is less than the number of bytes needed to decode {needed}" - )] - SignExtend { requested: usize, needed: usize }, + /// Describe a specific error happening with decimal representation + #[error( + "Number of bytes requested for decimal sign extension {requested} is less than the number of bytes needed to decode {needed}" + )] + SignExtend { requested: usize, needed: usize }, - #[error("Failed to read boolean bytes: {0}")] - ReadBoolean(#[source] std::io::Error), + #[error("Failed to read boolean bytes: {0}")] + ReadBoolean(#[source] std::io::Error), - #[error("Failed to read bytes: {0}")] - ReadBytes(#[source] std::io::Error), + #[error("Failed to read bytes: {0}")] + ReadBytes(#[source] std::io::Error), - #[error("Failed to read string: {0}")] - ReadString(#[source] std::io::Error), + #[error("Failed to read string: {0}")] + ReadString(#[source] std::io::Error), - #[error("Failed to read double: {0}")] - ReadDouble(#[source] std::io::Error), + #[error("Failed to read double: {0}")] + ReadDouble(#[source] std::io::Error), - #[error("Failed to read float: {0}")] - ReadFloat(#[source] std::io::Error), + #[error("Failed to read float: {0}")] + ReadFloat(#[source] std::io::Error), - #[error("Failed to read duration: {0}")] - ReadDuration(#[source] std::io::Error), + #[error("Failed to read duration: {0}")] + ReadDuration(#[source] std::io::Error), - #[error("Failed to read fixed number of bytes '{1}': : {0}")] - ReadFixed(#[source] std::io::Error, usize), + #[error("Failed to read fixed number of bytes '{1}': : {0}")] + ReadFixed(#[source] std::io::Error, usize), - #[error("Failed to convert &str to UUID: {0}")] - ConvertStrToUuid(#[source] uuid::Error), + #[error("Failed to convert &str to UUID: {0}")] + ConvertStrToUuid(#[source] uuid::Error), - #[error("Failed to convert Fixed bytes to UUID. It must be exactly 16 bytes, got {0}")] - ConvertFixedToUuid(usize), + #[error("Failed to convert Fixed bytes to UUID. It must be exactly 16 bytes, got {0}")] + ConvertFixedToUuid(usize), - #[error("Failed to convert Fixed bytes to UUID: {0}")] - ConvertSliceToUuid(#[source] uuid::Error), + #[error("Failed to convert Fixed bytes to UUID: {0}")] + ConvertSliceToUuid(#[source] uuid::Error), - #[error("Map key is not a string; key type is {0:?}")] - MapKeyType(ValueKind), + #[error("Map key is not a string; key type is {0:?}")] + MapKeyType(ValueKind), - #[error("Union index {index} out of bounds: {num_variants}")] - GetUnionVariant { index: i64, num_variants: usize }, + #[error("Union index {index} out of bounds: {num_variants}")] + GetUnionVariant { index: i64, num_variants: usize }, - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Enum symbol index out of bounds: {num_variants}")] - EnumSymbolIndex { index: usize, num_variants: usize }, + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Enum symbol index out of bounds: {num_variants}")] + EnumSymbolIndex { index: usize, num_variants: usize }, - #[error("Enum symbol not found {0}")] - GetEnumSymbol(String), + #[error("Enum symbol not found {0}")] + GetEnumSymbol(String), - #[error("Unable to decode enum index")] - GetEnumUnknownIndexValue, + #[error("Unable to decode enum index")] + GetEnumUnknownIndexValue, - #[error("Scale {scale} is greater than precision {precision}")] - GetScaleAndPrecision { scale: usize, precision: usize }, + #[error("Scale {scale} is greater than precision {precision}")] + GetScaleAndPrecision { scale: usize, precision: usize }, - #[error( - "Fixed type number of bytes {size} is not large enough to hold decimal values of precision {precision}" - )] - GetScaleWithFixedSize { size: usize, precision: usize }, + #[error( + "Fixed type number of bytes {size} is not large enough to hold decimal values of precision {precision}" + )] + GetScaleWithFixedSize { size: usize, precision: usize }, - #[error("Expected Value::Uuid, got: {0:?}")] - GetUuid(Value), + #[error("Expected Value::Uuid, got: {0:?}")] + GetUuid(Value), - #[error("Expected Value::BigDecimal, got: {0:?}")] - GetBigDecimal(Value), + #[error("Expected Value::BigDecimal, got: {0:?}")] + GetBigDecimal(Value), - #[error("Fixed bytes of size 12 expected, got Fixed of size {0}")] - GetDecimalFixedBytes(usize), + #[error("Fixed bytes of size 12 expected, got Fixed of size {0}")] + GetDecimalFixedBytes(usize), - #[error("Expected Value::Duration or Value::Fixed(12), got: {0:?}")] - ResolveDuration(Value), + #[error("Expected Value::Duration or Value::Fixed(12), got: {0:?}")] + ResolveDuration(Value), - #[error("Expected Value::Decimal, Value::Bytes or Value::Fixed, got: {0:?}")] - ResolveDecimal(Value), + #[error("Expected Value::Decimal, Value::Bytes or Value::Fixed, got: {0:?}")] + ResolveDecimal(Value), - #[error("Missing field in record: {0:?}")] - GetField(String), + #[error("Missing field in record: {0:?}")] + GetField(String), - #[error("Unable to convert to u8, got {0:?}")] - GetU8(Value), + #[error("Unable to convert to u8, got {0:?}")] + GetU8(Value), - #[error("Precision {precision} too small to hold decimal values with {num_bytes} bytes")] - ComparePrecisionAndSize { precision: usize, num_bytes: usize }, + #[error("Precision {precision} too small to hold decimal values with {num_bytes} bytes")] + ComparePrecisionAndSize { precision: usize, num_bytes: usize }, - #[error("Cannot convert length to i32: {1}")] - ConvertLengthToI32(#[source] std::num::TryFromIntError, usize), + #[error("Cannot convert length to i32: {1}")] + ConvertLengthToI32(#[source] std::num::TryFromIntError, usize), - #[error("Expected Value::Date or Value::Int, got: {0:?}")] - GetDate(Value), + #[error("Expected Value::Date or Value::Int, got: {0:?}")] + GetDate(Value), - #[error("Expected Value::TimeMillis or Value::Int, got: {0:?}")] - GetTimeMillis(Value), + #[error("Expected Value::TimeMillis or Value::Int, got: {0:?}")] + GetTimeMillis(Value), - #[error("Expected Value::TimeMicros, Value::Long or Value::Int, got: {0:?}")] - GetTimeMicros(Value), + #[error("Expected Value::TimeMicros, Value::Long or Value::Int, got: {0:?}")] + GetTimeMicros(Value), - #[error("Expected Value::TimestampMillis, Value::Long or Value::Int, got: {0:?}")] - GetTimestampMillis(Value), + #[error("Expected Value::TimestampMillis, Value::Long or Value::Int, got: {0:?}")] + GetTimestampMillis(Value), - #[error("Expected Value::TimestampMicros, Value::Long or Value::Int, got: {0:?}")] - GetTimestampMicros(Value), + #[error("Expected Value::TimestampMicros, Value::Long or Value::Int, got: {0:?}")] + GetTimestampMicros(Value), - #[error("Expected Value::TimestampNanos, Value::Long or Value::Int, got: {0:?}")] - GetTimestampNanos(Value), + #[error("Expected Value::TimestampNanos, Value::Long or Value::Int, got: {0:?}")] + GetTimestampNanos(Value), - #[error("Expected Value::LocalTimestampMillis, Value::Long or Value::Int, got: {0:?}")] - GetLocalTimestampMillis(Value), + #[error("Expected Value::LocalTimestampMillis, Value::Long or Value::Int, got: {0:?}")] + GetLocalTimestampMillis(Value), - #[error("Expected Value::LocalTimestampMicros, Value::Long or Value::Int, got: {0:?}")] - GetLocalTimestampMicros(Value), + #[error("Expected Value::LocalTimestampMicros, Value::Long or Value::Int, got: {0:?}")] + GetLocalTimestampMicros(Value), - #[error("Expected Value::LocalTimestampNanos, Value::Long or Value::Int, got: {0:?}")] - GetLocalTimestampNanos(Value), + #[error("Expected Value::LocalTimestampNanos, Value::Long or Value::Int, got: {0:?}")] + GetLocalTimestampNanos(Value), - #[error("Expected Value::Null, got: {0:?}")] - GetNull(Value), + #[error("Expected Value::Null, got: {0:?}")] + GetNull(Value), - #[error("Expected Value::Boolean, got: {0:?}")] - GetBoolean(Value), + #[error("Expected Value::Boolean, got: {0:?}")] + GetBoolean(Value), - #[error("Expected Value::Int, got: {0:?}")] - GetInt(Value), + #[error("Expected Value::Int, got: {0:?}")] + GetInt(Value), - #[error("Expected Value::Long or Value::Int, got: {0:?}")] - GetLong(Value), + #[error("Expected Value::Long or Value::Int, got: {0:?}")] + GetLong(Value), - #[error(r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"# + #[error(r#"Expected Value::Double, Value::Float, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"# )] - GetDouble(Value), + GetDouble(Value), - #[error(r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"# + #[error(r#"Expected Value::Float, Value::Double, Value::Int, Value::Long or Value::String ("NaN", "INF", "Infinity", "-INF" or "-Infinity"), got: {0:?}"# )] - GetFloat(Value), + GetFloat(Value), - #[error("Expected Value::Bytes, got: {0:?}")] - GetBytes(Value), + #[error("Expected Value::Bytes, got: {0:?}")] + GetBytes(Value), - #[error("Expected Value::String, Value::Bytes or Value::Fixed, got: {0:?}")] - GetString(Value), + #[error("Expected Value::String, Value::Bytes or Value::Fixed, got: {0:?}")] + GetString(Value), - #[error("Expected Value::Enum, got: {0:?}")] - GetEnum(Value), + #[error("Expected Value::Enum, got: {0:?}")] + GetEnum(Value), - #[error("Fixed size mismatch, expected: {size}, got: {n}")] - CompareFixedSizes { size: usize, n: usize }, + #[error("Fixed size mismatch, expected: {size}, got: {n}")] + CompareFixedSizes { size: usize, n: usize }, - #[error("String expected for fixed, got: {0:?}")] - GetStringForFixed(Value), + #[error("String expected for fixed, got: {0:?}")] + GetStringForFixed(Value), - #[error("Enum default {symbol:?} is not among allowed symbols {symbols:?}")] - GetEnumDefault { - symbol: String, - symbols: Vec, - }, + #[error("Enum default {symbol:?} is not among allowed symbols {symbols:?}")] + GetEnumDefault { + symbol: String, + symbols: Vec, + }, - #[error("Enum value index {index} is out of bounds {nsymbols}")] - GetEnumValue { index: usize, nsymbols: usize }, + #[error("Enum value index {index} is out of bounds {nsymbols}")] + GetEnumValue { index: usize, nsymbols: usize }, - #[error("Key {0} not found in decimal metadata JSON")] - GetDecimalMetadataFromJson(&'static str), + #[error("Key {0} not found in decimal metadata JSON")] + GetDecimalMetadataFromJson(&'static str), - #[error("Could not find matching type in {schema:?} for {value:?}")] - FindUnionVariant { schema: UnionSchema, value: Value }, + #[error("Could not find matching type in {schema:?} for {value:?}")] + FindUnionVariant { schema: UnionSchema, value: Value }, - #[error("Union type should not be empty")] - EmptyUnion, + #[error("Union type should not be empty")] + EmptyUnion, - #[error("Array({expected:?}) expected, got {other:?}")] - GetArray { expected: SchemaKind, other: Value }, + #[error("Array({expected:?}) expected, got {other:?}")] + GetArray { expected: SchemaKind, other: Value }, - #[error("Map({expected:?}) expected, got {other:?}")] - GetMap { expected: SchemaKind, other: Value }, + #[error("Map({expected:?}) expected, got {other:?}")] + GetMap { expected: SchemaKind, other: Value }, - #[error("Record with fields {expected:?} expected, got {other:?}")] - GetRecord { - expected: Vec<(String, SchemaKind)>, - other: Value, - }, + #[error("Record with fields {expected:?} expected, got {other:?}")] + GetRecord { + expected: Vec<(String, SchemaKind)>, + other: Value, + }, - #[error("No `name` field")] - GetNameField, + #[error("No `name` field")] + GetNameField, - #[error("No `name` in record field")] - GetNameFieldFromRecord, + #[error("No `name` in record field")] + GetNameFieldFromRecord, - #[error("Unions may not directly contain a union")] - GetNestedUnion, + #[error("Unions may not directly contain a union")] + GetNestedUnion, - #[error("Unions cannot contain duplicate types")] - GetUnionDuplicate, + #[error("Unions cannot contain duplicate types")] + GetUnionDuplicate, - #[error("One union type {0:?} must match the `default`'s value type {1:?}")] - GetDefaultUnion(SchemaKind, ValueKind), + #[error("One union type {0:?} must match the `default`'s value type {1:?}")] + GetDefaultUnion(SchemaKind, ValueKind), - #[error("`default`'s value type of field {0:?} in {1:?} must be {2:?}")] - GetDefaultRecordField(String, String, String), + #[error("`default`'s value type of field {0:?} in {1:?} must be {2:?}")] + GetDefaultRecordField(String, String, String), - #[error("JSON value {0} claims to be u64 but cannot be converted")] - GetU64FromJson(serde_json::Number), + #[error("JSON value {0} claims to be u64 but cannot be converted")] + GetU64FromJson(serde_json::Number), - #[error("JSON value {0} claims to be i64 but cannot be converted")] - GetI64FromJson(serde_json::Number), + #[error("JSON value {0} claims to be i64 but cannot be converted")] + GetI64FromJson(serde_json::Number), - #[error("Cannot convert u64 to usize: {1}")] - ConvertU64ToUsize(#[source] std::num::TryFromIntError, u64), + #[error("Cannot convert u64 to usize: {1}")] + ConvertU64ToUsize(#[source] std::num::TryFromIntError, u64), - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Cannot convert u32 to usize: {1}")] - ConvertU32ToUsize(#[source] std::num::TryFromIntError, u32), + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Cannot convert u32 to usize: {1}")] + ConvertU32ToUsize(#[source] std::num::TryFromIntError, u32), - #[error("Cannot convert i64 to usize: {1}")] - ConvertI64ToUsize(#[source] std::num::TryFromIntError, i64), + #[error("Cannot convert i64 to usize: {1}")] + ConvertI64ToUsize(#[source] std::num::TryFromIntError, i64), - #[error("Cannot convert i32 to usize: {1}")] - ConvertI32ToUsize(#[source] std::num::TryFromIntError, i32), + #[error("Cannot convert i32 to usize: {1}")] + ConvertI32ToUsize(#[source] std::num::TryFromIntError, i32), - #[error("Invalid JSON value for decimal precision/scale integer: {0}")] - GetPrecisionOrScaleFromJson(serde_json::Number), + #[error("Invalid JSON value for decimal precision/scale integer: {0}")] + GetPrecisionOrScaleFromJson(serde_json::Number), - #[error("Failed to parse schema from JSON")] - ParseSchemaJson(#[source] serde_json::Error), + #[error("Failed to parse schema from JSON")] + ParseSchemaJson(#[source] serde_json::Error), - #[error("Failed to read schema")] - ReadSchemaFromReader(#[source] std::io::Error), + #[error("Failed to read schema")] + ReadSchemaFromReader(#[source] std::io::Error), - #[error("Must be a JSON string, object or array")] - ParseSchemaFromValidJson, + #[error("Must be a JSON string, object or array")] + ParseSchemaFromValidJson, - #[error("Unknown primitive type: {0}")] - ParsePrimitive(String), + #[error("Unknown primitive type: {0}")] + ParsePrimitive(String), - #[error("invalid JSON for {key:?}: {value:?}")] - GetDecimalMetadataValueFromJson { - key: String, - value: serde_json::Value, - }, + #[error("invalid JSON for {key:?}: {value:?}")] + GetDecimalMetadataValueFromJson { + key: String, + value: serde_json::Value, + }, - #[error( - "The decimal precision ({precision}) must be bigger or equal to the scale ({scale})" - )] - DecimalPrecisionLessThanScale { precision: usize, scale: usize }, + #[error("The decimal precision ({precision}) must be bigger or equal to the scale ({scale})")] + DecimalPrecisionLessThanScale { precision: usize, scale: usize }, - #[error("The decimal precision ({precision}) must be a positive number")] - DecimalPrecisionMuBePositive { precision: usize }, + #[error("The decimal precision ({precision}) must be a positive number")] + DecimalPrecisionMuBePositive { precision: usize }, - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Unreadable big decimal sign")] - BigDecimalSign, + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Unreadable big decimal sign")] + BigDecimalSign, - #[error("Unreadable length for big decimal inner bytes: {0}")] - BigDecimalLen(#[source] Box), + #[error("Unreadable length for big decimal inner bytes: {0}")] + BigDecimalLen(#[source] Box), - #[error("Unreadable big decimal scale")] - BigDecimalScale, + #[error("Unreadable big decimal scale")] + BigDecimalScale, - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Unexpected `type` {0} variant for `logicalType`")] - GetLogicalTypeVariant(serde_json::Value), + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Unexpected `type` {0} variant for `logicalType`")] + GetLogicalTypeVariant(serde_json::Value), - #[error("No `type` field found for `logicalType`")] - GetLogicalTypeField, + #[error("No `type` field found for `logicalType`")] + GetLogicalTypeField, - #[error("logicalType must be a string, but is {0:?}")] - GetLogicalTypeFieldType(serde_json::Value), + #[error("logicalType must be a string, but is {0:?}")] + GetLogicalTypeFieldType(serde_json::Value), - #[error("Unknown complex type: {0}")] - GetComplexType(serde_json::Value), + #[error("Unknown complex type: {0}")] + GetComplexType(serde_json::Value), - #[error("No `type` in complex type")] - GetComplexTypeField, + #[error("No `type` in complex type")] + GetComplexTypeField, - #[error("No `fields` in record")] - GetRecordFieldsJson, + #[error("No `fields` in record")] + GetRecordFieldsJson, - #[error("No `symbols` field in enum")] - GetEnumSymbolsField, + #[error("No `symbols` field in enum")] + GetEnumSymbolsField, - #[error("Unable to parse `symbols` in enum")] - GetEnumSymbols, + #[error("Unable to parse `symbols` in enum")] + GetEnumSymbols, - #[error("Invalid enum symbol name {0}")] - EnumSymbolName(String), + #[error("Invalid enum symbol name {0}")] + EnumSymbolName(String), - #[error("Invalid field name {0}")] - FieldName(String), + #[error("Invalid field name {0}")] + FieldName(String), - #[error("Duplicate field name {0}")] - FieldNameDuplicate(String), + #[error("Duplicate field name {0}")] + FieldNameDuplicate(String), - #[error("Invalid schema name {0}. It must match the regex '{1}'")] - InvalidSchemaName(String, &'static str), + #[error("Invalid schema name {0}. It must match the regex '{1}'")] + InvalidSchemaName(String, &'static str), - #[error("Invalid namespace {0}. It must match the regex '{1}'")] - InvalidNamespace(String, &'static str), + #[error("Invalid namespace {0}. It must match the regex '{1}'")] + InvalidNamespace(String, &'static str), - #[error( - "Invalid schema: There is no type called '{0}', if you meant to define a non-primitive schema, it should be defined inside `type` attribute. Please review the specification" - )] - InvalidSchemaRecord(String), + #[error( + "Invalid schema: There is no type called '{0}', if you meant to define a non-primitive schema, it should be defined inside `type` attribute. Please review the specification" + )] + InvalidSchemaRecord(String), - #[error("Duplicate enum symbol {0}")] - EnumSymbolDuplicate(String), + #[error("Duplicate enum symbol {0}")] + EnumSymbolDuplicate(String), - #[error("Default value for enum must be a string! Got: {0}")] - EnumDefaultWrongType(serde_json::Value), + #[error("Default value for enum must be a string! Got: {0}")] + EnumDefaultWrongType(serde_json::Value), - #[error("No `items` in array")] - GetArrayItemsField, + #[error("No `items` in array")] + GetArrayItemsField, - #[error("No `values` in map")] - GetMapValuesField, + #[error("No `values` in map")] + GetMapValuesField, - #[error("Fixed schema `size` value must be a positive integer: {0}")] - GetFixedSizeFieldPositive(serde_json::Value), + #[error("Fixed schema `size` value must be a positive integer: {0}")] + GetFixedSizeFieldPositive(serde_json::Value), - #[error("Fixed schema has no `size`")] - GetFixedSizeField, + #[error("Fixed schema has no `size`")] + GetFixedSizeField, - #[error("Fixed schema's default value length ({0}) does not match its size ({1})")] - FixedDefaultLenSizeMismatch(usize, u64), + #[error("Fixed schema's default value length ({0}) does not match its size ({1})")] + FixedDefaultLenSizeMismatch(usize, u64), - #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] - #[error("Failed to compress with flate: {0}")] - DeflateCompress(#[source] std::io::Error), + #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")] + #[error("Failed to compress with flate: {0}")] + DeflateCompress(#[source] std::io::Error), - // no longer possible after migration from libflate to miniz_oxide - #[deprecated(since = "0.19.0", note = "This error can no longer occur")] - #[error("Failed to finish flate compressor: {0}")] - DeflateCompressFinish(#[source] std::io::Error), + // no longer possible after migration from libflate to miniz_oxide + #[deprecated(since = "0.19.0", note = "This error can no longer occur")] + #[error("Failed to finish flate compressor: {0}")] + DeflateCompressFinish(#[source] std::io::Error), - #[error("Failed to decompress with flate: {0}")] - DeflateDecompress(#[source] std::io::Error), + #[error("Failed to decompress with flate: {0}")] + DeflateDecompress(#[source] std::io::Error), - #[cfg(feature = "snappy")] - #[error("Failed to compress with snappy: {0}")] - SnappyCompress(#[source] snap::Error), + #[cfg(feature = "snappy")] + #[error("Failed to compress with snappy: {0}")] + SnappyCompress(#[source] snap::Error), - #[cfg(feature = "snappy")] - #[error("Failed to get snappy decompression length: {0}")] - GetSnappyDecompressLen(#[source] snap::Error), + #[cfg(feature = "snappy")] + #[error("Failed to get snappy decompression length: {0}")] + GetSnappyDecompressLen(#[source] snap::Error), - #[cfg(feature = "snappy")] - #[error("Failed to decompress with snappy: {0}")] - SnappyDecompress(#[source] snap::Error), + #[cfg(feature = "snappy")] + #[error("Failed to decompress with snappy: {0}")] + SnappyDecompress(#[source] snap::Error), - #[error("Failed to compress with zstd: {0}")] - ZstdCompress(#[source] std::io::Error), + #[error("Failed to compress with zstd: {0}")] + ZstdCompress(#[source] std::io::Error), - #[error("Failed to decompress with zstd: {0}")] - ZstdDecompress(#[source] std::io::Error), + #[error("Failed to decompress with zstd: {0}")] + ZstdDecompress(#[source] std::io::Error), - #[error("Failed to read header: {0}")] - ReadHeader(#[source] std::io::Error), + #[error("Failed to read header: {0}")] + ReadHeader(#[source] std::io::Error), - #[error("wrong magic in header")] - HeaderMagic, + #[error("wrong magic in header")] + HeaderMagic, - #[error("Message Header mismatch. Expected: {0:?}. Actual: {1:?}")] - SingleObjectHeaderMismatch(Vec, Vec), + #[error("Message Header mismatch. Expected: {0:?}. Actual: {1:?}")] + SingleObjectHeaderMismatch(Vec, Vec), - #[error("Failed to get JSON from avro.schema key in map")] - GetAvroSchemaFromMap, + #[error("Failed to get JSON from avro.schema key in map")] + GetAvroSchemaFromMap, - #[error("no metadata in header")] - GetHeaderMetadata, + #[error("no metadata in header")] + GetHeaderMetadata, - #[error("Failed to read marker bytes: {0}")] - ReadMarker(#[source] std::io::Error), + #[error("Failed to read marker bytes: {0}")] + ReadMarker(#[source] std::io::Error), - #[error("Failed to read block marker bytes: {0}")] - ReadBlockMarker(#[source] std::io::Error), + #[error("Failed to read block marker bytes: {0}")] + ReadBlockMarker(#[source] std::io::Error), - #[error("Read into buffer failed: {0}")] - ReadIntoBuf(#[source] std::io::Error), + #[error("Read into buffer failed: {0}")] + ReadIntoBuf(#[source] std::io::Error), - #[error( - "Invalid sync marker! The sync marker in the data block \ + #[error( + "Invalid sync marker! The sync marker in the data block \ doesn't match the file header's sync marker. This likely \ indicates data corruption, truncated file, or incorrectly \ concatenated Avro files. Verify file integrity and ensure \ proper file transmission or creation." - )] - GetBlockMarker, - - #[error("Overflow when decoding integer value")] - IntegerOverflow, + )] + GetBlockMarker, - #[error("Failed to read bytes for decoding variable length integer: {0}")] - ReadVariableIntegerBytes(#[source] std::io::Error), + #[error("Overflow when decoding integer value")] + IntegerOverflow, - #[error("Decoded integer out of range for i32: {1}: {0}")] - ZagI32(#[source] std::num::TryFromIntError, i64), + #[error("Failed to read bytes for decoding variable length integer: {0}")] + ReadVariableIntegerBytes(#[source] std::io::Error), - #[error("unable to read block")] - ReadBlock, + #[error("Decoded integer out of range for i32: {1}: {0}")] + ZagI32(#[source] std::num::TryFromIntError, i64), - #[error("Failed to serialize value into Avro value: {0}")] - SerializeValue(String), + #[error("unable to read block")] + ReadBlock, - #[error("Failed to serialize value of type {value_type} using schema {schema:?}: {value}")] - SerializeValueWithSchema { - value_type: &'static str, - value: String, - schema: Schema, - }, + #[error("Failed to serialize value into Avro value: {0}")] + SerializeValue(String), - #[error("Failed to serialize field '{field_name}' for record {record_schema:?}: {error}")] - SerializeRecordFieldWithSchema { - field_name: &'static str, - record_schema: Schema, - error: Box, - }, + #[error("Failed to serialize value of type {value_type} using schema {schema:?}: {value}")] + SerializeValueWithSchema { + value_type: &'static str, + value: String, + schema: Schema, + }, - #[error("Failed to deserialize Avro value into value: {0}")] - DeserializeValue(String), + #[error("Failed to serialize field '{field_name}' for record {record_schema:?}: {error}")] + SerializeRecordFieldWithSchema { + field_name: &'static str, + record_schema: Schema, + error: Box, + }, - #[error("Failed to write buffer bytes during flush: {0}")] - WriteBytes(#[source] std::io::Error), + #[error("Failed to deserialize Avro value into value: {0}")] + DeserializeValue(String), - #[error("Failed to flush inner writer during flush: {0}")] - FlushWriter(#[source] std::io::Error), + #[error("Failed to write buffer bytes during flush: {0}")] + WriteBytes(#[source] std::io::Error), - #[error("Failed to write marker: {0}")] - WriteMarker(#[source] std::io::Error), + #[error("Failed to flush inner writer during flush: {0}")] + FlushWriter(#[source] std::io::Error), - #[error("Failed to convert JSON to string: {0}")] - ConvertJsonToString(#[source] serde_json::Error), + #[error("Failed to write marker: {0}")] + WriteMarker(#[source] std::io::Error), - /// Error while converting float to json value - #[error("failed to convert avro float to json: {0}")] - ConvertF64ToJson(f64), + #[error("Failed to convert JSON to string: {0}")] + ConvertJsonToString(#[source] serde_json::Error), - /// Error while resolving Schema::Ref - #[error("Unresolved schema reference: {0}")] - SchemaResolutionError(Name), + /// Error while converting float to json value + #[error("failed to convert avro float to json: {0}")] + ConvertF64ToJson(f64), - #[error("The file metadata is already flushed.")] - FileHeaderAlreadyWritten, + /// Error while resolving Schema::Ref + #[error("Unresolved schema reference: {0}")] + SchemaResolutionError(Name), - #[error("Metadata keys starting with 'avro.' are reserved for internal usage: {0}.")] - InvalidMetadataKey(String), + #[error("The file metadata is already flushed.")] + FileHeaderAlreadyWritten, - /// Error when two named schema have the same fully qualified name - #[error("Two named schema defined for same fullname: {0}.")] - AmbiguousSchemaDefinition(Name), + #[error("Metadata keys starting with 'avro.' are reserved for internal usage: {0}.")] + InvalidMetadataKey(String), - #[error("Signed decimal bytes length {0} not equal to fixed schema size {1}.")] - EncodeDecimalAsFixedError(usize, usize), + /// Error when two named schema have the same fully qualified name + #[error("Two named schema defined for same fullname: {0}.")] + AmbiguousSchemaDefinition(Name), - #[error("There is no entry for '{0}' in the lookup table: {1}.")] - NoEntryInLookupTable(String, String), + #[error("Signed decimal bytes length {0} not equal to fixed schema size {1}.")] + EncodeDecimalAsFixedError(usize, usize), - #[error("Can only encode value type {value_kind:?} as one of {supported_schema:?}")] - EncodeValueAsSchemaError { - value_kind: ValueKind, - supported_schema: Vec, - }, - #[error( - "Internal buffer not drained properly. Re-initialize the single object writer struct!" - )] - IllegalSingleObjectWriterState, - - #[error("Codec '{0}' is not supported/enabled")] - CodecNotSupported(String), - - #[error("Invalid Avro data! Cannot read codec type from value that is not Value::Bytes.")] - BadCodecMetadata, - - #[error("Cannot convert a slice to Uuid: {0}")] - UuidFromSlice(#[source] uuid::Error), - } + #[error("There is no entry for '{0}' in the lookup table: {1}.")] + NoEntryInLookupTable(String, String), - #[derive(thiserror::Error, PartialEq)] - pub enum CompatibilityError { - #[error( - "Incompatible schema types! Writer schema is '{writer_schema_type}', but reader schema is '{reader_schema_type}'" - )] - WrongType { - writer_schema_type: String, - reader_schema_type: String, - }, - - #[error("Incompatible schema types! The {schema_type} should have been {expected_type:?}")] - TypeExpected { - schema_type: String, - expected_type: Vec, - }, - - #[error( - "Incompatible schemata! Field '{0}' in reader schema does not match the type in the writer schema" - )] - FieldTypeMismatch(String, #[source] Box), - - #[error("Incompatible schemata! Field '{0}' in reader schema must have a default value")] - MissingDefaultValue(String), + #[error("Can only encode value type {value_kind:?} as one of {supported_schema:?}")] + EncodeValueAsSchemaError { + value_kind: ValueKind, + supported_schema: Vec, + }, + #[error("Internal buffer not drained properly. Re-initialize the single object writer struct!")] + IllegalSingleObjectWriterState, - #[error("Incompatible schemata! Reader's symbols must contain all writer's symbols")] - MissingSymbols, + #[error("Codec '{0}' is not supported/enabled")] + CodecNotSupported(String), - #[error("Incompatible schemata! All elements in union must match for both schemas")] - MissingUnionElements, + #[error("Invalid Avro data! Cannot read codec type from value that is not Value::Bytes.")] + BadCodecMetadata, - #[error("Incompatible schemata! Name and size don't match for fixed")] - FixedMismatch, - - #[error( - "Incompatible schemata! The name must be the same for both schemas. Writer's name {writer_name} and reader's name {reader_name}" - )] - NameMismatch { - writer_name: String, - reader_name: String, - }, + #[error("Cannot convert a slice to Uuid: {0}")] + UuidFromSlice(#[source] uuid::Error), +} - #[error( - "Incompatible schemata! Unknown type for '{0}'. Make sure that the type is a valid one" - )] - Inconclusive(String), - } +#[derive(thiserror::Error, PartialEq)] +pub enum CompatibilityError { + #[error( + "Incompatible schema types! Writer schema is '{writer_schema_type}', but reader schema is '{reader_schema_type}'" + )] + WrongType { + writer_schema_type: String, + reader_schema_type: String, + }, + + #[error("Incompatible schema types! The {schema_type} should have been {expected_type:?}")] + TypeExpected { + schema_type: String, + expected_type: Vec, + }, + + #[error( + "Incompatible schemata! Field '{0}' in reader schema does not match the type in the writer schema" + )] + FieldTypeMismatch(String, #[source] Box), + + #[error("Incompatible schemata! Field '{0}' in reader schema must have a default value")] + MissingDefaultValue(String), + + #[error("Incompatible schemata! Reader's symbols must contain all writer's symbols")] + MissingSymbols, + + #[error("Incompatible schemata! All elements in union must match for both schemas")] + MissingUnionElements, + + #[error("Incompatible schemata! Name and size don't match for fixed")] + FixedMismatch, + + #[error( + "Incompatible schemata! The name must be the same for both schemas. Writer's name {writer_name} and reader's name {reader_name}" + )] + NameMismatch { + writer_name: String, + reader_name: String, + }, + + #[error( + "Incompatible schemata! Unknown type for '{0}'. Make sure that the type is a valid one" + )] + Inconclusive(String), +} - impl serde::ser::Error for Details { - fn custom(msg: T) -> Self { - Details::SerializeValue(msg.to_string()) - } +impl serde::ser::Error for Details { + fn custom(msg: T) -> Self { + Details::SerializeValue(msg.to_string()) } +} - impl serde::de::Error for Details { - fn custom(msg: T) -> Self { - Details::DeserializeValue(msg.to_string()) - } +impl serde::de::Error for Details { + fn custom(msg: T) -> Self { + Details::DeserializeValue(msg.to_string()) } +} - impl fmt::Debug for Details { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut msg = self.to_string(); - if let Some(e) = self.source() { - msg.extend([": ", &e.to_string()]); - } - write!(f, "{msg}") +impl fmt::Debug for Details { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut msg = self.to_string(); + if let Some(e) = self.source() { + msg.extend([": ", &e.to_string()]); } + write!(f, "{msg}") } +} - impl fmt::Debug for CompatibilityError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut msg = self.to_string(); - if let Some(e) = self.source() { - msg.extend([": ", &e.to_string()]); - } - write!(f, "{msg}") +impl fmt::Debug for CompatibilityError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut msg = self.to_string(); + if let Some(e) = self.source() { + msg.extend([": ", &e.to_string()]); } + write!(f, "{msg}") } } +// } diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 2a99e4d7..ae78718b 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -25,7 +25,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -39,14 +38,8 @@ mod headers { use uuid::Uuid; - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] use crate::AvroResult; - use crate::{ - error::tokio::Details, rabin::Rabin, schema::tokio::Schema, - schema::tokio::SchemaFingerprint, - }; + use crate::{error::Details, rabin::Rabin, schema::Schema, schema::SchemaFingerprint}; /// This trait represents that an object is able to construct an Avro message header. It is /// implemented for some known header types already. If you need a header type that is not already @@ -141,7 +134,7 @@ mod headers { #[cfg(test)] mod test { use super::*; - use crate::error::tokio::{Error, Details}; + use crate::error::{Details, Error}; use apache_avro_test_helper::TestResult; #[tokio::test] @@ -163,7 +156,7 @@ mod headers { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let header_builder = RabinFingerprintHeader::from_schema(&schema); let computed_header = header_builder.build_header(); let expected_header: Vec = vec![195, 1, 232, 198, 194, 12, 97, 95, 44, 71]; diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 895fe20e..12f50236 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -129,7 +129,7 @@ //! "#; //! //! // if the schema is not valid, this function will return an error -//! let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! //! // schemas can be printed for debugging //! println!("{:?}", schema); @@ -159,7 +159,7 @@ //! }"#; //! //! // if the schemas are not valid, this function will return an error -//! let schemas = Schema::parse_list(&[raw_schema_1, raw_schema_2]).unwrap(); +//! let schemas = SchemaExt::parse_list(&[raw_schema_1, raw_schema_2]).unwrap(); //! //! // schemas can be printed for debugging //! println!("{:?}", schemas); @@ -206,7 +206,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! // a writer needs a schema and something to write to //! let mut writer = Writer::new(&schema, Vec::new()); //! @@ -261,7 +261,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! // a writer needs a schema and something to write to //! let mut writer = Writer::new(&schema, Vec::new()); //! @@ -326,7 +326,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate(DeflateSettings::default())); //! ``` //! @@ -352,7 +352,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let mut record = Record::new(writer.schema()).unwrap(); //! # record.put("a", 27i64); @@ -381,7 +381,7 @@ //! # ] //! # } //! # "#; -//! # let writer_schema = Schema::parse_str(writer_raw_schema).await.unwrap(); +//! # let writer_schema = SchemaExt::parse_str(writer_raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&writer_schema, Vec::new()); //! # let mut record = Record::new(writer.schema()).unwrap(); //! # record.put("a", 27i64); @@ -401,7 +401,7 @@ //! } //! "#; //! -//! let reader_schema = Schema::parse_str(reader_raw_schema).await.unwrap(); +//! let reader_schema = SchemaExt::parse_str(reader_raw_schema).await.unwrap(); //! //! // reader creation can fail in case the input to read from is not Avro-compatible or malformed //! let reader = Reader::with_schema(&reader_schema, &input[..]).unwrap(); @@ -440,8 +440,8 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let mut record = Record::new(writer.schema()).unwrap(); //! # record.put("a", 27i64); @@ -486,7 +486,7 @@ //! # ] //! # } //! # "#; -//! # let schema = Schema::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let test = Test { //! # a: 27, @@ -529,7 +529,7 @@ //! } //! "#; //! -//! let schema = Schema::parse_str(raw_schema).await?; +//! let schema = SchemaExt::parse_str(raw_schema).await?; //! //! println!("{:?}", schema); //! @@ -655,7 +655,7 @@ //! } //! "#; //! -//! let schema = Schema::parse_str(raw_schema).await?; +//! let schema = SchemaExt::parse_str(raw_schema).await?; //! //! println!("{:?}", schema); //! @@ -715,7 +715,7 @@ //! ] //! } //! "#; -//! let schema = Schema::parse_str(raw_schema).await?; +//! let schema = SchemaExt::parse_str(raw_schema).await?; //! println!("{}", schema.fingerprint::()); //! println!("{}", schema.fingerprint::()); //! println!("{}", schema.fingerprint::()); @@ -766,8 +766,8 @@ //! ```rust //! use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; //! -//! let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); -//! let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); +//! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); +//! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); //! assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_ok()); //! ``` //! @@ -779,8 +779,8 @@ //! ```rust //! use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; //! -//! let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); -//! let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); +//! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); +//! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); //! assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_err()); //! ``` //! ## Custom names validators @@ -882,10 +882,7 @@ pub mod schema_equality; pub mod types; pub mod validator; -#[cfg(feature = "sync")] -pub use crate::bigdecimal::sync::BigDecimal; -#[cfg(feature = "tokio")] -pub use crate::bigdecimal::tokio::BigDecimal as AsyncBigDecimal; +pub use crate::bigdecimal::BigDecimal; pub use crate::bytes::{ serde_avro_bytes, serde_avro_bytes_opt, serde_avro_fixed, serde_avro_fixed_opt, serde_avro_slice, serde_avro_slice_opt, @@ -904,15 +901,9 @@ pub use codec::zstandard::ZstandardSettings; pub use de::sync::from_value; #[cfg(feature = "tokio")] pub use de::tokio::from_value as async_from_value; -#[cfg(feature = "sync")] -pub use decimal::sync::Decimal; -#[cfg(feature = "tokio")] -pub use decimal::tokio::Decimal as AsyncDecimal; +pub use decimal::Decimal; pub use duration::{Days, Duration, Millis, Months}; -#[cfg(feature = "sync")] -pub use error::sync::Error; -#[cfg(feature = "tokio")] -pub use error::tokio::Error as AsyncError; +pub use error::Error; #[cfg(feature = "sync")] pub use reader::sync::{ GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, @@ -926,10 +917,15 @@ pub use reader::tokio::{ from_avro_datum_reader_schemata as async_from_avro_datum_reader_schemata, from_avro_datum_schemata as async_from_avro_datum_schemata, read_marker as async_read_marker, }; +pub use schema::Schema; +#[cfg(feature = "sync")] +pub use schema::sync::AvroSchema; #[cfg(feature = "sync")] -pub use schema::sync::{AvroSchema, Schema}; +use schema::sync::SchemaExt; +#[cfg(feature = "tokio")] +pub use schema::tokio::AvroSchema as AsyncAvroSchema; #[cfg(feature = "tokio")] -pub use schema::tokio::{AvroSchema as AsyncAvroSchema, Schema as AsyncSchema}; +use schema::tokio::SchemaExt as AsyncSchemaExt; #[cfg(feature = "sync")] pub use ser::sync::to_value; #[cfg(feature = "tokio")] @@ -953,10 +949,6 @@ pub use writer::tokio::{ #[cfg(feature = "derive")] pub use apache_avro_derive::*; -/// A convenience type alias for `Result`s with `Error`s. -#[cfg(feature = "tokio")] -pub type AsyncAvroResult = Result; -#[cfg(feature = "sync")] pub type AvroResult = Result; #[synca::synca( @@ -980,8 +972,8 @@ mod tests { use crate::{ codec::tokio::Codec, reader::tokio::{Reader, from_avro_datum}, - schema::tokio::Schema, - types::tokio::{Record, Value}, + schema::Schema, + types::{Record, Value}, writer::tokio::Writer, }; #[synca::cfg(tokio)] @@ -1020,8 +1012,8 @@ mod tests { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema).await.unwrap(); - let reader_schema = Schema::parse_str(reader_raw_schema).await.unwrap(); + let writer_schema = SchemaExt::parse_str(writer_raw_schema).await.unwrap(); + let reader_schema = SchemaExt::parse_str(reader_raw_schema).await.unwrap(); let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); @@ -1064,7 +1056,7 @@ mod tests { ] } "#; - let schema = Schema::parse_str(raw_schema).await.unwrap(); + let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); @@ -1106,7 +1098,7 @@ mod tests { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema).await.unwrap(); + let writer_schema = SchemaExt::parse_str(writer_raw_schema).await.unwrap(); let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); @@ -1138,7 +1130,7 @@ mod tests { } "#; - let schema = Schema::parse_str(raw_schema).await.unwrap(); + let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); // Would allocated 18446744073709551605 bytes let illformed: &[u8] = &[0x3e, 0x15, 0xff, 0x1f, 0x15, 0xff]; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 99e77290..6d5ffb97 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -24,7 +24,6 @@ replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, crate::codec::tokio => crate::codec::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -38,9 +37,7 @@ } )] mod reader { - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] + use crate::AvroResult; #[synca::cfg(tokio)] use crate::async_from_value as from_value; @@ -58,14 +55,15 @@ mod reader { use crate::util::tokio::safe_len; use crate::{ codec::tokio::Codec, - error::tokio::Details, - error::tokio::Error, + error::Details, + error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, - schema::tokio::{ - AvroSchema, Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, + schema::tokio::{AvroSchema, SchemaExt}, + schema::{ + Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, resolve_names_with_schemata, }, - types::tokio::Value, + types::Value, util::tokio::read_long, }; use log::warn; @@ -278,10 +276,10 @@ mod reader { .iter() .map(|(name, schema)| (name.clone(), (*schema).clone())) .collect(); - self.writer_schema = Schema::parse_with_names(&json, names).await?; + self.writer_schema = SchemaExt::parse_with_names(&json, names).await?; resolve_names_with_schemata(&self.schemata, &mut self.names_refs, &None)?; } else { - self.writer_schema = Schema::parse(&json).await?; + self.writer_schema = SchemaExt::parse(&json).await?; resolve_names(&self.writer_schema, &mut self.names_refs, &None)?; } Ok(()) @@ -704,7 +702,7 @@ mod reader { use super::*; use crate::{ encode::tokio::encode, headers::tokio::GlueSchemaUuidHeader, rabin::Rabin, - types::tokio::Record, + types::Record, }; use apache_avro_test_helper::TestResult; #[synca::cfg(tokio)] @@ -752,7 +750,7 @@ mod reader { #[tokio::test] async fn test_from_avro_datum() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; let mut record = Record::new(&schema).unwrap(); @@ -812,7 +810,7 @@ mod reader { a_nullable_string: Option, } - let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240).await?; + let schema = SchemaExt::parse_str(TEST_RECORD_SCHEMA_3240).await?; let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111]; let expected_record: TestRecord3240 = TestRecord3240 { @@ -837,7 +835,7 @@ mod reader { #[tokio::test] async fn test_null_union() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA).await?; + let schema = SchemaExt::parse_str(UNION_SCHEMA).await?; let mut encoded: &'static [u8] = &[2, 0]; assert_eq!( @@ -850,7 +848,7 @@ mod reader { #[tokio::test] async fn test_reader_iterator() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut reader = Reader::with_schema(&schema, ENCODED).await?; let mut record1 = Record::new(&schema).unwrap(); @@ -874,7 +872,7 @@ mod reader { #[tokio::test] async fn test_reader_invalid_header() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let invalid = ENCODED.iter().copied().skip(1).collect::>(); assert!(Reader::with_schema(&schema, &invalid[..]).await.is_err()); @@ -883,7 +881,7 @@ mod reader { #[tokio::test] async fn test_reader_invalid_block() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let invalid = ENCODED .iter() .copied() @@ -924,7 +922,7 @@ mod reader { async fn test_avro_3405_read_user_metadata_success() -> TestResult { use crate::writer::tokio::Writer; - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut user_meta_data: HashMap> = HashMap::new(); @@ -987,7 +985,7 @@ mod reader { ] } "#; - Schema::parse_str(schema).await.unwrap() + SchemaExt::parse_str(schema).await.unwrap() } } diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 1a5c0b7f..4c7d8e84 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -17,695 +17,1396 @@ //! Logic for parsing and interacting with schemas in Avro format. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::de::tokio => crate::de::sync, - crate::decimal::tokio => crate::decimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::reader::tokio => crate::reader::sync, - crate::writer::tokio => crate::writer::sync, - crate::ser::tokio => crate::ser::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, - crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, - #[tokio::test] => #[test] - ); - } -)] -mod schema { +use crate::AvroResult; +use crate::error::{Details, Error}; +use crate::schema_equality::compare_schemata; +use crate::types::Value; +use crate::util::MapHelper; +use crate::validator::{validate_namespace, validate_schema_name}; +use digest::Digest; +use serde::{Deserialize, Serialize, Serializer, ser::SerializeMap, ser::SerializeSeq}; +use std::collections::HashMap; +use std::collections::{BTreeMap, HashSet}; +use strum_macros::{Display, EnumDiscriminants, EnumString}; + +/// Represents documentation for complex Avro schemas. +pub type Documentation = Option; +/// Represents the aliases for Named Schema +pub type Aliases = Option>; +/// Represents Schema lookup within a schema env +pub(crate) type Names = HashMap; +/// Represents Schema lookup within a schema +// pub type NamesRef<'a> = HashMap; +/// Represents the namespace for Named Schema +pub type Namespace = Option; + +/// Represents any valid order for a `field` in a `record` Avro schema. +#[derive(Clone, Debug, Eq, PartialEq, EnumString)] +#[strum(serialize_all = "kebab_case")] +pub enum RecordFieldOrder { + Ascending, + Descending, + Ignore, +} - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; - use crate::{ - error::tokio::{Details, Error}, - schema_equality::tokio::compare_schemata, - types::tokio::{Value, ValueKind}, - util::tokio::MapHelper, - validator::tokio::{ - validate_enum_symbol_name, validate_namespace, validate_record_field_name, - validate_schema_name, - }, - }; - use digest::Digest; - use log::{debug, error, warn}; - use serde::{ - Deserialize, Serialize, Serializer, - ser::{SerializeMap, SerializeSeq}, - }; - use std::{ - collections::{BTreeMap, HashMap, HashSet}, - fmt::Debug, - hash::Hash, - io::Read, - str::FromStr, - }; - use strum_macros::{Display, EnumDiscriminants, EnumString}; +/// Represents names for `record`, `enum` and `fixed` Avro schemas. +/// +/// Each of these `Schema`s have a `fullname` composed of two parts: +/// * a name +/// * a namespace +/// +/// `aliases` can also be defined, to facilitate schema evolution. +/// +/// More information about schema names can be found in the +/// [Avro specification](https://avro.apache.org/docs/current/specification/#names) +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct Name { + pub name: String, + pub namespace: Namespace, +} - /// Represents an Avro schema fingerprint - /// More information about Avro schema fingerprints can be found in the - /// [Avro Schema Fingerprint documentation](https://avro.apache.org/docs/current/specification/#schema-fingerprints) - pub struct SchemaFingerprint { - pub bytes: Vec, +impl Name { + /// Create a new `Name`. + /// Parses the optional `namespace` from the `name` string. + /// `aliases` will not be defined. + pub fn new(name: &str) -> AvroResult { + let (name, namespace) = Name::get_name_and_namespace(name)?; + Ok(Self { + name, + namespace: namespace.filter(|ns| !ns.is_empty()), + }) } - impl std::fmt::Display for SchemaFingerprint { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "{}", - self.bytes - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::>() - .join("") - ) + fn get_name_and_namespace(name: &str) -> AvroResult<(String, Namespace)> { + validate_schema_name(name) + } + + /// Parse a `serde_json::Value` into a `Name`. + pub(crate) fn parse( + complex: &serde_json::Map, + enclosing_namespace: &Namespace, + ) -> AvroResult { + let (name, namespace_from_name) = complex + .name() + .map(|name| Name::get_name_and_namespace(name.as_str()).unwrap()) + .ok_or(Details::GetNameField)?; + // FIXME Reading name from the type is wrong ! The name there is just a metadata (AVRO-3430) + let type_name = match complex.get("type") { + Some(serde_json::Value::Object(complex_type)) => complex_type.name().or(None), + _ => None, + }; + + let namespace = namespace_from_name + .or_else(|| { + complex + .string("namespace") + .or_else(|| enclosing_namespace.clone()) + }) + .filter(|ns| !ns.is_empty()); + + if let Some(ref ns) = namespace { + validate_namespace(ns)?; } + + Ok(Self { + name: type_name.unwrap_or(name), + namespace, + }) } - /// Represents any valid Avro schema - /// More information about Avro schemas can be found in the - /// [Avro Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) - #[derive(Clone, Debug, EnumDiscriminants, Display)] - #[strum_discriminants(name(SchemaKind), derive(Hash, Ord, PartialOrd))] - pub enum Schema { - /// A `null` Avro schema. - Null, - /// A `boolean` Avro schema. - Boolean, - /// An `int` Avro schema. - Int, - /// A `long` Avro schema. - Long, - /// A `float` Avro schema. - Float, - /// A `double` Avro schema. - Double, - /// A `bytes` Avro schema. - /// `Bytes` represents a sequence of 8-bit unsigned bytes. - Bytes, - /// A `string` Avro schema. - /// `String` represents a unicode character sequence. - String, - /// A `array` Avro schema. Avro arrays are required to have the same type for each element. - /// This variant holds the `Schema` for the array element type. - Array(ArraySchema), - /// A `map` Avro schema. - /// `Map` holds a pointer to the `Schema` of its values, which must all be the same schema. - /// `Map` keys are assumed to be `string`. - Map(MapSchema), - /// A `union` Avro schema. - Union(UnionSchema), - /// A `record` Avro schema. - Record(RecordSchema), - /// An `enum` Avro schema. - Enum(EnumSchema), - /// A `fixed` Avro schema. - Fixed(FixedSchema), - /// Logical type which represents `Decimal` values. The underlying type is serialized and - /// deserialized as `Schema::Bytes` or `Schema::Fixed`. - Decimal(DecimalSchema), - /// Logical type which represents `Decimal` values without predefined scale. - /// The underlying type is serialized and deserialized as `Schema::Bytes` - BigDecimal, - /// A universally unique identifier, annotating a string. - Uuid, - /// Logical type which represents the number of days since the unix epoch. - /// Serialization format is `Schema::Int`. - Date, - /// The time of day in number of milliseconds after midnight with no reference any calendar, - /// time zone or date in particular. - TimeMillis, - /// The time of day in number of microseconds after midnight with no reference any calendar, - /// time zone or date in particular. - TimeMicros, - /// An instant in time represented as the number of milliseconds after the UNIX epoch. - TimestampMillis, - /// An instant in time represented as the number of microseconds after the UNIX epoch. - TimestampMicros, - /// An instant in time represented as the number of nanoseconds after the UNIX epoch. - TimestampNanos, - /// An instant in localtime represented as the number of milliseconds after the UNIX epoch. - LocalTimestampMillis, - /// An instant in local time represented as the number of microseconds after the UNIX epoch. - LocalTimestampMicros, - /// An instant in local time represented as the number of nanoseconds after the UNIX epoch. - LocalTimestampNanos, - /// An amount of time defined by a number of months, days and milliseconds. - Duration, - /// A reference to another schema. - Ref { name: Name }, + /// Return the `fullname` of this `Name` + /// + /// More information about fullnames can be found in the + /// [Avro specification](https://avro.apache.org/docs/current/specification/#names) + pub fn fullname(&self, default_namespace: Namespace) -> String { + if self.name.contains('.') { + self.name.clone() + } else { + let namespace = self.namespace.clone().or(default_namespace); + + match namespace { + Some(ref namespace) if !namespace.is_empty() => { + format!("{}.{}", namespace, self.name) + } + _ => self.name.clone(), + } + } } - #[derive(Clone, Debug, PartialEq)] - pub struct MapSchema { - pub types: Box, - pub attributes: BTreeMap, + /// Return the fully qualified name needed for indexing or searching for the schema within a schema/schema env context. Puts the enclosing namespace into the name's namespace for clarity in schema/schema env parsing + /// ```ignore + /// use apache_avro::schema::Name; + /// + /// assert_eq!( + /// Name::new("some_name")?.fully_qualified_name(&Some("some_namespace".into())), + /// Name::new("some_namespace.some_name")? + /// ); + /// assert_eq!( + /// Name::new("some_namespace.some_name")?.fully_qualified_name(&Some("other_namespace".into())), + /// Name::new("some_namespace.some_name")? + /// ); + /// ``` + pub fn fully_qualified_name(&self, enclosing_namespace: &Namespace) -> Name { + Name { + name: self.name.clone(), + namespace: self + .namespace + .clone() + .or_else(|| enclosing_namespace.clone().filter(|ns| !ns.is_empty())), + } } +} - #[derive(Clone, Debug, PartialEq)] - pub struct ArraySchema { - pub items: Box, - pub attributes: BTreeMap, +impl From<&str> for Name { + fn from(name: &str) -> Self { + Name::new(name).unwrap() } +} - impl PartialEq for Schema { - /// Assess equality of two `Schema` based on [Parsing Canonical Form]. - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas - fn eq(&self, other: &Self) -> bool { - compare_schemata(self, other) - } +impl std::fmt::Display for Name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.fullname(None)[..]) } +} - impl SchemaKind { - pub fn is_primitive(self) -> bool { - matches!( - self, - SchemaKind::Null - | SchemaKind::Boolean - | SchemaKind::Int - | SchemaKind::Long - | SchemaKind::Double - | SchemaKind::Float - | SchemaKind::Bytes - | SchemaKind::String, - ) - } +impl<'de> Deserialize<'de> for Name { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + serde_json::Value::deserialize(deserializer).and_then(|value| { + use serde::de::Error; + if let serde_json::Value::Object(json) = value { + Name::parse(&json, &None).map_err(Error::custom) + } else { + Err(Error::custom(format!("Expected a JSON object: {value:?}"))) + } + }) + } +} - pub fn is_named(self) -> bool { - matches!( - self, - SchemaKind::Record | SchemaKind::Enum | SchemaKind::Fixed | SchemaKind::Ref - ) +/// Newtype pattern for `Name` to better control the `serde_json::Value` representation. +/// Aliases are serialized as an array of plain strings in the JSON representation. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct Alias(Name); + +impl Alias { + pub fn new(name: &str) -> AvroResult { + Name::new(name).map(Self) + } + + pub fn name(&self) -> String { + self.0.name.clone() + } + + pub fn namespace(&self) -> Namespace { + self.0.namespace.clone() + } + + pub fn fullname(&self, default_namespace: Namespace) -> String { + self.0.fullname(default_namespace) + } + + pub fn fully_qualified_name(&self, default_namespace: &Namespace) -> Name { + self.0.fully_qualified_name(default_namespace) + } +} + +impl From<&str> for Alias { + fn from(name: &str) -> Self { + Alias::new(name).unwrap() + } +} + +impl Serialize for Alias { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.fullname(None)) + } +} + +#[derive(Debug)] +pub struct ResolvedSchema<'s> { + names_ref: Names, + schemata: Vec<&'s Schema>, +} + +impl<'s> TryFrom<&'s Schema> for ResolvedSchema<'s> { + type Error = Error; + + fn try_from(schema: &'s Schema) -> AvroResult { + let mut rs = ResolvedSchema { + names_ref: Default::default(), + schemata: vec![schema], + }; + rs.resolve(rs.get_schemata(), &None, None)?; + Ok(rs) + } +} + +impl<'s> TryFrom> for ResolvedSchema<'s> { + type Error = Error; + + fn try_from(schemata: Vec<&'s Schema>) -> AvroResult { + let mut rs = ResolvedSchema { + names_ref: Default::default(), + schemata, + }; + rs.resolve(rs.get_schemata(), &None, None)?; + Ok(rs) + } +} + +impl<'s> ResolvedSchema<'s> { + pub fn get_schemata(&self) -> Vec<&'s Schema> { + self.schemata.clone() + } + + pub fn get_names(&self) -> &Names { + &self.names_ref + } + + /// Creates `ResolvedSchema` with some already known schemas. + /// + /// Those schemata would be used to resolve references if needed. + pub fn new_with_known_schemata<'n>( + schemata_to_resolve: Vec<&'s Schema>, + enclosing_namespace: &Namespace, + known_schemata: &Names, + ) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedSchema { + names_ref: names, + schemata: schemata_to_resolve, + }; + rs.resolve(rs.get_schemata(), enclosing_namespace, Some(known_schemata))?; + Ok(rs) + } + + fn resolve<'n>( + &mut self, + schemata: Vec<&'s Schema>, + enclosing_namespace: &Namespace, + known_schemata: Option<&Names>, + ) -> AvroResult<()> { + for schema in schemata { + match schema { + Schema::Array(schema) => { + self.resolve(vec![&schema.items], enclosing_namespace, known_schemata)? + } + Schema::Map(schema) => { + self.resolve(vec![&schema.types], enclosing_namespace, known_schemata)? + } + Schema::Union(UnionSchema { schemas, .. }) => { + for schema in schemas { + self.resolve(vec![schema], enclosing_namespace, known_schemata)? + } + } + Schema::Enum(EnumSchema { name, .. }) | Schema::Fixed(FixedSchema { name, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if self + .names_ref + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + return Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()); + } + } + Schema::Record(RecordSchema { name, fields, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if self + .names_ref + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + return Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()); + } else { + let record_namespace = fully_qualified_name.namespace; + for field in fields { + self.resolve(vec![&field.schema], &record_namespace, known_schemata)? + } + } + } + Schema::Ref { name } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + // first search for reference in current schemata, then look into external references. + if !self.names_ref.contains_key(&fully_qualified_name) { + let is_resolved_with_known_schemas = known_schemata + .as_ref() + .map(|names| names.contains_key(&fully_qualified_name)) + .unwrap_or(false); + if !is_resolved_with_known_schemas { + return Err(Details::SchemaResolutionError(fully_qualified_name).into()); + } + } + } + _ => (), + } } + Ok(()) } +} + +pub(crate) struct ResolvedOwnedSchema { + names: Names, + root_schema: Schema, +} + +impl TryFrom for ResolvedOwnedSchema { + type Error = Error; - impl From<&Value> for SchemaKind { - fn from(value: &Value) -> Self { - use crate::types::tokio::Value; - match value { - Value::Null => Self::Null, - Value::Boolean(_) => Self::Boolean, - Value::Int(_) => Self::Int, - Value::Long(_) => Self::Long, - Value::Float(_) => Self::Float, - Value::Double(_) => Self::Double, - Value::Bytes(_) => Self::Bytes, - Value::String(_) => Self::String, - Value::Array(_) => Self::Array, - Value::Map(_) => Self::Map, - Value::Union(_, _) => Self::Union, - Value::Record(_) => Self::Record, - Value::Enum(_, _) => Self::Enum, - Value::Fixed(_, _) => Self::Fixed, - Value::Decimal { .. } => Self::Decimal, - Value::BigDecimal(_) => Self::BigDecimal, - Value::Uuid(_) => Self::Uuid, - Value::Date(_) => Self::Date, - Value::TimeMillis(_) => Self::TimeMillis, - Value::TimeMicros(_) => Self::TimeMicros, - Value::TimestampMillis(_) => Self::TimestampMillis, - Value::TimestampMicros(_) => Self::TimestampMicros, - Value::TimestampNanos(_) => Self::TimestampNanos, - Value::LocalTimestampMillis(_) => Self::LocalTimestampMillis, - Value::LocalTimestampMicros(_) => Self::LocalTimestampMicros, - Value::LocalTimestampNanos(_) => Self::LocalTimestampNanos, - Value::Duration { .. } => Self::Duration, + fn try_from(schema: Schema) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedOwnedSchema { + names, + root_schema: schema, + }; + resolve_names(&rs.root_schema, &mut rs.names, &None)?; + Ok(rs) + } +} + +impl ResolvedOwnedSchema { + pub(crate) fn get_root_schema(&self) -> &Schema { + &self.root_schema + } + pub(crate) fn get_names(&self) -> &Names { + &self.names + } +} + +pub(crate) fn resolve_names( + schema: &Schema, + names: &mut Names, + enclosing_namespace: &Namespace, +) -> AvroResult<()> { + match schema { + Schema::Array(schema) => resolve_names(&schema.items, names, enclosing_namespace), + Schema::Map(schema) => resolve_names(&schema.types, names, enclosing_namespace), + Schema::Union(UnionSchema { schemas, .. }) => { + for schema in schemas { + resolve_names(schema, names, enclosing_namespace)? + } + Ok(()) + } + Schema::Enum(EnumSchema { name, .. }) | Schema::Fixed(FixedSchema { name, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if names + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) + } else { + Ok(()) + } + } + Schema::Record(RecordSchema { name, fields, .. }) => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if names + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) + } else { + let record_namespace = fully_qualified_name.namespace; + for field in fields { + resolve_names(&field.schema, names, &record_namespace)? + } + Ok(()) } } + Schema::Ref { name } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + names + .get(&fully_qualified_name) + .map(|_| ()) + .ok_or_else(|| Details::SchemaResolutionError(fully_qualified_name).into()) + } + _ => Ok(()), } +} - /// Represents names for `record`, `enum` and `fixed` Avro schemas. - /// - /// Each of these `Schema`s have a `fullname` composed of two parts: - /// * a name - /// * a namespace - /// - /// `aliases` can also be defined, to facilitate schema evolution. - /// - /// More information about schema names can be found in the - /// [Avro specification](https://avro.apache.org/docs/current/specification/#names) - #[derive(Clone, Debug, Hash, PartialEq, Eq)] - pub struct Name { - pub name: String, - pub namespace: Namespace, +pub(crate) fn resolve_names_with_schemata( + schemata: &Vec<&Schema>, + names: &mut Names, + enclosing_namespace: &Namespace, +) -> AvroResult<()> { + for schema in schemata { + resolve_names(schema, names, enclosing_namespace)?; } + Ok(()) +} - /// Represents documentation for complex Avro schemas. - pub type Documentation = Option; - /// Represents the aliases for Named Schema - pub type Aliases = Option>; - /// Represents Schema lookup within a schema env - pub(crate) type Names = HashMap; - /// Represents Schema lookup within a schema - // pub type NamesRef<'a> = HashMap; - /// Represents the namespace for Named Schema - pub type Namespace = Option; - - impl Name { - /// Create a new `Name`. - /// Parses the optional `namespace` from the `name` string. - /// `aliases` will not be defined. - pub fn new(name: &str) -> AvroResult { - let (name, namespace) = Name::get_name_and_namespace(name)?; - Ok(Self { - name, - namespace: namespace.filter(|ns| !ns.is_empty()), - }) +/// Represents a `field` in a `record` Avro schema. +#[derive(bon::Builder, Clone, Debug, PartialEq)] +pub struct RecordField { + /// Name of the field. + pub name: String, + /// Documentation of the field. + #[builder(default)] + pub doc: Documentation, + /// Aliases of the field's name. They have no namespace. + pub aliases: Option>, + /// Default value of the field. + /// This value will be used when reading Avro datum if schema resolution + /// is enabled. + pub default: Option, + /// Schema of the field. + pub schema: Schema, + /// Order of the field. + /// + /// **NOTE** This currently has no effect. + #[builder(default = RecordFieldOrder::Ignore)] + pub order: RecordFieldOrder, + /// Position of the field in the list of `field` of its parent `Schema` + #[builder(default)] + pub position: usize, + /// A collection of all unknown fields in the record field. + #[builder(default = BTreeMap::new())] + pub custom_attributes: BTreeMap, +} + +/// A description of an Enum schema. +#[derive(bon::Builder, Debug, Clone)] +pub struct RecordSchema { + /// The name of the schema + pub name: Name, + /// The aliases of the schema + #[builder(default)] + pub aliases: Aliases, + /// The documentation of the schema + #[builder(default)] + pub doc: Documentation, + /// The set of fields of the schema + pub fields: Vec, + /// The `lookup` table maps field names to their position in the `Vec` + /// of `fields`. + pub lookup: BTreeMap, + /// The custom attributes of the schema + #[builder(default = BTreeMap::new())] + pub attributes: BTreeMap, +} + +/// A description of an Enum schema. +#[derive(bon::Builder, Debug, Clone)] +pub struct EnumSchema { + /// The name of the schema + pub name: Name, + /// The aliases of the schema + #[builder(default)] + pub aliases: Aliases, + /// The documentation of the schema + #[builder(default)] + pub doc: Documentation, + /// The set of symbols of the schema + pub symbols: Vec, + /// An optional default symbol used for compatibility + pub default: Option, + /// The custom attributes of the schema + #[builder(default = BTreeMap::new())] + pub attributes: BTreeMap, +} + +/// A description of a Union schema. +#[derive(bon::Builder, Debug, Clone)] +pub struct FixedSchema { + /// The name of the schema + pub name: Name, + /// The aliases of the schema + #[builder(default)] + pub aliases: Aliases, + /// The documentation of the schema + #[builder(default)] + pub doc: Documentation, + /// The size of the fixed schema + pub size: usize, + /// An optional default symbol used for compatibility + pub default: Option, + /// The custom attributes of the schema + #[builder(default = BTreeMap::new())] + pub attributes: BTreeMap, +} + +impl FixedSchema { + fn serialize_to_map(&self, mut map: S::SerializeMap) -> Result + where + S: Serializer, + { + map.serialize_entry("type", "fixed")?; + if let Some(ref n) = self.name.namespace { + map.serialize_entry("namespace", n)?; } + map.serialize_entry("name", &self.name.name)?; + if let Some(ref docstr) = self.doc { + map.serialize_entry("doc", docstr)?; + } + map.serialize_entry("size", &self.size)?; - fn get_name_and_namespace(name: &str) -> AvroResult<(String, Namespace)> { - validate_schema_name(name) + if let Some(ref aliases) = self.aliases { + map.serialize_entry("aliases", aliases)?; } - /// Parse a `serde_json::Value` into a `Name`. - pub(crate) fn parse( - complex: &serde_json::Map, - enclosing_namespace: &Namespace, - ) -> AvroResult { - let (name, namespace_from_name) = complex - .name() - .map(|name| Name::get_name_and_namespace(name.as_str()).unwrap()) - .ok_or(Details::GetNameField)?; - // FIXME Reading name from the type is wrong ! The name there is just a metadata (AVRO-3430) - let type_name = match complex.get("type") { - Some(serde_json::Value::Object(complex_type)) => complex_type.name().or(None), - _ => None, - }; + for attr in &self.attributes { + map.serialize_entry(attr.0, attr.1)?; + } - let namespace = namespace_from_name - .or_else(|| { - complex - .string("namespace") - .or_else(|| enclosing_namespace.clone()) - }) - .filter(|ns| !ns.is_empty()); + Ok(map) + } +} + +// No need to compare variant_index, it is derivative of schemas. +impl PartialEq for UnionSchema { + fn eq(&self, other: &UnionSchema) -> bool { + self.schemas.eq(&other.schemas) + } +} - if let Some(ref ns) = namespace { - validate_namespace(ns)?; +type DecimalMetadata = usize; +pub(crate) type Precision = DecimalMetadata; +pub(crate) type Scale = DecimalMetadata; + +/// A description of a Decimal schema. +/// +/// `scale` defaults to 0 and is an integer greater than or equal to 0 and `precision` is an +/// integer greater than 0. +#[derive(Debug, Clone)] +pub struct DecimalSchema { + /// The number of digits in the unscaled value + pub precision: Precision, + /// The number of digits to the right of the decimal point + pub scale: Scale, + /// The inner schema of the decimal (fixed or bytes) + pub inner: Box, +} + +/// A description of a Union schema +#[derive(Debug, Clone)] +pub struct UnionSchema { + /// The schemas that make up this union + pub(crate) schemas: Vec, + // Used to ensure uniqueness of schema inputs, and provide constant time finding of the + // schema index given a value. + // **NOTE** that this approach does not work for named types, and will have to be modified + // to support that. A simple solution is to also keep a mapping of the names used. + variant_index: BTreeMap, +} + +impl UnionSchema { + /// Creates a new UnionSchema from a vector of schemas. + pub fn new(schemas: Vec) -> AvroResult { + let mut vindex = BTreeMap::new(); + for (i, schema) in schemas.iter().enumerate() { + if let Schema::Union(_) = schema { + return Err(Details::GetNestedUnion.into()); + } + let kind = SchemaKind::from(schema); + if !kind.is_named() && vindex.insert(kind, i).is_some() { + return Err(Details::GetUnionDuplicate.into()); } + } + Ok(UnionSchema { + schemas, + variant_index: vindex, + }) + } - Ok(Self { - name: type_name.unwrap_or(name), - namespace, - }) + /// Returns a slice to all variants of this schema. + pub fn variants(&self) -> &[Schema] { + &self.schemas + } + + /// Returns true if the any of the variants of this `UnionSchema` is `Null`. + pub fn is_nullable(&self) -> bool { + self.schemas.iter().any(|x| matches!(x, Schema::Null)) + } +} + +/// Represents an Avro schema fingerprint +/// More information about Avro schema fingerprints can be found in the +/// [Avro Schema Fingerprint documentation](https://avro.apache.org/docs/current/specification/#schema-fingerprints) +pub struct SchemaFingerprint { + pub bytes: Vec, +} + +impl std::fmt::Display for SchemaFingerprint { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + self.bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join("") + ) + } +} + +/// Represents any valid Avro schema +/// More information about Avro schemas can be found in the +/// [Avro Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) +#[derive(Clone, Debug, EnumDiscriminants, Display)] +#[strum_discriminants(name(SchemaKind), derive(Hash, Ord, PartialOrd))] +pub enum Schema { + /// A `null` Avro schema. + Null, + /// A `boolean` Avro schema. + Boolean, + /// An `int` Avro schema. + Int, + /// A `long` Avro schema. + Long, + /// A `float` Avro schema. + Float, + /// A `double` Avro schema. + Double, + /// A `bytes` Avro schema. + /// `Bytes` represents a sequence of 8-bit unsigned bytes. + Bytes, + /// A `string` Avro schema. + /// `String` represents a unicode character sequence. + String, + /// A `array` Avro schema. Avro arrays are required to have the same type for each element. + /// This variant holds the `Schema` for the array element type. + Array(ArraySchema), + /// A `map` Avro schema. + /// `Map` holds a pointer to the `Schema` of its values, which must all be the same schema. + /// `Map` keys are assumed to be `string`. + Map(MapSchema), + /// A `union` Avro schema. + Union(UnionSchema), + /// A `record` Avro schema. + Record(RecordSchema), + /// An `enum` Avro schema. + Enum(EnumSchema), + /// A `fixed` Avro schema. + Fixed(FixedSchema), + /// Logical type which represents `Decimal` values. The underlying type is serialized and + /// deserialized as `Schema::Bytes` or `Schema::Fixed`. + Decimal(DecimalSchema), + /// Logical type which represents `Decimal` values without predefined scale. + /// The underlying type is serialized and deserialized as `Schema::Bytes` + BigDecimal, + /// A universally unique identifier, annotating a string. + Uuid, + /// Logical type which represents the number of days since the unix epoch. + /// Serialization format is `Schema::Int`. + Date, + /// The time of day in number of milliseconds after midnight with no reference any calendar, + /// time zone or date in particular. + TimeMillis, + /// The time of day in number of microseconds after midnight with no reference any calendar, + /// time zone or date in particular. + TimeMicros, + /// An instant in time represented as the number of milliseconds after the UNIX epoch. + TimestampMillis, + /// An instant in time represented as the number of microseconds after the UNIX epoch. + TimestampMicros, + /// An instant in time represented as the number of nanoseconds after the UNIX epoch. + TimestampNanos, + /// An instant in localtime represented as the number of milliseconds after the UNIX epoch. + LocalTimestampMillis, + /// An instant in local time represented as the number of microseconds after the UNIX epoch. + LocalTimestampMicros, + /// An instant in local time represented as the number of nanoseconds after the UNIX epoch. + LocalTimestampNanos, + /// An amount of time defined by a number of months, days and milliseconds. + Duration, + /// A reference to another schema. + Ref { name: Name }, +} + +impl Schema { + /// Converts `self` into its [Parsing Canonical Form]. + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + pub fn canonical_form(&self) -> String { + let json = serde_json::to_value(self) + .unwrap_or_else(|e| panic!("Cannot parse Schema from JSON: {e}")); + let mut defined_names = HashSet::new(); + Self::parsing_canonical_form(&json, &mut defined_names) + } + + /// Returns the [Parsing Canonical Form] of `self` that is self contained (not dependent on + /// any definitions in `schemata`) + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + pub fn independent_canonical_form(&self, schemata: &[Schema]) -> Result { + let mut this = self.clone(); + this.denormalize(schemata)?; + Ok(this.canonical_form()) + } + + /// Parses a **valid** avro schema into the Parsing Canonical Form. + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + fn parsing_canonical_form( + schema: &serde_json::Value, + defined_names: &mut HashSet, + ) -> String { + match schema { + serde_json::Value::Object(map) => Self::pcf_map(map, defined_names), + serde_json::Value::String(s) => Self::pcf_string(s), + serde_json::Value::Array(v) => Self::pcf_array(v, defined_names), + json => panic!("got invalid JSON value for canonical form of schema: {json}"), } + } - /// Return the `fullname` of this `Name` - /// - /// More information about fullnames can be found in the - /// [Avro specification](https://avro.apache.org/docs/current/specification/#names) - pub fn fullname(&self, default_namespace: Namespace) -> String { - if self.name.contains('.') { - self.name.clone() + fn pcf_map( + schema: &serde_json::Map, + defined_names: &mut HashSet, + ) -> String { + // Look for the namespace variant up front. + let ns = schema.get("namespace").and_then(|v| v.as_str()); + let typ = schema.get("type").and_then(|v| v.as_str()); + let raw_name = schema.get("name").and_then(|v| v.as_str()); + let name = if Self::is_named_type(typ) { + Some(format!( + "{}{}", + ns.map_or("".to_string(), |n| { format!("{n}.") }), + raw_name.unwrap_or_default() + )) + } else { + None + }; + + //if this is already a defined type, early return + if let Some(ref n) = name { + if defined_names.contains(n) { + return Self::pcf_string(n); } else { - let namespace = self.namespace.clone().or(default_namespace); + defined_names.insert(n.clone()); + } + } - match namespace { - Some(ref namespace) if !namespace.is_empty() => { - format!("{}.{}", namespace, self.name) - } - _ => self.name.clone(), + let mut fields = Vec::new(); + for (k, v) in schema { + // Reduce primitive types to their simple form. ([PRIMITIVE] rule) + if schema.len() == 1 && k == "type" { + // Invariant: function is only callable from a valid schema, so this is acceptable. + if let serde_json::Value::String(s) = v { + return Self::pcf_string(s); } } - } - /// Return the fully qualified name needed for indexing or searching for the schema within a schema/schema env context. Puts the enclosing namespace into the name's namespace for clarity in schema/schema env parsing - /// ```ignore - /// use apache_avro::schema::Name; - /// - /// assert_eq!( - /// Name::new("some_name")?.fully_qualified_name(&Some("some_namespace".into())), - /// Name::new("some_namespace.some_name")? - /// ); - /// assert_eq!( - /// Name::new("some_namespace.some_name")?.fully_qualified_name(&Some("other_namespace".into())), - /// Name::new("some_namespace.some_name")? - /// ); - /// ``` - pub fn fully_qualified_name(&self, enclosing_namespace: &Namespace) -> Name { - Name { - name: self.name.clone(), - namespace: self - .namespace - .clone() - .or_else(|| enclosing_namespace.clone().filter(|ns| !ns.is_empty())), + // Strip out unused fields ([STRIP] rule) + if Self::field_ordering_position(k).is_none() + || k == "default" + || k == "doc" + || k == "aliases" + || k == "logicalType" + { + continue; + } + + // Fully qualify the name, if it isn't already ([FULLNAMES] rule). + if k == "name" { + if let Some(ref n) = name { + fields.push(( + "name", + format!("{}:{}", Self::pcf_string(k), Self::pcf_string(n)), + )); + continue; + } } + + // Strip off quotes surrounding "size" type, if they exist ([INTEGERS] rule). + if k == "size" || k == "precision" || k == "scale" { + let i = match v.as_str() { + Some(s) => s.parse::().expect("Only valid schemas are accepted!"), + None => v.as_i64().unwrap(), + }; + fields.push((k, format!("{}:{}", Self::pcf_string(k), i))); + continue; + } + + // For anything else, recursively process the result. + fields.push(( + k, + format!( + "{}:{}", + Self::pcf_string(k), + Self::parsing_canonical_form(v, defined_names) + ), + )); } + + // Sort the fields by their canonical ordering ([ORDER] rule). + fields.sort_unstable_by_key(|(k, _)| Self::field_ordering_position(k).unwrap()); + let inter = fields + .into_iter() + .map(|(_, v)| v) + .collect::>() + .join(","); + format!("{{{inter}}}") } - impl From<&str> for Name { - fn from(name: &str) -> Self { - Name::new(name).unwrap() - } + fn is_named_type(typ: Option<&str>) -> bool { + matches!( + typ, + Some("record") | Some("enum") | Some("fixed") | Some("ref") + ) } - impl std::fmt::Display for Name { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.fullname(None)[..]) - } + fn pcf_array(arr: &[serde_json::Value], defined_names: &mut HashSet) -> String { + let inter = arr + .iter() + .map(|a| parsing_canonical_form(a, defined_names)) + .collect::>() + .join(","); + format!("[{inter}]") } - impl<'de> Deserialize<'de> for Name { - fn deserialize(deserializer: D) -> Result - where - D: serde::de::Deserializer<'de>, - { - serde_json::Value::deserialize(deserializer).and_then(|value| { - use serde::de::Error; - if let serde_json::Value::Object(json) = value { - Name::parse(&json, &None).map_err(Error::custom) - } else { - Err(Error::custom(format!("Expected a JSON object: {value:?}"))) - } - }) - } + fn pcf_string(s: &str) -> String { + format!("\"{s}\"") } - /// Newtype pattern for `Name` to better control the `serde_json::Value` representation. - /// Aliases are serialized as an array of plain strings in the JSON representation. - #[derive(Clone, Debug, Hash, PartialEq, Eq)] - pub struct Alias(Name); + const RESERVED_FIELDS: &[&str] = &[ + "name", + "type", + "fields", + "symbols", + "items", + "values", + "size", + "logicalType", + "order", + "doc", + "aliases", + "default", + "precision", + "scale", + ]; - impl Alias { - pub fn new(name: &str) -> AvroResult { - Name::new(name).map(Self) - } + // Used to define the ordering and inclusion of fields. + fn field_ordering_position(field: &str) -> Option { + Self::RESERVED_FIELDS + .iter() + .position(|&f| f == field) + .map(|pos| pos + 1) + } - pub fn name(&self) -> String { - self.0.name.clone() + /// Generate [fingerprint] of Schema's [Parsing Canonical Form]. + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas + /// [fingerprint]: + /// https://avro.apache.org/docs/current/specification/#schema-fingerprints + pub fn fingerprint(&self) -> SchemaFingerprint { + let mut d = D::new(); + d.update(self.canonical_form()); + SchemaFingerprint { + bytes: d.finalize().to_vec(), } + } - pub fn namespace(&self) -> Namespace { - self.0.namespace.clone() + /// Returns the custom attributes (metadata) if the schema supports them. + pub fn custom_attributes(&self) -> Option<&BTreeMap> { + match self { + Schema::Record(RecordSchema { attributes, .. }) + | Schema::Enum(EnumSchema { attributes, .. }) + | Schema::Fixed(FixedSchema { attributes, .. }) + | Schema::Array(ArraySchema { attributes, .. }) + | Schema::Map(MapSchema { attributes, .. }) => Some(attributes), + _ => None, } + } - pub fn fullname(&self, default_namespace: Namespace) -> String { - self.0.fullname(default_namespace) + /// Returns the name of the schema if it has one. + pub fn name(&self) -> Option<&Name> { + match self { + Schema::Ref { name, .. } + | Schema::Record(RecordSchema { name, .. }) + | Schema::Enum(EnumSchema { name, .. }) + | Schema::Fixed(FixedSchema { name, .. }) => Some(name), + _ => None, } + } - pub fn fully_qualified_name(&self, default_namespace: &Namespace) -> Name { - self.0.fully_qualified_name(default_namespace) - } + /// Returns the namespace of the schema if it has one. + pub fn namespace(&self) -> Namespace { + self.name().and_then(|n| n.namespace.clone()) } - impl From<&str> for Alias { - fn from(name: &str) -> Self { - Alias::new(name).unwrap() + /// Returns the aliases of the schema if it has ones. + pub fn aliases(&self) -> Option<&Vec> { + match self { + Schema::Record(RecordSchema { aliases, .. }) + | Schema::Enum(EnumSchema { aliases, .. }) + | Schema::Fixed(FixedSchema { aliases, .. }) => aliases.as_ref(), + _ => None, } } - impl Serialize for Alias { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.fullname(None)) + /// Returns the doc of the schema if it has one. + pub fn doc(&self) -> Option<&String> { + match self { + Schema::Record(RecordSchema { doc, .. }) + | Schema::Enum(EnumSchema { doc, .. }) + | Schema::Fixed(FixedSchema { doc, .. }) => doc.as_ref(), + _ => None, } } - #[derive(Debug)] - pub struct ResolvedSchema<'s> { - names_ref: Names, - schemata: Vec<&'s Schema>, + /// Returns a Schema::Map with the given types. + pub fn map(types: Schema) -> Self { + Schema::Map(MapSchema { + types: Box::new(types), + attributes: Default::default(), + }) } - impl<'s> TryFrom<&'s Schema> for ResolvedSchema<'s> { - type Error = Error; - - fn try_from(schema: &'s Schema) -> AvroResult { - let mut rs = ResolvedSchema { - names_ref: Default::default(), - schemata: vec![schema], - }; - rs.resolve(rs.get_schemata(), &None, None)?; - Ok(rs) - } + /// Returns a Schema::Map with the given types and custom attributes. + pub fn map_with_attributes( + types: Schema, + attributes: BTreeMap, + ) -> Self { + Schema::Map(MapSchema { + types: Box::new(types), + attributes, + }) } - impl<'s> TryFrom> for ResolvedSchema<'s> { - type Error = Error; - - fn try_from(schemata: Vec<&'s Schema>) -> AvroResult { - let mut rs = ResolvedSchema { - names_ref: Default::default(), - schemata, - }; - rs.resolve(rs.get_schemata(), &None, None)?; - Ok(rs) - } + /// Returns a Schema::Array with the given items. + pub fn array(items: Schema) -> Self { + Schema::Array(ArraySchema { + items: Box::new(items), + attributes: Default::default(), + }) } - impl<'s> ResolvedSchema<'s> { - pub fn get_schemata(&self) -> Vec<&'s Schema> { - self.schemata.clone() - } - - pub fn get_names(&self) -> &Names { - &self.names_ref - } - - /// Creates `ResolvedSchema` with some already known schemas. - /// - /// Those schemata would be used to resolve references if needed. - pub fn new_with_known_schemata<'n>( - schemata_to_resolve: Vec<&'s Schema>, - enclosing_namespace: &Namespace, - known_schemata: &Names, - ) -> AvroResult { - let names = HashMap::new(); - let mut rs = ResolvedSchema { - names_ref: names, - schemata: schemata_to_resolve, - }; - rs.resolve(rs.get_schemata(), enclosing_namespace, Some(known_schemata))?; - Ok(rs) - } + /// Returns a Schema::Array with the given items and custom attributes. + pub fn array_with_attributes( + items: Schema, + attributes: BTreeMap, + ) -> Self { + Schema::Array(ArraySchema { + items: Box::new(items), + attributes, + }) + } - fn resolve<'n>( - &mut self, - schemata: Vec<&'s Schema>, - enclosing_namespace: &Namespace, - known_schemata: Option<&Names>, - ) -> AvroResult<()> { - for schema in schemata { - match schema { - Schema::Array(schema) => { - self.resolve(vec![&schema.items], enclosing_namespace, known_schemata)? - } - Schema::Map(schema) => { - self.resolve(vec![&schema.types], enclosing_namespace, known_schemata)? - } - Schema::Union(UnionSchema { schemas, .. }) => { - for schema in schemas { - self.resolve(vec![schema], enclosing_namespace, known_schemata)? - } - } - Schema::Enum(EnumSchema { name, .. }) - | Schema::Fixed(FixedSchema { name, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if self - .names_ref - .insert(fully_qualified_name.clone(), schema.clone()) - .is_some() - { - return Err( - Details::AmbiguousSchemaDefinition(fully_qualified_name).into() - ); - } - } - Schema::Record(RecordSchema { name, fields, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if self - .names_ref - .insert(fully_qualified_name.clone(), schema.clone()) - .is_some() - { - return Err( - Details::AmbiguousSchemaDefinition(fully_qualified_name).into() - ); - } else { - let record_namespace = fully_qualified_name.namespace; - for field in fields { - self.resolve( - vec![&field.schema], - &record_namespace, - known_schemata, - )? - } - } - } - Schema::Ref { name } => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - // first search for reference in current schemata, then look into external references. - if !self.names_ref.contains_key(&fully_qualified_name) { - let is_resolved_with_known_schemas = known_schemata - .as_ref() - .map(|names| names.contains_key(&fully_qualified_name)) - .unwrap_or(false); - if !is_resolved_with_known_schemas { - return Err( - Details::SchemaResolutionError(fully_qualified_name).into() - ); - } - } - } - _ => (), + fn denormalize(&mut self, schemata: &[Schema]) -> AvroResult<()> { + match self { + Schema::Ref { name } => { + let replacement_schema = schemata + .iter() + .find(|s| s.name().map(|n| *n == *name).unwrap_or(false)); + if let Some(schema) = replacement_schema { + let mut denorm = schema.clone(); + denorm.denormalize(schemata)?; + *self = denorm; + } else { + return Err(Details::SchemaResolutionError(name.clone()).into()); } } - Ok(()) + Schema::Record(record_schema) => { + for field in &mut record_schema.fields { + field.schema.denormalize(schemata)?; + } + } + Schema::Array(array_schema) => { + array_schema.items.denormalize(schemata)?; + } + Schema::Map(map_schema) => { + map_schema.types.denormalize(schemata)?; + } + Schema::Union(union_schema) => { + for schema in &mut union_schema.schemas { + schema.denormalize(schemata)?; + } + } + _ => (), } + Ok(()) } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct MapSchema { + pub types: Box, + pub attributes: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ArraySchema { + pub items: Box, + pub attributes: BTreeMap, +} - pub(crate) struct ResolvedOwnedSchema { - names: Names, - root_schema: Schema, +impl PartialEq for Schema { + /// Assess equality of two `Schema` based on [Parsing Canonical Form]. + /// + /// [Parsing Canonical Form]: + /// https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas + fn eq(&self, other: &Self) -> bool { + compare_schemata(self, other) } +} - impl TryFrom for ResolvedOwnedSchema { - type Error = Error; +impl SchemaKind { + pub fn is_primitive(self) -> bool { + matches!( + self, + SchemaKind::Null + | SchemaKind::Boolean + | SchemaKind::Int + | SchemaKind::Long + | SchemaKind::Double + | SchemaKind::Float + | SchemaKind::Bytes + | SchemaKind::String, + ) + } - fn try_from(schema: Schema) -> AvroResult { - let names = HashMap::new(); - let mut rs = ResolvedOwnedSchema { - names, - root_schema: schema, - }; - resolve_names(&rs.root_schema, &mut rs.names, &None)?; - Ok(rs) - } + pub fn is_named(self) -> bool { + matches!( + self, + SchemaKind::Record | SchemaKind::Enum | SchemaKind::Fixed | SchemaKind::Ref + ) } +} - impl ResolvedOwnedSchema { - pub(crate) fn get_root_schema(&self) -> &Schema { - &self.root_schema - } - pub(crate) fn get_names(&self) -> &Names { - &self.names +impl From<&Value> for SchemaKind { + fn from(value: &Value) -> Self { + use crate::types::Value; + match value { + Value::Null => Self::Null, + Value::Boolean(_) => Self::Boolean, + Value::Int(_) => Self::Int, + Value::Long(_) => Self::Long, + Value::Float(_) => Self::Float, + Value::Double(_) => Self::Double, + Value::Bytes(_) => Self::Bytes, + Value::String(_) => Self::String, + Value::Array(_) => Self::Array, + Value::Map(_) => Self::Map, + Value::Union(_, _) => Self::Union, + Value::Record(_) => Self::Record, + Value::Enum(_, _) => Self::Enum, + Value::Fixed(_, _) => Self::Fixed, + Value::Decimal { .. } => Self::Decimal, + Value::BigDecimal(_) => Self::BigDecimal, + Value::Uuid(_) => Self::Uuid, + Value::Date(_) => Self::Date, + Value::TimeMillis(_) => Self::TimeMillis, + Value::TimeMicros(_) => Self::TimeMicros, + Value::TimestampMillis(_) => Self::TimestampMillis, + Value::TimestampMicros(_) => Self::TimestampMicros, + Value::TimestampNanos(_) => Self::TimestampNanos, + Value::LocalTimestampMillis(_) => Self::LocalTimestampMillis, + Value::LocalTimestampMicros(_) => Self::LocalTimestampMicros, + Value::LocalTimestampNanos(_) => Self::LocalTimestampNanos, + Value::Duration { .. } => Self::Duration, } } +} - pub(crate) fn resolve_names( - schema: &Schema, - names: &mut Names, - enclosing_namespace: &Namespace, - ) -> AvroResult<()> { - match schema { - Schema::Array(schema) => resolve_names(&schema.items, names, enclosing_namespace), - Schema::Map(schema) => resolve_names(&schema.types, names, enclosing_namespace), - Schema::Union(UnionSchema { schemas, .. }) => { - for schema in schemas { - resolve_names(schema, names, enclosing_namespace)? +impl Serialize for Schema { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match *self { + Schema::Ref { ref name } => serializer.serialize_str(&name.fullname(None)), + Schema::Null => serializer.serialize_str("null"), + Schema::Boolean => serializer.serialize_str("boolean"), + Schema::Int => serializer.serialize_str("int"), + Schema::Long => serializer.serialize_str("long"), + Schema::Float => serializer.serialize_str("float"), + Schema::Double => serializer.serialize_str("double"), + Schema::Bytes => serializer.serialize_str("bytes"), + Schema::String => serializer.serialize_str("string"), + Schema::Array(ref inner) => { + let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; + map.serialize_entry("type", "array")?; + map.serialize_entry("items", &*inner.items.clone())?; + for attr in &inner.attributes { + map.serialize_entry(attr.0, attr.1)?; } - Ok(()) - } - Schema::Enum(EnumSchema { name, .. }) | Schema::Fixed(FixedSchema { name, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if names - .insert(fully_qualified_name.clone(), schema.clone()) - .is_some() - { - Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) - } else { - Ok(()) + map.end() + } + Schema::Map(ref inner) => { + let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; + map.serialize_entry("type", "map")?; + map.serialize_entry("values", &*inner.types.clone())?; + for attr in &inner.attributes { + map.serialize_entry(attr.0, attr.1)?; } + map.end() } - Schema::Record(RecordSchema { name, fields, .. }) => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - if names - .insert(fully_qualified_name.clone(), schema.clone()) - .is_some() - { - Err(Details::AmbiguousSchemaDefinition(fully_qualified_name).into()) - } else { - let record_namespace = fully_qualified_name.namespace; - for field in fields { - resolve_names(&field.schema, names, &record_namespace)? + Schema::Union(ref inner) => { + let variants = inner.variants(); + let mut seq = serializer.serialize_seq(Some(variants.len()))?; + for v in variants { + seq.serialize_element(v)?; + } + seq.end() + } + Schema::Record(RecordSchema { + ref name, + ref aliases, + ref doc, + ref fields, + ref attributes, + .. + }) => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "record")?; + if let Some(ref n) = name.namespace { + map.serialize_entry("namespace", n)?; + } + map.serialize_entry("name", &name.name)?; + if let Some(docstr) = doc { + map.serialize_entry("doc", docstr)?; + } + if let Some(aliases) = aliases { + map.serialize_entry("aliases", aliases)?; + } + map.serialize_entry("fields", fields)?; + for attr in attributes { + map.serialize_entry(attr.0, attr.1)?; + } + map.end() + } + Schema::Enum(EnumSchema { + ref name, + ref symbols, + ref aliases, + ref attributes, + .. + }) => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "enum")?; + if let Some(ref n) = name.namespace { + map.serialize_entry("namespace", n)?; + } + map.serialize_entry("name", &name.name)?; + map.serialize_entry("symbols", symbols)?; + + if let Some(aliases) = aliases { + map.serialize_entry("aliases", aliases)?; + } + for attr in attributes { + map.serialize_entry(attr.0, attr.1)?; + } + map.end() + } + Schema::Fixed(ref fixed_schema) => { + let mut map = serializer.serialize_map(None)?; + map = fixed_schema.serialize_to_map::(map)?; + map.end() + } + Schema::Decimal(DecimalSchema { + ref scale, + ref precision, + ref inner, + }) => { + let mut map = serializer.serialize_map(None)?; + match inner.as_ref() { + Schema::Fixed(fixed_schema) => { + map = fixed_schema.serialize_to_map::(map)?; + } + Schema::Bytes => { + map.serialize_entry("type", "bytes")?; + } + others => { + return Err(serde::ser::Error::custom(format!( + "DecimalSchema inner type must be Fixed or Bytes, got {:?}", + SchemaKind::from(others) + ))); } - Ok(()) } + map.serialize_entry("logicalType", "decimal")?; + map.serialize_entry("scale", scale)?; + map.serialize_entry("precision", precision)?; + map.end() + } + + Schema::BigDecimal => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "bytes")?; + map.serialize_entry("logicalType", "big-decimal")?; + map.end() + } + Schema::Uuid => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "string")?; + map.serialize_entry("logicalType", "uuid")?; + map.end() + } + Schema::Date => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "int")?; + map.serialize_entry("logicalType", "date")?; + map.end() + } + Schema::TimeMillis => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "int")?; + map.serialize_entry("logicalType", "time-millis")?; + map.end() + } + Schema::TimeMicros => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "time-micros")?; + map.end() + } + Schema::TimestampMillis => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "timestamp-millis")?; + map.end() + } + Schema::TimestampMicros => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "timestamp-micros")?; + map.end() + } + Schema::TimestampNanos => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "timestamp-nanos")?; + map.end() + } + Schema::LocalTimestampMillis => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "local-timestamp-millis")?; + map.end() + } + Schema::LocalTimestampMicros => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "local-timestamp-micros")?; + map.end() + } + Schema::LocalTimestampNanos => { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("type", "long")?; + map.serialize_entry("logicalType", "local-timestamp-nanos")?; + map.end() + } + Schema::Duration => { + let mut map = serializer.serialize_map(None)?; + + // the Avro doesn't indicate what the name of the underlying fixed type of a + // duration should be or typically is. + let inner = Schema::Fixed(FixedSchema { + name: Name::new("duration").unwrap(), + aliases: None, + doc: None, + size: 12, + default: None, + attributes: Default::default(), + }); + map.serialize_entry("type", &inner)?; + map.serialize_entry("logicalType", "duration")?; + map.end() } - Schema::Ref { name } => { - let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - names - .get(&fully_qualified_name) - .map(|_| ()) - .ok_or_else(|| Details::SchemaResolutionError(fully_qualified_name).into()) - } - _ => Ok(()), } } +} - pub(crate) fn resolve_names_with_schemata( - schemata: &Vec<&Schema>, - names: &mut Names, - enclosing_namespace: &Namespace, - ) -> AvroResult<()> { - for schema in schemata { - resolve_names(schema, names, enclosing_namespace)?; +impl Serialize for RecordField { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("name", &self.name)?; + map.serialize_entry("type", &self.schema)?; + + if let Some(ref default) = self.default { + map.serialize_entry("default", default)?; } - Ok(()) - } - /// Represents a `field` in a `record` Avro schema. - #[derive(bon::Builder, Clone, Debug, PartialEq)] - pub struct RecordField { - /// Name of the field. - pub name: String, - /// Documentation of the field. - #[builder(default)] - pub doc: Documentation, - /// Aliases of the field's name. They have no namespace. - pub aliases: Option>, - /// Default value of the field. - /// This value will be used when reading Avro datum if schema resolution - /// is enabled. - pub default: Option, - /// Schema of the field. - pub schema: Schema, - /// Order of the field. - /// - /// **NOTE** This currently has no effect. - #[builder(default = RecordFieldOrder::Ignore)] - pub order: RecordFieldOrder, - /// Position of the field in the list of `field` of its parent `Schema` - #[builder(default)] - pub position: usize, - /// A collection of all unknown fields in the record field. - #[builder(default = BTreeMap::new())] - pub custom_attributes: BTreeMap, - } + if let Some(ref aliases) = self.aliases { + map.serialize_entry("aliases", aliases)?; + } - /// Represents any valid order for a `field` in a `record` Avro schema. - #[derive(Clone, Debug, Eq, PartialEq, EnumString)] - #[strum(serialize_all = "kebab_case")] - pub enum RecordFieldOrder { - Ascending, - Descending, - Ignore, + for attr in &self.custom_attributes { + map.serialize_entry(attr.0, attr.1)?; + } + + map.end() } +} + +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio {}, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::de::tokio => crate::de::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::reader::tokio => crate::reader::sync, + crate::writer::tokio => crate::writer::sync, + crate::ser::tokio => crate::ser::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::util::tokio => crate::util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod schema { + + use crate::AvroResult; + use crate::error::{Details, Error}; + use crate::schema::{ + Alias, Aliases, ArraySchema, DecimalMetadata, DecimalSchema, Documentation, EnumSchema, + FixedSchema, MapSchema, Name, Names, Namespace, Precision, RecordField, RecordFieldOrder, + RecordSchema, ResolvedSchema, Scale, Schema, SchemaFingerprint, SchemaKind, UnionSchema, + }; + use crate::util::MapHelper; + use crate::{ + types::{Value, ValueKind}, + validator::{ + validate_enum_symbol_name, validate_namespace, validate_record_field_name, + validate_schema_name, + }, + }; + use digest::Digest; + use log::{debug, error, warn}; + use serde::{ + Deserialize, Serialize, Serializer, + ser::{SerializeMap, SerializeSeq}, + }; + #[synca::cfg(sync)] + use std::io::Read as AvroRead; + use std::{ + collections::{BTreeMap, HashMap, HashSet}, + fmt::Debug, + hash::Hash, + io::Read, + str::FromStr, + }; + #[synca::cfg(tokio)] + use tokio::io::AsyncRead as AvroRead; + #[cfg(feature = "tokio")] + use tokio::io::AsyncReadExt; - impl RecordField { + pub struct RecordFieldExt; + + impl RecordFieldExt { /// Parse a `serde_json::Value` into a `RecordField`. async fn parse( field: &serde_json::Map, position: usize, parser: &mut Parser, enclosing_record: &Name, - ) -> AvroResult { + ) -> AvroResult { let name = field.name().ok_or(Details::GetNameFieldFromRecord)?; validate_record_field_name(&name)?; @@ -752,7 +1453,7 @@ mod schema { aliases, order, position, - custom_attributes: RecordField::get_field_custom_attributes(field, &schema), + custom_attributes: RecordFieldExt::get_field_custom_attributes(field, &schema), schema, }) } @@ -842,149 +1543,9 @@ mod schema { } } - /// A description of an Enum schema. - #[derive(bon::Builder, Debug, Clone)] - pub struct RecordSchema { - /// The name of the schema - pub name: Name, - /// The aliases of the schema - #[builder(default)] - pub aliases: Aliases, - /// The documentation of the schema - #[builder(default)] - pub doc: Documentation, - /// The set of fields of the schema - pub fields: Vec, - /// The `lookup` table maps field names to their position in the `Vec` - /// of `fields`. - pub lookup: BTreeMap, - /// The custom attributes of the schema - #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, - } - - /// A description of an Enum schema. - #[derive(bon::Builder, Debug, Clone)] - pub struct EnumSchema { - /// The name of the schema - pub name: Name, - /// The aliases of the schema - #[builder(default)] - pub aliases: Aliases, - /// The documentation of the schema - #[builder(default)] - pub doc: Documentation, - /// The set of symbols of the schema - pub symbols: Vec, - /// An optional default symbol used for compatibility - pub default: Option, - /// The custom attributes of the schema - #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, - } - - /// A description of a Union schema. - #[derive(bon::Builder, Debug, Clone)] - pub struct FixedSchema { - /// The name of the schema - pub name: Name, - /// The aliases of the schema - #[builder(default)] - pub aliases: Aliases, - /// The documentation of the schema - #[builder(default)] - pub doc: Documentation, - /// The size of the fixed schema - pub size: usize, - /// An optional default symbol used for compatibility - pub default: Option, - /// The custom attributes of the schema - #[builder(default = BTreeMap::new())] - pub attributes: BTreeMap, - } - - impl FixedSchema { - fn serialize_to_map(&self, mut map: S::SerializeMap) -> Result - where - S: Serializer, - { - map.serialize_entry("type", "fixed")?; - if let Some(ref n) = self.name.namespace { - map.serialize_entry("namespace", n)?; - } - map.serialize_entry("name", &self.name.name)?; - if let Some(ref docstr) = self.doc { - map.serialize_entry("doc", docstr)?; - } - map.serialize_entry("size", &self.size)?; - - if let Some(ref aliases) = self.aliases { - map.serialize_entry("aliases", aliases)?; - } - - for attr in &self.attributes { - map.serialize_entry(attr.0, attr.1)?; - } - - Ok(map) - } - } - - /// A description of a Union schema. - /// - /// `scale` defaults to 0 and is an integer greater than or equal to 0 and `precision` is an - /// integer greater than 0. - #[derive(Debug, Clone)] - pub struct DecimalSchema { - /// The number of digits in the unscaled value - pub precision: DecimalMetadata, - /// The number of digits to the right of the decimal point - pub scale: DecimalMetadata, - /// The inner schema of the decimal (fixed or bytes) - pub inner: Box, - } - - /// A description of a Union schema - #[derive(Debug, Clone)] - pub struct UnionSchema { - /// The schemas that make up this union - pub(crate) schemas: Vec, - // Used to ensure uniqueness of schema inputs, and provide constant time finding of the - // schema index given a value. - // **NOTE** that this approach does not work for named types, and will have to be modified - // to support that. A simple solution is to also keep a mapping of the names used. - variant_index: BTreeMap, - } - - impl UnionSchema { - /// Creates a new UnionSchema from a vector of schemas. - pub fn new(schemas: Vec) -> AvroResult { - let mut vindex = BTreeMap::new(); - for (i, schema) in schemas.iter().enumerate() { - if let Schema::Union(_) = schema { - return Err(Details::GetNestedUnion.into()); - } - let kind = SchemaKind::from(schema); - if !kind.is_named() && vindex.insert(kind, i).is_some() { - return Err(Details::GetUnionDuplicate.into()); - } - } - Ok(UnionSchema { - schemas, - variant_index: vindex, - }) - } - - /// Returns a slice to all variants of this schema. - pub fn variants(&self) -> &[Schema] { - &self.schemas - } - - /// Returns true if the any of the variants of this `UnionSchema` is `Null`. - pub fn is_nullable(&self) -> bool { - self.schemas.iter().any(|x| matches!(x, Schema::Null)) - } + pub struct UnionSchemaExt; + impl UnionSchemaExt { /// Optionally returns a reference to the schema matched by this value, as well as its position /// within this union. /// @@ -1039,17 +1600,6 @@ mod schema { } } - // No need to compare variant_index, it is derivative of schemas. - impl PartialEq for UnionSchema { - fn eq(&self, other: &UnionSchema) -> bool { - self.schemas.eq(&other.schemas) - } - } - - type DecimalMetadata = usize; - pub(crate) type Precision = DecimalMetadata; - pub(crate) type Scale = DecimalMetadata; - fn parse_json_integer_for_decimal( value: &serde_json::Number, ) -> Result { @@ -1090,46 +1640,12 @@ mod schema { input_order: Vec, /// A map of name -> fully parsed Schema /// Used to avoid parsing the same schema twice - parsed_schemas: Names, - } - - impl Schema { - /// Converts `self` into its [Parsing Canonical Form]. - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - pub fn canonical_form(&self) -> String { - let json = serde_json::to_value(self) - .unwrap_or_else(|e| panic!("Cannot parse Schema from JSON: {e}")); - let mut defined_names = HashSet::new(); - parsing_canonical_form(&json, &mut defined_names) - } - - /// Returns the [Parsing Canonical Form] of `self` that is self contained (not dependent on - /// any definitions in `schemata`) - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - pub fn independent_canonical_form(&self, schemata: &[Schema]) -> Result { - let mut this = self.clone(); - this.denormalize(schemata)?; - Ok(this.canonical_form()) - } + parsed_schemas: Names, + } - /// Generate [fingerprint] of Schema's [Parsing Canonical Form]. - /// - /// [Parsing Canonical Form]: - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - /// [fingerprint]: - /// https://avro.apache.org/docs/current/specification/#schema-fingerprints - pub fn fingerprint(&self) -> SchemaFingerprint { - let mut d = D::new(); - d.update(self.canonical_form()); - SchemaFingerprint { - bytes: d.finalize().to_vec(), - } - } + pub struct SchemaExt; + impl SchemaExt { /// Create a `Schema` from a string representing a JSON Avro schema. pub async fn parse_str(input: &str) -> Result { let mut parser = Parser::default(); @@ -1225,9 +1741,11 @@ mod schema { } /// Create a `Schema` from a reader which implements [`Read`]. - pub async fn parse_reader(reader: &mut (impl Read + ?Sized)) -> AvroResult { + pub async fn parse_reader( + reader: &mut (impl AvroRead + ?Sized), + ) -> AvroResult { let mut buf = String::new(); - match reader.read_to_string(&mut buf) { + match reader.read_to_string(&mut buf).await { Ok(_) => Self::parse_str(&buf).await, Err(e) => Err(Details::ReadSchemaFromReader(e).into()), } @@ -1253,127 +1771,6 @@ mod schema { }; parser.parse(value, &None).await } - - /// Returns the custom attributes (metadata) if the schema supports them. - pub fn custom_attributes(&self) -> Option<&BTreeMap> { - match self { - Schema::Record(RecordSchema { attributes, .. }) - | Schema::Enum(EnumSchema { attributes, .. }) - | Schema::Fixed(FixedSchema { attributes, .. }) - | Schema::Array(ArraySchema { attributes, .. }) - | Schema::Map(MapSchema { attributes, .. }) => Some(attributes), - _ => None, - } - } - - /// Returns the name of the schema if it has one. - pub fn name(&self) -> Option<&Name> { - match self { - Schema::Ref { name, .. } - | Schema::Record(RecordSchema { name, .. }) - | Schema::Enum(EnumSchema { name, .. }) - | Schema::Fixed(FixedSchema { name, .. }) => Some(name), - _ => None, - } - } - - /// Returns the namespace of the schema if it has one. - pub fn namespace(&self) -> Namespace { - self.name().and_then(|n| n.namespace.clone()) - } - - /// Returns the aliases of the schema if it has ones. - pub fn aliases(&self) -> Option<&Vec> { - match self { - Schema::Record(RecordSchema { aliases, .. }) - | Schema::Enum(EnumSchema { aliases, .. }) - | Schema::Fixed(FixedSchema { aliases, .. }) => aliases.as_ref(), - _ => None, - } - } - - /// Returns the doc of the schema if it has one. - pub fn doc(&self) -> Option<&String> { - match self { - Schema::Record(RecordSchema { doc, .. }) - | Schema::Enum(EnumSchema { doc, .. }) - | Schema::Fixed(FixedSchema { doc, .. }) => doc.as_ref(), - _ => None, - } - } - - /// Returns a Schema::Map with the given types. - pub fn map(types: Schema) -> Self { - Schema::Map(MapSchema { - types: Box::new(types), - attributes: Default::default(), - }) - } - - /// Returns a Schema::Map with the given types and custom attributes. - pub fn map_with_attributes( - types: Schema, - attributes: BTreeMap, - ) -> Self { - Schema::Map(MapSchema { - types: Box::new(types), - attributes, - }) - } - - /// Returns a Schema::Array with the given items. - pub fn array(items: Schema) -> Self { - Schema::Array(ArraySchema { - items: Box::new(items), - attributes: Default::default(), - }) - } - - /// Returns a Schema::Array with the given items and custom attributes. - pub fn array_with_attributes( - items: Schema, - attributes: BTreeMap, - ) -> Self { - Schema::Array(ArraySchema { - items: Box::new(items), - attributes, - }) - } - - fn denormalize(&mut self, schemata: &[Schema]) -> AvroResult<()> { - match self { - Schema::Ref { name } => { - let replacement_schema = schemata - .iter() - .find(|s| s.name().map(|n| *n == *name).unwrap_or(false)); - if let Some(schema) = replacement_schema { - let mut denorm = schema.clone(); - denorm.denormalize(schemata)?; - *self = denorm; - } else { - return Err(Details::SchemaResolutionError(name.clone()).into()); - } - } - Schema::Record(record_schema) => { - for field in &mut record_schema.fields { - field.schema.denormalize(schemata)?; - } - } - Schema::Array(array_schema) => { - array_schema.items.denormalize(schemata)?; - } - Schema::Map(map_schema) => { - map_schema.types.denormalize(schemata)?; - } - Schema::Union(union_schema) => { - for schema in &mut union_schema.schemas { - schema.denormalize(schemata)?; - } - } - _ => (), - } - Ok(()) - } } impl Parser { @@ -2167,384 +2564,6 @@ mod schema { } } - impl Serialize for Schema { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match *self { - Schema::Ref { ref name } => serializer.serialize_str(&name.fullname(None)), - Schema::Null => serializer.serialize_str("null"), - Schema::Boolean => serializer.serialize_str("boolean"), - Schema::Int => serializer.serialize_str("int"), - Schema::Long => serializer.serialize_str("long"), - Schema::Float => serializer.serialize_str("float"), - Schema::Double => serializer.serialize_str("double"), - Schema::Bytes => serializer.serialize_str("bytes"), - Schema::String => serializer.serialize_str("string"), - Schema::Array(ref inner) => { - let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; - map.serialize_entry("type", "array")?; - map.serialize_entry("items", &*inner.items.clone())?; - for attr in &inner.attributes { - map.serialize_entry(attr.0, attr.1)?; - } - map.end() - } - Schema::Map(ref inner) => { - let mut map = serializer.serialize_map(Some(2 + inner.attributes.len()))?; - map.serialize_entry("type", "map")?; - map.serialize_entry("values", &*inner.types.clone())?; - for attr in &inner.attributes { - map.serialize_entry(attr.0, attr.1)?; - } - map.end() - } - Schema::Union(ref inner) => { - let variants = inner.variants(); - let mut seq = serializer.serialize_seq(Some(variants.len()))?; - for v in variants { - seq.serialize_element(v)?; - } - seq.end() - } - Schema::Record(RecordSchema { - ref name, - ref aliases, - ref doc, - ref fields, - ref attributes, - .. - }) => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "record")?; - if let Some(ref n) = name.namespace { - map.serialize_entry("namespace", n)?; - } - map.serialize_entry("name", &name.name)?; - if let Some(docstr) = doc { - map.serialize_entry("doc", docstr)?; - } - if let Some(aliases) = aliases { - map.serialize_entry("aliases", aliases)?; - } - map.serialize_entry("fields", fields)?; - for attr in attributes { - map.serialize_entry(attr.0, attr.1)?; - } - map.end() - } - Schema::Enum(EnumSchema { - ref name, - ref symbols, - ref aliases, - ref attributes, - .. - }) => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "enum")?; - if let Some(ref n) = name.namespace { - map.serialize_entry("namespace", n)?; - } - map.serialize_entry("name", &name.name)?; - map.serialize_entry("symbols", symbols)?; - - if let Some(aliases) = aliases { - map.serialize_entry("aliases", aliases)?; - } - for attr in attributes { - map.serialize_entry(attr.0, attr.1)?; - } - map.end() - } - Schema::Fixed(ref fixed_schema) => { - let mut map = serializer.serialize_map(None)?; - map = fixed_schema.serialize_to_map::(map)?; - map.end() - } - Schema::Decimal(DecimalSchema { - ref scale, - ref precision, - ref inner, - }) => { - let mut map = serializer.serialize_map(None)?; - match inner.as_ref() { - Schema::Fixed(fixed_schema) => { - map = fixed_schema.serialize_to_map::(map)?; - } - Schema::Bytes => { - map.serialize_entry("type", "bytes")?; - } - others => { - return Err(serde::ser::Error::custom(format!( - "DecimalSchema inner type must be Fixed or Bytes, got {:?}", - SchemaKind::from(others) - ))); - } - } - map.serialize_entry("logicalType", "decimal")?; - map.serialize_entry("scale", scale)?; - map.serialize_entry("precision", precision)?; - map.end() - } - - Schema::BigDecimal => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "bytes")?; - map.serialize_entry("logicalType", "big-decimal")?; - map.end() - } - Schema::Uuid => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "string")?; - map.serialize_entry("logicalType", "uuid")?; - map.end() - } - Schema::Date => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "int")?; - map.serialize_entry("logicalType", "date")?; - map.end() - } - Schema::TimeMillis => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "int")?; - map.serialize_entry("logicalType", "time-millis")?; - map.end() - } - Schema::TimeMicros => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "time-micros")?; - map.end() - } - Schema::TimestampMillis => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "timestamp-millis")?; - map.end() - } - Schema::TimestampMicros => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "timestamp-micros")?; - map.end() - } - Schema::TimestampNanos => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "timestamp-nanos")?; - map.end() - } - Schema::LocalTimestampMillis => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "local-timestamp-millis")?; - map.end() - } - Schema::LocalTimestampMicros => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "local-timestamp-micros")?; - map.end() - } - Schema::LocalTimestampNanos => { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("type", "long")?; - map.serialize_entry("logicalType", "local-timestamp-nanos")?; - map.end() - } - Schema::Duration => { - let mut map = serializer.serialize_map(None)?; - - // the Avro doesn't indicate what the name of the underlying fixed type of a - // duration should be or typically is. - let inner = Schema::Fixed(FixedSchema { - name: Name::new("duration").unwrap(), - aliases: None, - doc: None, - size: 12, - default: None, - attributes: Default::default(), - }); - map.serialize_entry("type", &inner)?; - map.serialize_entry("logicalType", "duration")?; - map.end() - } - } - } - } - - impl Serialize for RecordField { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map = serializer.serialize_map(None)?; - map.serialize_entry("name", &self.name)?; - map.serialize_entry("type", &self.schema)?; - - if let Some(ref default) = self.default { - map.serialize_entry("default", default)?; - } - - if let Some(ref aliases) = self.aliases { - map.serialize_entry("aliases", aliases)?; - } - - for attr in &self.custom_attributes { - map.serialize_entry(attr.0, attr.1)?; - } - - map.end() - } - } - - /// Parses a **valid** avro schema into the Parsing Canonical Form. - /// https://avro.apache.org/docs/current/specification/#parsing-canonical-form-for-schemas - fn parsing_canonical_form( - schema: &serde_json::Value, - defined_names: &mut HashSet, - ) -> String { - match schema { - serde_json::Value::Object(map) => pcf_map(map, defined_names), - serde_json::Value::String(s) => pcf_string(s), - serde_json::Value::Array(v) => pcf_array(v, defined_names), - json => panic!("got invalid JSON value for canonical form of schema: {json}"), - } - } - - fn pcf_map( - schema: &serde_json::Map, - defined_names: &mut HashSet, - ) -> String { - // Look for the namespace variant up front. - let ns = schema.get("namespace").and_then(|v| v.as_str()); - let typ = schema.get("type").and_then(|v| v.as_str()); - let raw_name = schema.get("name").and_then(|v| v.as_str()); - let name = if is_named_type(typ) { - Some(format!( - "{}{}", - ns.map_or("".to_string(), |n| { format!("{n}.") }), - raw_name.unwrap_or_default() - )) - } else { - None - }; - - //if this is already a defined type, early return - if let Some(ref n) = name { - if defined_names.contains(n) { - return pcf_string(n); - } else { - defined_names.insert(n.clone()); - } - } - - let mut fields = Vec::new(); - for (k, v) in schema { - // Reduce primitive types to their simple form. ([PRIMITIVE] rule) - if schema.len() == 1 && k == "type" { - // Invariant: function is only callable from a valid schema, so this is acceptable. - if let serde_json::Value::String(s) = v { - return pcf_string(s); - } - } - - // Strip out unused fields ([STRIP] rule) - if field_ordering_position(k).is_none() - || k == "default" - || k == "doc" - || k == "aliases" - || k == "logicalType" - { - continue; - } - - // Fully qualify the name, if it isn't already ([FULLNAMES] rule). - if k == "name" { - if let Some(ref n) = name { - fields.push(("name", format!("{}:{}", pcf_string(k), pcf_string(n)))); - continue; - } - } - - // Strip off quotes surrounding "size" type, if they exist ([INTEGERS] rule). - if k == "size" || k == "precision" || k == "scale" { - let i = match v.as_str() { - Some(s) => s.parse::().expect("Only valid schemas are accepted!"), - None => v.as_i64().unwrap(), - }; - fields.push((k, format!("{}:{}", pcf_string(k), i))); - continue; - } - - // For anything else, recursively process the result. - fields.push(( - k, - format!( - "{}:{}", - pcf_string(k), - parsing_canonical_form(v, defined_names) - ), - )); - } - - // Sort the fields by their canonical ordering ([ORDER] rule). - fields.sort_unstable_by_key(|(k, _)| field_ordering_position(k).unwrap()); - let inter = fields - .into_iter() - .map(|(_, v)| v) - .collect::>() - .join(","); - format!("{{{inter}}}") - } - - fn is_named_type(typ: Option<&str>) -> bool { - matches!( - typ, - Some("record") | Some("enum") | Some("fixed") | Some("ref") - ) - } - - fn pcf_array(arr: &[serde_json::Value], defined_names: &mut HashSet) -> String { - let inter = arr - .iter() - .map(|a| parsing_canonical_form(a, defined_names)) - .collect::>() - .join(","); - format!("[{inter}]") - } - - fn pcf_string(s: &str) -> String { - format!("\"{s}\"") - } - - const RESERVED_FIELDS: &[&str] = &[ - "name", - "type", - "fields", - "symbols", - "items", - "values", - "size", - "logicalType", - "order", - "doc", - "aliases", - "default", - "precision", - "scale", - ]; - - // Used to define the ordering and inclusion of fields. - fn field_ordering_position(field: &str) -> Option { - RESERVED_FIELDS - .iter() - .position(|&f| f == field) - .map(|pos| pos + 1) - } - /// Trait for types that serve as an Avro data model. Derive implementation available /// through `derive` feature. Do not implement directly! /// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait @@ -2747,7 +2766,7 @@ mod schema { use crate::{ Uuid, de::tokio::from_value, - error::tokio::Details, + error::Details, rabin::Rabin, reader::tokio::from_avro_datum, ser::tokio::to_value, @@ -2763,34 +2782,34 @@ mod schema { #[tokio::test] async fn test_invalid_schema() { - assert!(Schema::parse_str("invalid").await.is_err()); + assert!(SchemaExt::parse_str("invalid").await.is_err()); } #[tokio::test] async fn test_primitive_schema() -> TestResult { - assert_eq!(Schema::Null, Schema::parse_str("\"null\"").await?); - assert_eq!(Schema::Int, Schema::parse_str("\"int\"").await?); - assert_eq!(Schema::Double, Schema::parse_str("\"double\"").await?); + assert_eq!(Schema::Null, SchemaExt::parse_str("\"null\"").await?); + assert_eq!(Schema::Int, SchemaExt::parse_str("\"int\"").await?); + assert_eq!(Schema::Double, SchemaExt::parse_str("\"double\"").await?); Ok(()) } #[tokio::test] async fn test_array_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "array", "items": "string"}"#).await?; + let schema = SchemaExt::parse_str(r#"{"type": "array", "items": "string"}"#).await?; assert_eq!(Schema::array(Schema::String), schema); Ok(()) } #[tokio::test] async fn test_map_schema() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "map", "values": "double"}"#).await?; + let schema = SchemaExt::parse_str(r#"{"type": "map", "values": "double"}"#).await?; assert_eq!(Schema::map(Schema::Double), schema); Ok(()) } #[tokio::test] async fn test_union_schema() -> TestResult { - let schema = Schema::parse_str(r#"["null", "int"]"#).await?; + let schema = SchemaExt::parse_str(r#"["null", "int"]"#).await?; assert_eq!( Schema::Union(UnionSchema::new(vec![Schema::Null, Schema::Int])?), schema @@ -2800,13 +2819,14 @@ mod schema { #[tokio::test] async fn test_union_unsupported_schema() { - let schema = Schema::parse_str(r#"["null", ["null", "int"], "string"]"#).await; + let schema = SchemaExt::parse_str(r#"["null", ["null", "int"], "string"]"#).await; assert!(schema.is_err()); } #[tokio::test] async fn test_multi_union_schema() -> TestResult { - let schema = Schema::parse_str(r#"["null", "int", "float", "string", "bytes"]"#).await; + let schema = + SchemaExt::parse_str(r#"["null", "int", "float", "string", "bytes"]"#).await; assert!(schema.is_ok()); let schema = schema?; assert_eq!(SchemaKind::from(&schema), SchemaKind::Union); @@ -2897,7 +2917,7 @@ mod schema { ] }"#; - let schema_c = Schema::parse_list([schema_str_a, schema_str_b, schema_str_c]) + let schema_c = SchemaExt::parse_list([schema_str_a, schema_str_b, schema_str_c]) .await? .last() .unwrap() @@ -2949,7 +2969,7 @@ mod schema { let schema_str_c = r#"["A", "B"]"#; let (schema_c, schemata) = - Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]).await?; + SchemaExt::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]).await?; let schema_a_expected = Schema::Record(RecordSchema { name: Name::new("A")?, @@ -3024,7 +3044,8 @@ mod schema { let schema_str_c = r#"["A", "A"]"#; - match Schema::parse_str_with_list(schema_str_c, [schema_str_a1, schema_str_a2]).await { + match SchemaExt::parse_str_with_list(schema_str_c, [schema_str_a1, schema_str_a2]).await + { Ok(_) => unreachable!("Expected an error that the name is already defined"), Err(e) => assert_eq!( e.to_string(), @@ -3055,7 +3076,7 @@ mod schema { let schema_str_c = r#"["A", "A"]"#; - match Schema::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]).await { + match SchemaExt::parse_str_with_list(schema_str_c, [schema_str_a, schema_str_b]).await { Ok(_) => { unreachable!("Expected an error that schema_str_b is missing a name field") } @@ -3080,7 +3101,7 @@ mod schema { "fields": [ {"name": "field_one", "type": "A"} ] }"#; - let list = Schema::parse_list([schema_str_a, schema_str_b]).await?; + let list = SchemaExt::parse_list([schema_str_a, schema_str_b]).await?; let schema_a = list.first().unwrap().clone(); @@ -3120,7 +3141,7 @@ mod schema { ] }"#; - let schema_option_a = Schema::parse_list([schema_str_a, schema_str_option_a]) + let schema_option_a = SchemaExt::parse_list([schema_str_a, schema_str_option_a]) .await? .last() .unwrap() @@ -3156,7 +3177,7 @@ mod schema { #[tokio::test] async fn test_record_schema() -> TestResult { - let parsed = Schema::parse_str( + let parsed = SchemaExt::parse_str( r#" { "type": "record", @@ -3211,7 +3232,7 @@ mod schema { #[tokio::test] async fn test_avro_3302_record_schema_with_currently_parsing_schema() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", @@ -3341,7 +3362,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let schema_str = schema.canonical_form(); let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"gender","type":{"name":"office.Gender","type":"enum","symbols":["male","female"]}}]},{"name":"office.Manager","type":"record","fields":[{"name":"gender","type":"office.Gender"}]}]}]}"#; assert_eq!(schema_str, expected); @@ -3391,7 +3412,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let schema_str = schema.canonical_form(); let expected = r#"{"name":"office.User","type":"record","fields":[{"name":"details","type":[{"name":"office.Employee","type":"record","fields":[{"name":"id","type":{"name":"office.EmployeeId","type":"fixed","size":16}}]},{"name":"office.Manager","type":"record","fields":[{"name":"id","type":"office.EmployeeId"}]}]}]}"#; assert_eq!(schema_str, expected); @@ -3402,7 +3423,7 @@ mod schema { #[tokio::test] async fn test_avro_3302_record_schema_with_currently_parsing_schema_aliases() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", @@ -3473,7 +3494,7 @@ mod schema { #[tokio::test] async fn test_avro_3370_record_schema_with_currently_parsing_schema_named_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type" : "record", @@ -3540,7 +3561,7 @@ mod schema { #[tokio::test] async fn test_avro_3370_record_schema_with_currently_parsing_schema_named_enum() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type" : "record", @@ -3629,7 +3650,7 @@ mod schema { #[tokio::test] async fn test_avro_3370_record_schema_with_currently_parsing_schema_named_fixed() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type" : "record", @@ -3714,7 +3735,7 @@ mod schema { #[tokio::test] async fn test_enum_schema() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "hearts"]}"#, ).await?; @@ -3740,7 +3761,7 @@ mod schema { #[tokio::test] async fn test_enum_schema_duplicate() -> TestResult { // Duplicate "diamonds" - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": "enum", "name": "Suit", "symbols": ["diamonds", "spades", "clubs", "diamonds"]}"#, ).await; assert!(schema.is_err()); @@ -3751,7 +3772,7 @@ mod schema { #[tokio::test] async fn test_enum_schema_name() -> TestResult { // Invalid name "0000" does not match [A-Za-z_][A-Za-z0-9_]* - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": "enum", "name": "Enum", "symbols": ["0000", "variant"]}"#, ) .await; @@ -3763,7 +3784,7 @@ mod schema { #[tokio::test] async fn test_fixed_schema() -> TestResult { let schema = - Schema::parse_str(r#"{"type": "fixed", "name": "test", "size": 16}"#).await?; + SchemaExt::parse_str(r#"{"type": "fixed", "name": "test", "size": 16}"#).await?; let expected = Schema::Fixed(FixedSchema { name: Name::new("test")?, @@ -3781,7 +3802,7 @@ mod schema { #[tokio::test] async fn test_fixed_schema_with_documentation() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": "fixed", "name": "test", "size": 16, "doc": "FixedSchema documentation"}"#, ).await?; @@ -3801,7 +3822,7 @@ mod schema { #[tokio::test] async fn test_no_documentation() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": "enum", "name": "Coin", "symbols": ["heads", "tails"]}"#, ) .await?; @@ -3818,7 +3839,7 @@ mod schema { #[tokio::test] async fn test_documentation() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": "enum", "name": "Coin", "doc": "Some documentation", "symbols": ["heads", "tails"]}"#, ).await?; @@ -3869,7 +3890,7 @@ mod schema { } "#; - let schema = Schema::parse_str(raw_schema).await?; + let schema = SchemaExt::parse_str(raw_schema).await?; assert_eq!( "7eb3b28d73dfc99bdd9af1848298b40804a2f8ad5d2642be2ecc2ad34842b987", format!("{}", schema.fingerprint::()) @@ -3889,11 +3910,12 @@ mod schema { #[tokio::test] async fn test_logical_types() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "int", "logicalType": "date"}"#).await?; + let schema = SchemaExt::parse_str(r#"{"type": "int", "logicalType": "date"}"#).await?; assert_eq!(schema, Schema::Date); let schema = - Schema::parse_str(r#"{"type": "long", "logicalType": "timestamp-micros"}"#).await?; + SchemaExt::parse_str(r#"{"type": "long", "logicalType": "timestamp-micros"}"#) + .await?; assert_eq!(schema, Schema::TimestampMicros); Ok(()) @@ -3901,7 +3923,7 @@ mod schema { #[tokio::test] async fn test_nullable_logical_type() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{"type": ["null", {"type": "long", "logicalType": "timestamp-micros"}]}"#, ) .await?; @@ -3939,7 +3961,7 @@ mod schema { #[tokio::test] async fn test_avro_3374_preserve_namespace_for_primitive() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type" : "record", @@ -3978,7 +4000,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let expected = r#"{"name":"test.test","type":"record","fields":[{"name":"bar","type":{"name":"test.foo","type":"record","fields":[{"name":"id","type":"long"}]}},{"name":"baz","type":"test.foo"}]}"#; assert_eq!(schema.canonical_form(), expected); @@ -4001,7 +4023,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; if let Schema::Record(RecordSchema { name, .. }) = schema { assert_eq!(name.name, "name"); assert_eq!(name.namespace, Some("space".to_string())); @@ -4028,7 +4050,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; if let Schema::Record(RecordSchema { name, .. }) = schema { assert_eq!(name.namespace, Some("space1".to_string())); } else { @@ -4054,7 +4076,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; if let Schema::Record(RecordSchema { name, .. }) = schema { assert_eq!(name.namespace, Some("space2".to_string())); } else { @@ -4123,7 +4145,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_record_name"] { @@ -4164,7 +4186,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_record_name"] { @@ -4200,7 +4222,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_enum_name"] { @@ -4236,7 +4258,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_enum_name"] { @@ -4272,7 +4294,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_fixed_name"] { @@ -4308,7 +4330,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.inner_fixed_name"] { @@ -4350,7 +4372,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "inner_space.inner_record_name"] { @@ -4387,7 +4409,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "inner_space.inner_enum_name"] { @@ -4424,7 +4446,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "inner_space.inner_fixed_name"] { @@ -4478,7 +4500,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 3); for s in &[ @@ -4537,7 +4559,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 3); for s in &[ @@ -4597,7 +4619,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 3); for s in &[ @@ -4642,7 +4664,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.in_array_record"] { @@ -4683,7 +4705,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); assert_eq!(rs.get_names().len(), 2); for s in &["space.record_name", "space.in_map_record"] { @@ -4720,7 +4742,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); // confirm we have expected 2 full-names @@ -4731,7 +4753,9 @@ mod schema { // convert Schema back to JSON string let schema_str = serde_json::to_string(&schema).expect("test failed"); - let _schema = Schema::parse_str(&schema_str).await.expect("test failed"); + let _schema = SchemaExt::parse_str(&schema_str) + .await + .expect("test failed"); assert_eq!(schema, _schema); Ok(()) @@ -4764,7 +4788,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let rs = ResolvedSchema::try_from(&schema).expect("Schema didn't successfully parse"); // confirm we have expected 2 full-names @@ -4775,7 +4799,9 @@ mod schema { // convert Schema back to JSON string let schema_str = serde_json::to_string(&schema).expect("test failed"); - let _schema = Schema::parse_str(&schema_str).await.expect("test failed"); + let _schema = SchemaExt::parse_str(&schema_str) + .await + .expect("test failed"); assert_eq!(schema, _schema); Ok(()) @@ -4797,7 +4823,7 @@ mod schema { #[tokio::test] async fn avro_3512_alias_with_null_namespace_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", @@ -4823,7 +4849,7 @@ mod schema { #[tokio::test] async fn avro_3512_alias_with_null_namespace_enum() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "enum", @@ -4849,7 +4875,7 @@ mod schema { #[tokio::test] async fn avro_3512_alias_with_null_namespace_fixed() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "fixed", @@ -4873,7 +4899,7 @@ mod schema { #[tokio::test] async fn avro_3518_serialize_aliases_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", @@ -4900,14 +4926,14 @@ mod schema { r#"{"aliases":["space.b","x.y","c"],"fields":[{"aliases":["time1","ns.time2"],"default":123,"name":"time","type":"long"}],"name":"a","namespace":"space","type":"record"}"#, &serialized ); - assert_eq!(schema, Schema::parse_str(&serialized).await?); + assert_eq!(schema, SchemaExt::parse_str(&serialized).await?); Ok(()) } #[tokio::test] async fn avro_3518_serialize_aliases_enum() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "enum", @@ -4928,14 +4954,14 @@ mod schema { r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","symbols":["symbol1","symbol2"],"type":"enum"}"#, &serialized ); - assert_eq!(schema, Schema::parse_str(&serialized).await?); + assert_eq!(schema, SchemaExt::parse_str(&serialized).await?); Ok(()) } #[tokio::test] async fn avro_3518_serialize_aliases_fixed() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "fixed", @@ -4954,7 +4980,7 @@ mod schema { r#"{"aliases":["space.b","x.y","c"],"name":"a","namespace":"space","size":12,"type":"fixed"}"#, &serialized ); - assert_eq!(schema, Schema::parse_str(&serialized).await?); + assert_eq!(schema, SchemaExt::parse_str(&serialized).await?); Ok(()) } @@ -4980,7 +5006,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; if let Schema::Record(RecordSchema { name, fields, .. }) = schema { assert_eq!(name, Name::new("AccountEvent")?); @@ -5039,7 +5065,7 @@ mod schema { "#, ]; for schema_str in schemata_str.iter() { - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; assert_eq!(schema.custom_attributes(), Some(&Default::default())); } @@ -5092,7 +5118,7 @@ mod schema { ]; for schema_str in schemata_str.iter() { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( schema_str .to_owned() .replace("{{{}}}", CUSTOM_ATTRS_SUFFIX) @@ -5150,7 +5176,7 @@ mod schema { ); let schema = - Schema::parse_str(schema_str.replace("{{{}}}", CUSTOM_ATTRS_SUFFIX).as_str()) + SchemaExt::parse_str(schema_str.replace("{{{}}}", CUSTOM_ATTRS_SUFFIX).as_str()) .await?; match schema { @@ -5181,7 +5207,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str).await?; + let schema = SchemaExt::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5220,7 +5246,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str).await?; + let schema = SchemaExt::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5258,7 +5284,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str).await?; + let schema = SchemaExt::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5297,7 +5323,7 @@ mod schema { "#, ); - let schema = Schema::parse_str(&schema_str).await?; + let schema = SchemaExt::parse_str(&schema_str).await?; match schema { Schema::Record(RecordSchema { name, fields, .. }) => { @@ -5337,7 +5363,7 @@ mod schema { } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; if let Schema::Record(RecordSchema { fields, .. }) = schema { let num_field = &fields[0]; assert_eq!(num_field.name, "num"); @@ -5405,7 +5431,7 @@ mod schema { pub bar_use: Bar, } - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let foo = Foo { bar_init: Bar::Bar0, @@ -5505,7 +5531,7 @@ mod schema { ] }"#; - let writer_schema = Schema::parse_str(writer_schema).await?; + let writer_schema = SchemaExt::parse_str(writer_schema).await?; let foo = Foo { bar_init: Bar::Bar0, bar_use: Bar::Bar1, @@ -5517,7 +5543,7 @@ mod schema { ); let datum = to_avro_datum(&writer_schema, avro_value).await?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema).await?; + let reader_schema = SchemaExt::parse_str(reader_schema).await?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema)).await?; match deser_value { Value::Record(fields) => { @@ -5551,7 +5577,7 @@ mod schema { ] }); - let parse_result = Schema::parse(&schema).await; + let parse_result = SchemaExt::parse(&schema).await; assert!( parse_result.is_ok(), "parse result must be ok, got: {parse_result:?}" @@ -5582,7 +5608,7 @@ mod schema { } "#; - match Schema::parse_str(schema).await { + match SchemaExt::parse_str(schema).await { Err(err) => { assert_eq!( err.to_string(), @@ -5624,7 +5650,7 @@ mod schema { "#; let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"a","type":{"name":"my_enum","type":"enum","symbols":["a","b"]}},{"name":"b","type":{"name":"my_fixed","type":"fixed","size":10}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5666,7 +5692,7 @@ mod schema { "#; let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"my_ns.fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5700,7 +5726,7 @@ mod schema { "#; let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5732,7 +5758,7 @@ mod schema { "#; let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"f1.ns.enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"f2.ns.fixed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5781,7 +5807,7 @@ mod schema { "#; let expected = r#"{"name":"my_ns.my_schema","type":"record","fields":[{"name":"f1","type":{"name":"my_ns.inner_record1","type":"record","fields":[{"name":"f1_1","type":{"name":"my_ns.enum1","type":"enum","symbols":["a"]}}]}},{"name":"f2","type":{"name":"inner_ns.inner_record2","type":"record","fields":[{"name":"f2_1","type":{"name":"inner_ns.enum2","type":"enum","symbols":["a"]}}]}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5798,7 +5824,7 @@ mod schema { } ); - let parse_result = Schema::parse(&schema).await; + let parse_result = SchemaExt::parse(&schema).await; assert!( parse_result.is_ok(), "parse result must be ok, got: {parse_result:?}" @@ -5837,7 +5863,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -5872,7 +5898,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -5911,7 +5937,7 @@ mod schema { "#; let expected = r#"{"name":"my_schema","type":"record","fields":[{"name":"f1","type":{"name":"a","type":"record","fields":[{"name":"f1","type":{"name":"b","type":"record","fields":[]}}]}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5950,7 +5976,7 @@ mod schema { "#; let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -5981,7 +6007,7 @@ mod schema { "#; let expected = r#"{"name":"record1","type":"record","fields":[{"name":"f1","type":{"name":"enum1","type":"enum","symbols":["a"]}},{"name":"f2","type":{"name":"fxed1","type":"fixed","size":1}}]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -6096,7 +6122,7 @@ mod schema { }; // Serialize using the writer schema. - let writer_schema = Schema::parse(&writer_schema).await?; + let writer_schema = SchemaExt::parse(&writer_schema).await?; let avro_value = to_value(s)?; assert!( avro_value.validate(&writer_schema).await, @@ -6105,7 +6131,7 @@ mod schema { let datum = to_avro_datum(&writer_schema, avro_value).await?; // Now, attempt to deserialize using the reader schema. - let reader_schema = Schema::parse(&reader_schema).await?; + let reader_schema = SchemaExt::parse(&reader_schema).await?; let mut x = &datum[..]; // Deserialization should succeed and we should be able to resolve the schema. @@ -6131,7 +6157,7 @@ mod schema { "#; let expected = r#"{"name":"ns1.record1","type":"record","fields":[]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -6146,7 +6172,7 @@ mod schema { "#; let expected = r#"{"name":"ns1.foo.bar.enum1","type":"enum","symbols":["a"]}"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); assert_eq!(canonical_form, expected); @@ -6160,7 +6186,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -6180,7 +6206,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -6200,7 +6226,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -6220,7 +6246,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -6240,7 +6266,7 @@ mod schema { } "#; - match Schema::parse_str(schema_str) + match SchemaExt::parse_str(schema_str) .await .map_err(Error::into_details) { @@ -6275,7 +6301,7 @@ mod schema { r#""int""#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6318,7 +6344,7 @@ mod schema { .to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6355,7 +6381,7 @@ mod schema { r#"{"name":"ns.enum1","type":"enum","symbols":["a","b","c"]}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6392,7 +6418,7 @@ mod schema { r#"{"name":"ns.fixed1","type":"fixed","size":3}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6426,7 +6452,7 @@ mod schema { r#"{"type":"array","items":"int"}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6460,7 +6486,7 @@ mod schema { r#"{"type":"map","values":"string"}"#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6505,7 +6531,7 @@ mod schema { r#""ns.record2""#.to_string(), ) .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6528,7 +6554,7 @@ mod schema { } "#; let expected = Details::EnumDefaultWrongType(100.into()).to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6550,7 +6576,7 @@ mod schema { symbols: vec!["a".to_string(), "b".to_string(), "c".to_string()], } .to_string(); - let result = Schema::parse_str(schema_str).await; + let result = SchemaExt::parse_str(schema_str).await; assert!(result.is_err()); let err = result .map_err(|e| e.to_string()) @@ -6576,7 +6602,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let expected = vec![Alias::new("ns1.r1")?, Alias::new("ns2.r2")?]; match schema.aliases() { Some(aliases) => assert_eq!(aliases, &expected), @@ -6594,7 +6620,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match schema.aliases() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6610,7 +6636,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let expected = vec![Alias::new("ns1.en1")?, Alias::new("ns2.en2")?]; match schema.aliases() { Some(aliases) => assert_eq!(aliases, &expected), @@ -6625,7 +6651,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match schema.aliases() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6641,7 +6667,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let expected = vec![Alias::new("ns1.fx1")?, Alias::new("ns2.fx2")?]; match schema.aliases() { Some(aliases) => assert_eq!(aliases, &expected), @@ -6656,7 +6682,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match schema.aliases() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6686,7 +6712,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let expected = "Record Document"; match schema.doc() { Some(doc) => assert_eq!(doc, expected), @@ -6703,7 +6729,7 @@ mod schema { ] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match schema.doc() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6718,7 +6744,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let expected = "Enum Document"; match schema.doc() { Some(doc) => assert_eq!(doc, expected), @@ -6732,7 +6758,7 @@ mod schema { "symbols": ["a", "b", "c"] } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match schema.doc() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6747,7 +6773,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let expected = "Fixed Document"; match schema.doc() { Some(doc) => assert_eq!(doc, expected), @@ -6761,7 +6787,7 @@ mod schema { "size": 10 } "#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match schema.doc() { None => (), some => panic!("Expected None, got {some:?}"), @@ -6864,7 +6890,7 @@ mod schema { "precision": 9, "scale": 2 }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert!(matches!( parse_result, Schema::Decimal(DecimalSchema { @@ -6881,7 +6907,7 @@ mod schema { "name": "LongDecimal", "logicalType": "decimal" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; // assert!(matches!(parse_result, Schema::Long)); assert_eq!(parse_result, Schema::Long); @@ -6897,7 +6923,7 @@ mod schema { "name": "StringUUID", "logicalType": "uuid" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert_eq!(parse_result, Schema::Uuid); Ok(()) @@ -6914,7 +6940,7 @@ mod schema { #[cfg_attr(feature = "tokio", async_trait::async_trait)] impl AvroSchema for Comment { async fn get_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type" : "record", "name" : "Comment", @@ -6965,7 +6991,7 @@ mod schema { "size": 16, "logicalType": "uuid" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert_eq!(parse_result, Schema::Uuid); assert_not_logged( r#"Ignoring uuid logical type for a Fixed schema because its size (6) is not 16! Schema: Fixed(FixedSchema { name: Name { name: "FixedUUID", namespace: None }, aliases: None, doc: None, size: 6, attributes: {"logicalType": String("uuid")} })"#, @@ -6983,7 +7009,7 @@ mod schema { "size": 6, "logicalType": "uuid" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert_eq!( parse_result, @@ -7012,7 +7038,7 @@ mod schema { "name": "LongTimestampMillis", "logicalType": "timestamp-millis" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert_eq!(parse_result, Schema::TimestampMillis); // int timestamp-millis, represents as native complex type. @@ -7022,7 +7048,7 @@ mod schema { "name": "IntTimestampMillis", "logicalType": "timestamp-millis" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert_eq!(parse_result, Schema::Int); Ok(()) @@ -7037,7 +7063,7 @@ mod schema { "name": "BytesLog", "logicalType": "custom" }); - let parse_result = Schema::parse(&schema).await?; + let parse_result = SchemaExt::parse(&schema).await?; assert_eq!(parse_result, Schema::Bytes); assert_eq!(parse_result.custom_attributes(), None); @@ -7046,7 +7072,7 @@ mod schema { #[tokio::test] async fn test_avro_3899_parse_decimal_type() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "name": "InvalidDecimal", "type": "fixed", @@ -7073,7 +7099,7 @@ mod schema { _ => unreachable!("Expected Schema::Fixed, got {:?}", schema), } - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "name": "ValidDecimal", "type": "bytes", @@ -7128,7 +7154,7 @@ mod schema { r#"{"aliases":["LinkedLongs"],"custom-attribute":"value","fields":[{"field-id":1,"name":"value","type":"long"}],"name":"LongList","type":"record"}"#, &serialized ); - assert_eq!(expected, Schema::parse_str(&serialized).await?); + assert_eq!(expected, SchemaExt::parse_str(&serialized).await?); Ok(()) } @@ -7214,7 +7240,7 @@ mod schema { r#"{"field-id":"1","items":"long","type":"array"}"#, &serialized ); - let actual_schema = Schema::parse_str(&serialized).await?; + let actual_schema = SchemaExt::parse_str(&serialized).await?; assert_eq!(expected, actual_schema); assert_eq!( expected.custom_attributes(), @@ -7237,7 +7263,7 @@ mod schema { r#"{"field-id":"1","type":"map","values":"long"}"#, &serialized ); - let actual_schema = Schema::parse_str(&serialized).await?; + let actual_schema = SchemaExt::parse_str(&serialized).await?; assert_eq!(expected, actual_schema); assert_eq!( expected.custom_attributes(), @@ -7260,7 +7286,7 @@ mod schema { } ] }"#; - match Schema::parse_str(schema).await? { + match SchemaExt::parse_str(schema).await? { Schema::Record(record_schema) => { assert_eq!(record_schema.fields.len(), 1); let field = record_schema.fields.first().unwrap(); @@ -7292,7 +7318,7 @@ mod schema { ] }"#; - let _ = Schema::parse_str(schema).await?; + let _ = SchemaExt::parse_str(schema).await?; assert_logged( "Union schema with just one member! Consider dropping the union! \ @@ -7318,7 +7344,7 @@ mod schema { ] }"#; - let _ = Schema::parse_str(schema).await?; + let _ = SchemaExt::parse_str(schema).await?; assert_logged( "Union schemas should have at least two members! \ @@ -7331,7 +7357,7 @@ mod schema { #[tokio::test] async fn avro_3965_fixed_schema_with_default_bigger_than_size() -> TestResult { - match Schema::parse_str( + match SchemaExt::parse_str( r#"{ "type": "fixed", "name": "test", @@ -7366,7 +7392,7 @@ mod schema { ] }"#; - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let canonical_form = schema.canonical_form(); let fp_rabin = schema.fingerprint::(); assert_eq!( @@ -7400,7 +7426,7 @@ mod schema { ] }"#; - let schema = Schema::parse_str(invalid_schema_str).await; + let schema = SchemaExt::parse_str(invalid_schema_str).await; assert!(schema.is_err()); assert_eq!( schema.unwrap_err().to_string(), @@ -7429,7 +7455,7 @@ mod schema { } ] }"#; - let schema = Schema::parse_str(valid_schema).await; + let schema = SchemaExt::parse_str(valid_schema).await; assert!(schema.is_ok()); Ok(()) diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index 658c9762..0211333d 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -26,16 +26,13 @@ replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, crate::codec::tokio => crate::codec::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, crate::schema::tokio => crate::schema::sync, crate::reader::tokio => crate::reader::sync, crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, crate::writer::tokio => crate::writer::sync, #[tokio::test] => #[test] ); @@ -44,8 +41,8 @@ mod schema_compatibility { use crate::{ - error::tokio::CompatibilityError, - schema::tokio::{EnumSchema, FixedSchema, RecordSchema, Schema, SchemaKind}, + error::CompatibilityError, + schema::{EnumSchema, FixedSchema, RecordSchema, Schema, SchemaKind}, }; use std::{ collections::{HashSet, hash_map::DefaultHasher}, @@ -561,7 +558,8 @@ mod schema_compatibility { use crate::{ codec::tokio::Codec, reader::tokio::Reader, - types::tokio::{Record, Value}, + schema::tokio::SchemaExt, + types::{Record, Value}, writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; @@ -570,79 +568,79 @@ mod schema_compatibility { use rstest::*; async fn int_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"int"}"#) + SchemaExt::parse_str(r#"{"type":"array", "items":"int"}"#) .await .unwrap() } async fn long_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"long"}"#) + SchemaExt::parse_str(r#"{"type":"array", "items":"long"}"#) .await .unwrap() } async fn string_array_schema() -> Schema { - Schema::parse_str(r#"{"type":"array", "items":"string"}"#) + SchemaExt::parse_str(r#"{"type":"array", "items":"string"}"#) .await .unwrap() } async fn int_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"int"}"#) + SchemaExt::parse_str(r#"{"type":"map", "values":"int"}"#) .await .unwrap() } async fn long_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"long"}"#) + SchemaExt::parse_str(r#"{"type":"map", "values":"long"}"#) .await .unwrap() } async fn string_map_schema() -> Schema { - Schema::parse_str(r#"{"type":"map", "values":"string"}"#) + SchemaExt::parse_str(r#"{"type":"map", "values":"string"}"#) .await .unwrap() } async fn enum1_ab_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B"]}"#) + SchemaExt::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B"]}"#) .await .unwrap() } async fn enum1_abc_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}"#) + SchemaExt::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}"#) .await .unwrap() } async fn enum1_bc_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["B","C"]}"#) + SchemaExt::parse_str(r#"{"type":"enum", "name":"Enum1", "symbols":["B","C"]}"#) .await .unwrap() } async fn enum2_ab_schema() -> Schema { - Schema::parse_str(r#"{"type":"enum", "name":"Enum2", "symbols":["A","B"]}"#) + SchemaExt::parse_str(r#"{"type":"enum", "name":"Enum2", "symbols":["A","B"]}"#) .await .unwrap() } async fn empty_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[]}"#) + SchemaExt::parse_str(r#"{"type":"record", "name":"Record1", "fields":[]}"#) .await .unwrap() } async fn empty_record2_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record2", "fields": []}"#) + SchemaExt::parse_str(r#"{"type":"record", "name":"Record2", "fields": []}"#) .await .unwrap() } async fn a_int_record1_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}]}"#, ) .await @@ -650,7 +648,7 @@ mod schema_compatibility { } async fn a_long_record1_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"long"}]}"#, ) .await @@ -658,35 +656,35 @@ mod schema_compatibility { } async fn a_int_b_int_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}"#).await.unwrap() } async fn a_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}"#).await.unwrap() } async fn a_int_b_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}"#).await.unwrap() } async fn a_dint_b_dint_record1_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}"#).await.unwrap() } async fn nested_record() -> Schema { - Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}}]}"#).await.unwrap() } async fn nested_optional_record() -> Schema { - Schema::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":["null",{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}],"default":null}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record","name":"parent","fields":[{"name":"attribute","type":["null",{"type":"record","name":"child","fields":[{"name":"id","type":"string"}]}],"default":null}]}"#).await.unwrap() } async fn int_list_record_schema() -> Schema { - Schema::parse_str(r#"{"type":"record", "name":"List", "fields": [{"name": "head", "type": "int"},{"name": "tail", "type": "array", "items": "int"}]}"#).await.unwrap() + SchemaExt::parse_str(r#"{"type":"record", "name":"List", "fields": [{"name": "head", "type": "int"},{"name": "tail", "type": "array", "items": "int"}]}"#).await.unwrap() } async fn long_list_record_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" { "type":"record", "name":"List", "fields": [ @@ -705,7 +703,7 @@ mod schema_compatibility { .map(|s| s.canonical_form()) .collect::>() .join(","); - Schema::parse_str(&format!("[{schema_string}]")) + SchemaExt::parse_str(&format!("[{schema_string}]")) .await .unwrap() } @@ -892,8 +890,8 @@ mod schema_compatibility { #[case] writer_schema_str: &str, #[case] reader_schema_str: &str, ) { - let writer_schema = Schema::parse_str(writer_schema_str).await.unwrap(); - let reader_schema = Schema::parse_str(reader_schema_str).await.unwrap(); + let writer_schema = SchemaExt::parse_str(writer_schema_str).await.unwrap(); + let reader_schema = SchemaExt::parse_str(reader_schema_str).await.unwrap(); assert!(SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).is_ok()); } @@ -1074,8 +1072,8 @@ mod schema_compatibility { #[case] reader_schema_str: &str, #[case] expected_error: CompatibilityError, ) { - let writer_schema = Schema::parse_str(writer_schema_str).await.unwrap(); - let reader_schema = Schema::parse_str(reader_schema_str).await.unwrap(); + let writer_schema = SchemaExt::parse_str(writer_schema_str).await.unwrap(); + let reader_schema = SchemaExt::parse_str(reader_schema_str).await.unwrap(); assert_eq!( expected_error, @@ -1187,7 +1185,7 @@ mod schema_compatibility { } async fn writer_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, @@ -1201,7 +1199,7 @@ mod schema_compatibility { #[tokio::test] async fn test_missing_field() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"} @@ -1220,7 +1218,7 @@ mod schema_compatibility { #[tokio::test] async fn test_missing_second_field() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield2", "type":"string"} @@ -1239,7 +1237,7 @@ mod schema_compatibility { #[tokio::test] async fn test_all_fields() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, @@ -1256,7 +1254,7 @@ mod schema_compatibility { #[tokio::test] async fn test_new_field_with_default() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, @@ -1276,7 +1274,7 @@ mod schema_compatibility { #[tokio::test] async fn test_new_field() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"Record", "fields":[ {"name":"oldfield1", "type":"int"}, @@ -1347,7 +1345,7 @@ mod schema_compatibility { #[tokio::test] async fn test_incompatible_record_field() -> TestResult { - let string_schema = Schema::parse_str( + let string_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [ {"name":"field1", "type":"string"} @@ -1356,7 +1354,7 @@ mod schema_compatibility { ) .await?; - let int_schema = Schema::parse_str( + let int_schema = SchemaExt::parse_str( r#" {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [ {"name":"field1", "type":"int"} @@ -1385,15 +1383,16 @@ mod schema_compatibility { #[tokio::test] async fn test_enum_symbols() -> TestResult { - let enum_schema1 = Schema::parse_str( + let enum_schema1 = SchemaExt::parse_str( r#" {"type":"enum", "name":"MyEnum", "symbols":["A","B"]} "#, ) .await?; - let enum_schema2 = - Schema::parse_str(r#"{"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}"#) - .await?; + let enum_schema2 = SchemaExt::parse_str( + r#"{"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}"#, + ) + .await?; assert_eq!( CompatibilityError::MissingSymbols, SchemaCompatibility::can_read(&enum_schema2, &enum_schema1).unwrap_err() @@ -1404,7 +1403,7 @@ mod schema_compatibility { } async fn point_2d_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" {"type":"record", "name":"Point2D", "fields":[ {"name":"x", "type":"double"}, @@ -1417,7 +1416,7 @@ mod schema_compatibility { } async fn point_2d_fullname_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" {"type":"record", "name":"Point", "namespace":"written", "fields":[ {"name":"x", "type":"double"}, @@ -1430,7 +1429,7 @@ mod schema_compatibility { } async fn point_3d_no_default_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" {"type":"record", "name":"Point", "fields":[ {"name":"x", "type":"double"}, @@ -1444,7 +1443,7 @@ mod schema_compatibility { } async fn point_3d_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" {"type":"record", "name":"Point3D", "fields":[ {"name":"x", "type":"double"}, @@ -1458,7 +1457,7 @@ mod schema_compatibility { } async fn point_3d_match_name_schema() -> Schema { - Schema::parse_str( + SchemaExt::parse_str( r#" {"type":"record", "name":"Point", "fields":[ {"name":"x", "type":"double"}, @@ -1592,8 +1591,8 @@ mod schema_compatibility { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema).await?; - let reader_schema = Schema::parse_str(reader_raw_schema).await?; + let writer_schema = SchemaExt::parse_str(writer_raw_schema).await?; + let reader_schema = SchemaExt::parse_str(reader_raw_schema).await?; let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); @@ -1656,8 +1655,8 @@ mod schema_compatibility { ] } "#; - let writer_schema = Schema::parse_str(writer_raw_schema).await?; - let reader_schema = Schema::parse_str(reader_raw_schema).await?; + let writer_schema = SchemaExt::parse_str(writer_raw_schema).await?; + let reader_schema = SchemaExt::parse_str(reader_raw_schema).await?; let mut writer = Writer::with_codec(&writer_schema, Vec::new(), Codec::Null); let mut record = Record::new(writer.schema()).unwrap(); record.put("a", 27i64); @@ -1682,7 +1681,7 @@ mod schema_compatibility { #[tokio::test] async fn avro_3894_take_aliases_into_account_when_serializing_for_schema_compatibility() -> TestResult { - let schema_v1 = Schema::parse_str( + let schema_v1 = SchemaExt::parse_str( r#" { "type": "record", @@ -1696,7 +1695,7 @@ mod schema_compatibility { ) .await?; - let schema_v2 = Schema::parse_str( + let schema_v2 = SchemaExt::parse_str( r#" { "type": "record", @@ -1717,7 +1716,7 @@ mod schema_compatibility { #[tokio::test] async fn avro_3917_take_aliases_into_account_for_schema_compatibility() -> TestResult { - let schema_v1 = Schema::parse_str( + let schema_v1 = SchemaExt::parse_str( r#" { "type": "record", @@ -1731,7 +1730,7 @@ mod schema_compatibility { ) .await?; - let schema_v2 = Schema::parse_str( + let schema_v2 = SchemaExt::parse_str( r#" { "type": "record", @@ -1759,7 +1758,7 @@ mod schema_compatibility { let schemas = [ // Record schemas ( - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "record", "name": "Statistics", @@ -1772,7 +1771,7 @@ mod schema_compatibility { }"#, ) .await?, - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "record", "name": "Statistics", @@ -1789,7 +1788,7 @@ mod schema_compatibility { ), // Enum schemas ( - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "enum", "name": "Suit", @@ -1797,7 +1796,7 @@ mod schema_compatibility { }"#, ) .await?, - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "enum", "name": "Suit", @@ -1809,7 +1808,7 @@ mod schema_compatibility { ), // Fixed schemas ( - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "fixed", "name": "EmployeeId", @@ -1817,7 +1816,7 @@ mod schema_compatibility { }"#, ) .await?, - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "fixed", "name": "EmployeeId", @@ -1840,7 +1839,7 @@ mod schema_compatibility { async fn test_can_read_compatibility_errors() -> TestResult { let schemas = [ ( - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "record", "name": "StatisticsMap", @@ -1850,7 +1849,7 @@ mod schema_compatibility { ] }"#, ).await?, - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "record", "name": "StatisticsMap", @@ -1863,7 +1862,7 @@ mod schema_compatibility { "Incompatible schemata! Field 'success' in reader schema does not match the type in the writer schema", ), ( - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "record", "name": "StatisticsArray", @@ -1872,7 +1871,7 @@ mod schema_compatibility { ] }"#, ).await?, - Schema::parse_str( + SchemaExt::parse_str( r#"{ "type": "record", "name": "StatisticsArray", @@ -1927,7 +1926,7 @@ mod schema_compatibility { "#, ]; - let schemas = Schema::parse_list(schema_strs).await.unwrap(); + let schemas = SchemaExt::parse_list(schema_strs).await.unwrap(); SchemaCompatibility::can_read(&schemas[1], &schemas[1])?; Ok(()) diff --git a/avro/src/schema_equality.rs b/avro/src/schema_equality.rs index 803733da..efa7bfe9 100644 --- a/avro/src/schema_equality.rs +++ b/avro/src/schema_equality.rs @@ -15,274 +15,273 @@ // specific language governing permissions and limitations // under the License. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, - crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, - #[tokio::test] => #[test] - ); - } -)] -mod schema_equality { - use crate::schema::tokio::{ - ArraySchema, DecimalSchema, EnumSchema, FixedSchema, MapSchema, RecordField, RecordSchema, - Schema, UnionSchema, - }; - use log::{debug, error}; - use std::{fmt::Debug, sync::OnceLock}; - - /// A trait that compares two schemata for equality. - /// To register a custom one use [set_schemata_equality_comparator]. - pub trait SchemataEq: Debug + Send + Sync { - /// Compares two schemata for equality. - fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool; - } +// #[synca::synca( +// #[cfg(feature = "tokio")] +// pub mod tokio {}, +// #[cfg(feature = "sync")] +// pub mod sync { +// sync!(); +// replace!( +// crate::bigdecimal::tokio => crate::bigdecimal::sync, +// crate::decode::tokio => crate::decode::sync, +// crate::encode::tokio => crate::encode::sync, +// crate::error::tokio => crate::error::sync, +// crate::schema::tokio => crate::schema::sync, +// crate::util::tokio => crate::util::sync, +// crate::types::tokio => crate::types::sync, +// crate::schema_equality::tokio => crate::schema_equality::sync, +// crate::util::tokio => crate::util::sync, +// #[tokio::test] => #[test] +// ); +// } +// )] +// mod schema_equality { +use crate::schema::{ + ArraySchema, DecimalSchema, EnumSchema, FixedSchema, MapSchema, RecordField, RecordSchema, + Schema, UnionSchema, +}; +use log::{debug, error}; +use std::{fmt::Debug, sync::OnceLock}; + +/// A trait that compares two schemata for equality. +/// To register a custom one use [set_schemata_equality_comparator]. +pub trait SchemataEq: Debug + Send + Sync { + /// Compares two schemata for equality. + fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool; +} - /// Compares two schemas according to the Avro specification by using - /// their canonical forms. - /// See - #[derive(Debug)] - pub struct SpecificationEq; - impl SchemataEq for SpecificationEq { - fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { - schema_one.canonical_form() == schema_two.canonical_form() - } +/// Compares two schemas according to the Avro specification by using +/// their canonical forms. +/// See +#[derive(Debug)] +pub struct SpecificationEq; +impl SchemataEq for SpecificationEq { + fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { + schema_one.canonical_form() == schema_two.canonical_form() } +} - /// Compares two schemas for equality field by field, using only the fields that - /// are used to construct their canonical forms. - /// See - #[derive(Debug)] - pub struct StructFieldEq { - /// Whether to include custom attributes in the comparison. - /// The custom attributes are not used to construct the canonical form of the schema! - pub include_attributes: bool, - } +/// Compares two schemas for equality field by field, using only the fields that +/// are used to construct their canonical forms. +/// See +#[derive(Debug)] +pub struct StructFieldEq { + /// Whether to include custom attributes in the comparison. + /// The custom attributes are not used to construct the canonical form of the schema! + pub include_attributes: bool, +} - impl SchemataEq for StructFieldEq { - fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { - macro_rules! compare_primitive { - ($primitive:ident) => { - if let Schema::$primitive = schema_one { - if let Schema::$primitive = schema_two { - return true; - } - return false; +impl SchemataEq for StructFieldEq { + fn compare(&self, schema_one: &Schema, schema_two: &Schema) -> bool { + macro_rules! compare_primitive { + ($primitive:ident) => { + if let Schema::$primitive = schema_one { + if let Schema::$primitive = schema_two { + return true; } - }; - } + return false; + } + }; + } - if schema_one.name() != schema_two.name() { - return false; - } + if schema_one.name() != schema_two.name() { + return false; + } - compare_primitive!(Null); - compare_primitive!(Boolean); - compare_primitive!(Int); - compare_primitive!(Int); - compare_primitive!(Long); - compare_primitive!(Float); - compare_primitive!(Double); - compare_primitive!(Bytes); - compare_primitive!(String); - compare_primitive!(Uuid); - compare_primitive!(BigDecimal); - compare_primitive!(Date); - compare_primitive!(Duration); - compare_primitive!(TimeMicros); - compare_primitive!(TimeMillis); - compare_primitive!(TimestampMicros); - compare_primitive!(TimestampMillis); - compare_primitive!(TimestampNanos); - compare_primitive!(LocalTimestampMicros); - compare_primitive!(LocalTimestampMillis); - compare_primitive!(LocalTimestampNanos); - - if self.include_attributes - && schema_one.custom_attributes() != schema_two.custom_attributes() - { - return false; - } + compare_primitive!(Null); + compare_primitive!(Boolean); + compare_primitive!(Int); + compare_primitive!(Int); + compare_primitive!(Long); + compare_primitive!(Float); + compare_primitive!(Double); + compare_primitive!(Bytes); + compare_primitive!(String); + compare_primitive!(Uuid); + compare_primitive!(BigDecimal); + compare_primitive!(Date); + compare_primitive!(Duration); + compare_primitive!(TimeMicros); + compare_primitive!(TimeMillis); + compare_primitive!(TimestampMicros); + compare_primitive!(TimestampMillis); + compare_primitive!(TimestampNanos); + compare_primitive!(LocalTimestampMicros); + compare_primitive!(LocalTimestampMillis); + compare_primitive!(LocalTimestampNanos); + + if self.include_attributes + && schema_one.custom_attributes() != schema_two.custom_attributes() + { + return false; + } + if let Schema::Record(RecordSchema { + fields: fields_one, .. + }) = schema_one + { if let Schema::Record(RecordSchema { - fields: fields_one, .. - }) = schema_one + fields: fields_two, .. + }) = schema_two { - if let Schema::Record(RecordSchema { - fields: fields_two, .. - }) = schema_two - { - return self.compare_fields(fields_one, fields_two); - } - return false; + return self.compare_fields(fields_one, fields_two); } + return false; + } + if let Schema::Enum(EnumSchema { + symbols: symbols_one, + .. + }) = schema_one + { if let Schema::Enum(EnumSchema { - symbols: symbols_one, + symbols: symbols_two, .. - }) = schema_one + }) = schema_two { - if let Schema::Enum(EnumSchema { - symbols: symbols_two, - .. - }) = schema_two - { - return symbols_one == symbols_two; - } - return false; + return symbols_one == symbols_two; } + return false; + } - if let Schema::Fixed(FixedSchema { size: size_one, .. }) = schema_one { - if let Schema::Fixed(FixedSchema { size: size_two, .. }) = schema_two { - return size_one == size_two; - } - return false; + if let Schema::Fixed(FixedSchema { size: size_one, .. }) = schema_one { + if let Schema::Fixed(FixedSchema { size: size_two, .. }) = schema_two { + return size_one == size_two; } + return false; + } + if let Schema::Union(UnionSchema { + schemas: schemas_one, + .. + }) = schema_one + { if let Schema::Union(UnionSchema { - schemas: schemas_one, + schemas: schemas_two, .. - }) = schema_one + }) = schema_two { - if let Schema::Union(UnionSchema { - schemas: schemas_two, - .. - }) = schema_two - { - return schemas_one.len() == schemas_two.len() - && schemas_one - .iter() - .zip(schemas_two.iter()) - .all(|(s1, s2)| self.compare(s1, s2)); - } - return false; + return schemas_one.len() == schemas_two.len() + && schemas_one + .iter() + .zip(schemas_two.iter()) + .all(|(s1, s2)| self.compare(s1, s2)); } + return false; + } + if let Schema::Decimal(DecimalSchema { + precision: precision_one, + scale: scale_one, + .. + }) = schema_one + { if let Schema::Decimal(DecimalSchema { - precision: precision_one, - scale: scale_one, + precision: precision_two, + scale: scale_two, .. - }) = schema_one + }) = schema_two { - if let Schema::Decimal(DecimalSchema { - precision: precision_two, - scale: scale_two, - .. - }) = schema_two - { - return precision_one == precision_two && scale_one == scale_two; - } - return false; + return precision_one == precision_two && scale_one == scale_two; } + return false; + } + if let Schema::Array(ArraySchema { + items: items_one, .. + }) = schema_one + { if let Schema::Array(ArraySchema { - items: items_one, .. - }) = schema_one + items: items_two, .. + }) = schema_two { - if let Schema::Array(ArraySchema { - items: items_two, .. - }) = schema_two - { - return items_one == items_two; - } - return false; + return items_one == items_two; } + return false; + } + if let Schema::Map(MapSchema { + types: types_one, .. + }) = schema_one + { if let Schema::Map(MapSchema { - types: types_one, .. - }) = schema_one + types: types_two, .. + }) = schema_two { - if let Schema::Map(MapSchema { - types: types_two, .. - }) = schema_two - { - return self.compare(types_one, types_two); - } - return false; + return self.compare(types_one, types_two); } + return false; + } - if let Schema::Ref { name: name_one } = schema_one { - if let Schema::Ref { name: name_two } = schema_two { - return name_one == name_two; - } - return false; + if let Schema::Ref { name: name_one } = schema_one { + if let Schema::Ref { name: name_two } = schema_two { + return name_one == name_two; } + return false; + } - error!( - "This is a bug in schema_equality.rs! The following schemata types are not checked! \ + error!( + "This is a bug in schema_equality.rs! The following schemata types are not checked! \ Please report it to the Avro library maintainers! \ \n{schema_one:?}\n\n{schema_two:?}" - ); - false - } + ); + false } +} - impl StructFieldEq { - fn compare_fields(&self, fields_one: &[RecordField], fields_two: &[RecordField]) -> bool { - fields_one.len() == fields_two.len() - && fields_one - .iter() - .zip(fields_two.iter()) - .all(|(f1, f2)| self.compare(&f1.schema, &f2.schema)) - } +impl StructFieldEq { + fn compare_fields(&self, fields_one: &[RecordField], fields_two: &[RecordField]) -> bool { + fields_one.len() == fields_two.len() + && fields_one + .iter() + .zip(fields_two.iter()) + .all(|(f1, f2)| self.compare(&f1.schema, &f2.schema)) } +} - static SCHEMATA_COMPARATOR_ONCE: OnceLock> = OnceLock::new(); - - /// Sets a custom schemata equality comparator. - /// - /// Returns a unit if the registration was successful or the already - /// registered comparator if the registration failed. - /// - /// **Note**: This function must be called before parsing any schema because this will - /// register the default comparator and the registration is one time only! - pub fn set_schemata_equality_comparator( - comparator: Box, - ) -> Result<(), Box> { - debug!("Setting a custom schemata equality comparator: {comparator:?}."); - SCHEMATA_COMPARATOR_ONCE.set(comparator) - } +static SCHEMATA_COMPARATOR_ONCE: OnceLock> = OnceLock::new(); + +/// Sets a custom schemata equality comparator. +/// +/// Returns a unit if the registration was successful or the already +/// registered comparator if the registration failed. +/// +/// **Note**: This function must be called before parsing any schema because this will +/// register the default comparator and the registration is one time only! +pub fn set_schemata_equality_comparator( + comparator: Box, +) -> Result<(), Box> { + debug!("Setting a custom schemata equality comparator: {comparator:?}."); + SCHEMATA_COMPARATOR_ONCE.set(comparator) +} - pub(crate) fn compare_schemata(schema_one: &Schema, schema_two: &Schema) -> bool { - SCHEMATA_COMPARATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default schemata equality comparator: SpecificationEq.",); - Box::new(StructFieldEq { - include_attributes: false, - }) +pub(crate) fn compare_schemata(schema_one: &Schema, schema_two: &Schema) -> bool { + SCHEMATA_COMPARATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default schemata equality comparator: SpecificationEq.",); + Box::new(StructFieldEq { + include_attributes: false, }) - .compare(schema_one, schema_two) - } + }) + .compare(schema_one, schema_two) +} - #[cfg(test)] - #[allow(non_snake_case)] - mod tests { - use super::*; - use crate::schema::tokio::{Name, RecordFieldOrder}; - use apache_avro_test_helper::TestResult; - use serde_json::Value; - use std::collections::BTreeMap; - - const SPECIFICATION_EQ: SpecificationEq = SpecificationEq; - const STRUCT_FIELD_EQ: StructFieldEq = StructFieldEq { - include_attributes: false, - }; +#[cfg(test)] +#[allow(non_snake_case)] +mod tests { + use super::*; + use crate::schema::Name; + use crate::schema::RecordFieldOrder; + use apache_avro_test_helper::TestResult; + use serde_json::Value; + use std::collections::BTreeMap; + + const SPECIFICATION_EQ: SpecificationEq = SpecificationEq; + const STRUCT_FIELD_EQ: StructFieldEq = StructFieldEq { + include_attributes: false, + }; - macro_rules! test_primitives { + macro_rules! test_primitives { ($primitive:ident) => { paste::item! { #[test] @@ -295,314 +294,314 @@ mod schema_equality { }; } - test_primitives!(Null); - test_primitives!(Boolean); - test_primitives!(Int); - test_primitives!(Long); - test_primitives!(Float); - test_primitives!(Double); - test_primitives!(Bytes); - test_primitives!(String); - test_primitives!(Uuid); - test_primitives!(BigDecimal); - test_primitives!(Date); - test_primitives!(Duration); - test_primitives!(TimeMicros); - test_primitives!(TimeMillis); - test_primitives!(TimestampMicros); - test_primitives!(TimestampMillis); - test_primitives!(TimestampNanos); - test_primitives!(LocalTimestampMicros); - test_primitives!(LocalTimestampMillis); - test_primitives!(LocalTimestampNanos); - - #[test] - fn test_avro_3939_compare_named_schemata_with_different_names() { - let schema_one = Schema::Ref { - name: Name::from("name1"), - }; - - let schema_two = Schema::Ref { - name: Name::from("name2"), - }; - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - assert!(!specification_eq_res); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!(!struct_field_eq_res); + test_primitives!(Null); + test_primitives!(Boolean); + test_primitives!(Int); + test_primitives!(Long); + test_primitives!(Float); + test_primitives!(Double); + test_primitives!(Bytes); + test_primitives!(String); + test_primitives!(Uuid); + test_primitives!(BigDecimal); + test_primitives!(Date); + test_primitives!(Duration); + test_primitives!(TimeMicros); + test_primitives!(TimeMillis); + test_primitives!(TimestampMicros); + test_primitives!(TimestampMillis); + test_primitives!(TimestampNanos); + test_primitives!(LocalTimestampMicros); + test_primitives!(LocalTimestampMillis); + test_primitives!(LocalTimestampNanos); + + #[test] + fn test_avro_3939_compare_named_schemata_with_different_names() { + let schema_one = Schema::Ref { + name: Name::from("name1"), + }; - assert_eq!(specification_eq_res, struct_field_eq_res); - } + let schema_two = Schema::Ref { + name: Name::from("name2"), + }; - #[test] - fn test_avro_3939_compare_schemata_not_including_attributes() { - let schema_one = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), - ); - let schema_two = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), - ); - // STRUCT_FIELD_EQ does not include attributes ! - assert!(STRUCT_FIELD_EQ.compare(&schema_one, &schema_two)); - } + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + assert!(!specification_eq_res); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!(!struct_field_eq_res); - #[test] - fn test_avro_3939_compare_schemata_including_attributes() { - let struct_field_eq = StructFieldEq { - include_attributes: true, - }; - let schema_one = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), - ); - let schema_two = Schema::map_with_attributes( - Schema::Boolean, - BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), - ); - assert!(!struct_field_eq.compare(&schema_one, &schema_two)); - } + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_map_schemata() { - let schema_one = Schema::map(Schema::Boolean); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::map(Schema::Boolean); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Map failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Map failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_schemata_not_including_attributes() { + let schema_one = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), + ); + let schema_two = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), + ); + // STRUCT_FIELD_EQ does not include attributes ! + assert!(STRUCT_FIELD_EQ.compare(&schema_one, &schema_two)); + } - #[test] - fn test_avro_3939_compare_array_schemata() { - let schema_one = Schema::array(Schema::Boolean); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::array(Schema::Boolean); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Array failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Array failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_schemata_including_attributes() { + let struct_field_eq = StructFieldEq { + include_attributes: true, + }; + let schema_one = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key1".to_string(), Value::Bool(true))]), + ); + let schema_two = Schema::map_with_attributes( + Schema::Boolean, + BTreeMap::from_iter([("key2".to_string(), Value::Bool(true))]), + ); + assert!(!struct_field_eq.compare(&schema_one, &schema_two)); + } - #[test] - fn test_avro_3939_compare_decimal_schemata() { - let schema_one = Schema::Decimal(DecimalSchema { - precision: 10, - scale: 2, - inner: Box::new(Schema::Bytes), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Decimal(DecimalSchema { - precision: 10, - scale: 2, - inner: Box::new(Schema::Bytes), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Decimal failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Decimal failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_map_schemata() { + let schema_one = Schema::map(Schema::Boolean); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::map(Schema::Boolean); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Map failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Map failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_fixed_schemata() { - let schema_one = Schema::Fixed(FixedSchema { - name: Name::from("fixed"), - doc: None, - size: 10, - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + #[test] + fn test_avro_3939_compare_array_schemata() { + let schema_one = Schema::array(Schema::Boolean); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::array(Schema::Boolean); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Array failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Array failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - let schema_two = Schema::Fixed(FixedSchema { - name: Name::from("fixed"), - doc: None, - size: 10, - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Fixed failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Fixed failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_decimal_schemata() { + let schema_one = Schema::Decimal(DecimalSchema { + precision: 10, + scale: 2, + inner: Box::new(Schema::Bytes), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Decimal(DecimalSchema { + precision: 10, + scale: 2, + inner: Box::new(Schema::Bytes), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Decimal failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Decimal failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_enum_schemata() { - let schema_one = Schema::Enum(EnumSchema { - name: Name::from("enum"), - doc: None, - symbols: vec!["A".to_string(), "B".to_string()], - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + #[test] + fn test_avro_3939_compare_fixed_schemata() { + let schema_one = Schema::Fixed(FixedSchema { + name: Name::from("fixed"), + doc: None, + size: 10, + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Fixed(FixedSchema { + name: Name::from("fixed"), + doc: None, + size: 10, + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Fixed failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Fixed failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - let schema_two = Schema::Enum(EnumSchema { - name: Name::from("enum"), - doc: None, - symbols: vec!["A".to_string(), "B".to_string()], - default: None, - aliases: None, - attributes: BTreeMap::new(), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Enum failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Enum failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + #[test] + fn test_avro_3939_compare_enum_schemata() { + let schema_one = Schema::Enum(EnumSchema { + name: Name::from("enum"), + doc: None, + symbols: vec!["A".to_string(), "B".to_string()], + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Enum(EnumSchema { + name: Name::from("enum"), + doc: None, + symbols: vec!["A".to_string(), "B".to_string()], + default: None, + aliases: None, + attributes: BTreeMap::new(), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Enum failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Enum failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_ref_schemata() { - let schema_one = Schema::Ref { - name: Name::from("ref"), - }; - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + #[test] + fn test_avro_3939_compare_ref_schemata() { + let schema_one = Schema::Ref { + name: Name::from("ref"), + }; + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - let schema_two = Schema::Ref { - name: Name::from("ref"), - }; + let schema_two = Schema::Ref { + name: Name::from("ref"), + }; - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Ref failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Ref failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Ref failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Ref failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_record_schemata() { - let schema_one = Schema::Record(RecordSchema { - name: Name::from("record"), + #[test] + fn test_avro_3939_compare_record_schemata() { + let schema_one = Schema::Record(RecordSchema { + name: Name::from("record"), + doc: None, + fields: vec![RecordField { + name: "field".to_string(), doc: None, - fields: vec![RecordField { - name: "field".to_string(), - doc: None, - default: None, - schema: Schema::Boolean, - order: RecordFieldOrder::Ignore, - aliases: None, - custom_attributes: BTreeMap::new(), - position: 0, - }], + default: None, + schema: Schema::Boolean, + order: RecordFieldOrder::Ignore, aliases: None, - attributes: BTreeMap::new(), - lookup: Default::default(), - }); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Record(RecordSchema { - name: Name::from("record"), + custom_attributes: BTreeMap::new(), + position: 0, + }], + aliases: None, + attributes: BTreeMap::new(), + lookup: Default::default(), + }); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Record(RecordSchema { + name: Name::from("record"), + doc: None, + fields: vec![RecordField { + name: "field".to_string(), doc: None, - fields: vec![RecordField { - name: "field".to_string(), - doc: None, - default: None, - schema: Schema::Boolean, - order: RecordFieldOrder::Ignore, - aliases: None, - custom_attributes: BTreeMap::new(), - position: 0, - }], + default: None, + schema: Schema::Boolean, + order: RecordFieldOrder::Ignore, aliases: None, - attributes: BTreeMap::new(), - lookup: Default::default(), - }); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Record failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Record failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - } + custom_attributes: BTreeMap::new(), + position: 0, + }], + aliases: None, + attributes: BTreeMap::new(), + lookup: Default::default(), + }); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Record failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Record failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + } - #[test] - fn test_avro_3939_compare_union_schemata() -> TestResult { - let schema_one = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); - assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); - assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); - - let schema_two = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); - - let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); - let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); - assert!( - specification_eq_res, - "SpecificationEq: Equality of two Schema::Union failed!" - ); - assert!( - struct_field_eq_res, - "StructFieldEq: Equality of two Schema::Union failed!" - ); - assert_eq!(specification_eq_res, struct_field_eq_res); - Ok(()) - } + #[test] + fn test_avro_3939_compare_union_schemata() -> TestResult { + let schema_one = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); + assert!(!SPECIFICATION_EQ.compare(&schema_one, &Schema::Boolean)); + assert!(!STRUCT_FIELD_EQ.compare(&schema_one, &Schema::Boolean)); + + let schema_two = Schema::Union(UnionSchema::new(vec![Schema::Boolean, Schema::Int])?); + + let specification_eq_res = SPECIFICATION_EQ.compare(&schema_one, &schema_two); + let struct_field_eq_res = STRUCT_FIELD_EQ.compare(&schema_one, &schema_two); + assert!( + specification_eq_res, + "SpecificationEq: Equality of two Schema::Union failed!" + ); + assert!( + struct_field_eq_res, + "StructFieldEq: Equality of two Schema::Union failed!" + ); + assert_eq!(specification_eq_res, struct_field_eq_res); + Ok(()) } } +// } diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 991004d0..28968ebc 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -25,16 +25,13 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -43,8 +40,8 @@ mod ser { use crate::{ bytes::{BytesType, SER_BYTES_TYPE}, - error::tokio::Error, - types::tokio::Value, + error::Error, + types::Value, }; use serde::{Serialize, ser}; use std::{collections::HashMap, iter::once}; @@ -513,7 +510,7 @@ mod ser { #[cfg(test)] mod tests { use super::*; - use crate::decimal::tokio::Decimal; + use crate::decimal::Decimal; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 2e0e79e5..98725f51 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -26,16 +26,13 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -45,8 +42,8 @@ mod bigdecimal { use crate::{ bigdecimal::tokio::big_decimal_as_bytes, encode::tokio::{encode_int, encode_long}, - error::tokio::{Details, Error}, - schema::tokio::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, + error::{Details, Error}, + schema::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, }; use bigdecimal::BigDecimal; use serde::{Serialize, ser}; @@ -1848,9 +1845,10 @@ mod bigdecimal { #[cfg(test)] mod tests { use super::*; + use crate::schema::tokio::SchemaExt; use crate::{ - Days, Duration, Millis, Months, decimal::tokio::Decimal, error::tokio::Details, - schema::tokio::ResolvedSchema, + Days, Duration, Millis, Months, decimal::Decimal, error::Details, + schema::ResolvedSchema, }; use apache_avro_test_helper::TestResult; use bigdecimal::BigDecimal; @@ -2034,7 +2032,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "record", "name": "TestRecord", @@ -2087,7 +2085,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_empty_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "record", "name": "EmptyRecord", @@ -2140,7 +2138,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_enum() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "enum", "name": "Suit", @@ -2189,7 +2187,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_array() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "array", "items": "long" @@ -2228,7 +2226,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_map() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "map", "values": "long" @@ -2275,7 +2273,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_nullable_union() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": ["null", "long"] }"#, @@ -2324,7 +2322,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_union() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": ["null", "long", "string"] }"#, @@ -2376,7 +2374,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_fixed() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "fixed", "size": 8, @@ -2453,7 +2451,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_decimal_bytes() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "bytes", "logicalType": "decimal", @@ -2491,7 +2489,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_decimal_fixed() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "fixed", "name": "FixedDecimal", @@ -2532,7 +2530,7 @@ mod bigdecimal { #[tokio::test] #[serial(serde_is_human_readable)] async fn test_serialize_bigdecimal() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "bytes", "logicalType": "big-decimal" @@ -2557,7 +2555,7 @@ mod bigdecimal { #[tokio::test] #[serial(serde_is_human_readable)] async fn test_serialize_uuid() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "string", "logicalType": "uuid" @@ -2602,7 +2600,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_date() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "int", "logicalType": "date" @@ -2647,7 +2645,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_time_millis() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "int", "logicalType": "time-millis" @@ -2692,7 +2690,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_time_micros() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "long", "logicalType": "time-micros" @@ -2741,7 +2739,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_timestamp() -> TestResult { for precision in ["millis", "micros", "nanos"] { - let schema = Schema::parse_str(&format!( + let schema = SchemaExt::parse_str(&format!( r#"{{ "type": "long", "logicalType": "timestamp-{precision}" @@ -2799,7 +2797,7 @@ mod bigdecimal { #[tokio::test] async fn test_serialize_duration() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "fixed", "size": 12, @@ -2846,7 +2844,7 @@ mod bigdecimal { #[tokio::test] #[serial(serde_is_human_readable)] // for BigDecimal and Uuid async fn test_serialize_recursive_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "record", "name": "TestRecord", diff --git a/avro/src/types.rs b/avro/src/types.rs index 42431e85..612e2f33 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -17,315 +17,289 @@ //! Logic handling the intermediate representation of Avro values. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, - crate::ser::tokio => crate::ser::sync, - crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, - #[tokio::test] => #[test] - ); - } -)] -mod types { - - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; - use crate::{ - bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, - decimal::tokio::Decimal, - duration::Duration, - error::tokio::Details, - error::tokio::Error, - schema::tokio::{ - DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, - RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, - }, - }; - use bigdecimal::BigDecimal; - use log::{debug, error}; - use std::{ - collections::{BTreeMap, HashMap}, - fmt::Debug, - hash::BuildHasher, - str::FromStr, - }; - use uuid::Uuid; +use crate::Decimal; +use crate::Duration; +use crate::Uuid; +use crate::bigdecimal::BigDecimal; +use crate::schema::{RecordSchema, Schema}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Debug, + hash::BuildHasher, +}; + +/// A valid Avro value. +/// +/// More information about Avro values can be found in the [Avro +/// Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) +#[derive(Clone, Debug, PartialEq, strum_macros::EnumDiscriminants)] +#[strum_discriminants(name(ValueKind))] +pub enum Value { + /// A `null` Avro value. + Null, + /// A `boolean` Avro value. + Boolean(bool), + /// A `int` Avro value. + Int(i32), + /// A `long` Avro value. + Long(i64), + /// A `float` Avro value. + Float(f32), + /// A `double` Avro value. + Double(f64), + /// A `bytes` Avro value. + Bytes(Vec), + /// A `string` Avro value. + String(String), + /// A `fixed` Avro value. + /// The size of the fixed value is represented as a `usize`. + Fixed(usize, Vec), + /// An `enum` Avro value. + /// + /// An Enum is represented by a symbol and its position in the symbols list + /// of its corresponding schema. + /// This allows schema-less encoding, as well as schema resolution while + /// reading values. + Enum(u32, String), + /// An `union` Avro value. + /// + /// A Union is represented by the value it holds and its position in the type list + /// of its corresponding schema + /// This allows schema-less encoding, as well as schema resolution while + /// reading values. + Union(u32, Box), + /// An `array` Avro value. + Array(Vec), + /// A `map` Avro value. + Map(HashMap), + /// A `record` Avro value. + /// + /// A Record is represented by a vector of (``, `value`). + /// This allows schema-less encoding. + /// + /// See [Record](types.Record) for a more user-friendly support. + Record(Vec<(String, Value)>), + /// A date value. + /// + /// Serialized and deserialized as `i32` directly. Can only be deserialized properly with a + /// schema. + Date(i32), + /// An Avro Decimal value. Bytes are in big-endian order, per the Avro spec. + Decimal(Decimal), + /// An Avro Decimal value. + BigDecimal(BigDecimal), + /// Time in milliseconds. + TimeMillis(i32), + /// Time in microseconds. + TimeMicros(i64), + /// Timestamp in milliseconds. + TimestampMillis(i64), + /// Timestamp in microseconds. + TimestampMicros(i64), + /// Timestamp in nanoseconds. + TimestampNanos(i64), + /// Local timestamp in milliseconds. + LocalTimestampMillis(i64), + /// Local timestamp in microseconds. + LocalTimestampMicros(i64), + /// Local timestamp in nanoseconds. + LocalTimestampNanos(i64), + /// Avro Duration. An amount of time defined by months, days and milliseconds. + Duration(Duration), + /// Universally unique identifier. + Uuid(Uuid), +} - /// Compute the maximum decimal value precision of a byte array of length `len` could hold. - fn max_prec_for_len(len: usize) -> Result { - let len = i32::try_from(len).map_err(|e| Details::ConvertLengthToI32(e, len))?; - Ok((2.0_f64.powi(8 * len - 1) - 1.0).log10().floor() as usize) +impl From<()> for Value { + fn from(_: ()) -> Self { + Self::Null } +} - /// A valid Avro value. - /// - /// More information about Avro values can be found in the [Avro - /// Specification](https://avro.apache.org/docs/current/specification/#schema-declaration) - #[derive(Clone, Debug, PartialEq, strum_macros::EnumDiscriminants)] - #[strum_discriminants(name(ValueKind))] - pub enum Value { - /// A `null` Avro value. - Null, - /// A `boolean` Avro value. - Boolean(bool), - /// A `int` Avro value. - Int(i32), - /// A `long` Avro value. - Long(i64), - /// A `float` Avro value. - Float(f32), - /// A `double` Avro value. - Double(f64), - /// A `bytes` Avro value. - Bytes(Vec), - /// A `string` Avro value. - String(String), - /// A `fixed` Avro value. - /// The size of the fixed value is represented as a `usize`. - Fixed(usize, Vec), - /// An `enum` Avro value. - /// - /// An Enum is represented by a symbol and its position in the symbols list - /// of its corresponding schema. - /// This allows schema-less encoding, as well as schema resolution while - /// reading values. - Enum(u32, String), - /// An `union` Avro value. - /// - /// A Union is represented by the value it holds and its position in the type list - /// of its corresponding schema - /// This allows schema-less encoding, as well as schema resolution while - /// reading values. - Union(u32, Box), - /// An `array` Avro value. - Array(Vec), - /// A `map` Avro value. - Map(HashMap), - /// A `record` Avro value. - /// - /// A Record is represented by a vector of (``, `value`). - /// This allows schema-less encoding. - /// - /// See [Record](types.Record) for a more user-friendly support. - Record(Vec<(String, Value)>), - /// A date value. - /// - /// Serialized and deserialized as `i32` directly. Can only be deserialized properly with a - /// schema. - Date(i32), - /// An Avro Decimal value. Bytes are in big-endian order, per the Avro spec. - Decimal(Decimal), - /// An Avro Decimal value. - BigDecimal(BigDecimal), - /// Time in milliseconds. - TimeMillis(i32), - /// Time in microseconds. - TimeMicros(i64), - /// Timestamp in milliseconds. - TimestampMillis(i64), - /// Timestamp in microseconds. - TimestampMicros(i64), - /// Timestamp in nanoseconds. - TimestampNanos(i64), - /// Local timestamp in milliseconds. - LocalTimestampMillis(i64), - /// Local timestamp in microseconds. - LocalTimestampMicros(i64), - /// Local timestamp in nanoseconds. - LocalTimestampNanos(i64), - /// Avro Duration. An amount of time defined by months, days and milliseconds. - Duration(Duration), - /// Universally unique identifier. - Uuid(Uuid), +impl From for Value { + fn from(value: usize) -> Self { + i64::try_from(value) + .expect("cannot convert usize to i64") + .into() } +} - macro_rules! to_value( - ($type:ty, $variant_constructor:expr) => ( - impl From<$type> for Value { - fn from(value: $type) -> Self { - $variant_constructor(value) - } - } - ); -); - - to_value!(bool, Value::Boolean); - to_value!(i32, Value::Int); - to_value!(i64, Value::Long); - to_value!(f32, Value::Float); - to_value!(f64, Value::Double); - to_value!(String, Value::String); - to_value!(Vec, Value::Bytes); - to_value!(uuid::Uuid, Value::Uuid); - to_value!(Decimal, Value::Decimal); - to_value!(BigDecimal, Value::BigDecimal); - to_value!(Duration, Value::Duration); - - impl From<()> for Value { - fn from(_: ()) -> Self { - Self::Null - } +impl From<&str> for Value { + fn from(value: &str) -> Self { + Self::String(value.to_owned()) } +} - impl From for Value { - fn from(value: usize) -> Self { - i64::try_from(value) - .expect("cannot convert usize to i64") - .into() - } +impl From<&[u8]> for Value { + fn from(value: &[u8]) -> Self { + Self::Bytes(value.to_owned()) } +} - impl From<&str> for Value { - fn from(value: &str) -> Self { - Self::String(value.to_owned()) - } +impl From> for Value +where + T: Into, +{ + fn from(value: Option) -> Self { + // FIXME: this is incorrect in case first type in union is not "none" + Self::Union( + value.is_some() as u32, + Box::new(value.map_or_else(|| Self::Null, Into::into)), + ) } +} - impl From<&[u8]> for Value { - fn from(value: &[u8]) -> Self { - Self::Bytes(value.to_owned()) - } +impl From> for Value +where + K: Into, + V: Into, + S: BuildHasher, +{ + fn from(value: HashMap) -> Self { + Self::Map( + value + .into_iter() + .map(|(key, value)| (key.into(), value.into())) + .collect(), + ) } +} - impl From> for Value - where - T: Into, - { - fn from(value: Option) -> Self { - // FIXME: this is incorrect in case first type in union is not "none" - Self::Union( - value.is_some() as u32, - Box::new(value.map_or_else(|| Self::Null, Into::into)), - ) +/// Utility interface to build `Value::Record` objects. +#[derive(Debug, Clone)] +pub struct Record<'a> { + /// List of fields contained in the record. + /// Ordered according to the fields in the schema given to create this + /// `Record` object. Any unset field defaults to `Value::Null`. + pub fields: Vec<(String, Value)>, + schema_lookup: &'a BTreeMap, +} + +impl Record<'_> { + /// Create a `Record` given a `Schema`. + /// + /// If the `Schema` is not a `Schema::Record` variant, `None` will be returned. + pub fn new(schema: &Schema) -> Option> { + match *schema { + Schema::Record(RecordSchema { + fields: ref schema_fields, + lookup: ref schema_lookup, + .. + }) => { + let mut fields = Vec::with_capacity(schema_fields.len()); + for schema_field in schema_fields.iter() { + fields.push((schema_field.name.clone(), Value::Null)); + } + + Some(Record { + fields, + schema_lookup, + }) + } + _ => None, } } - impl From> for Value + /// Put a compatible value (implementing the `ToAvro` trait) in the + /// `Record` for a given `field` name. + /// + /// **NOTE** Only ensure that the field name is present in the `Schema` given when creating + /// this `Record`. Does not perform any schema validation. + pub fn put(&mut self, field: &str, value: V) where - K: Into, - V: Into, - S: BuildHasher, + V: Into, { - fn from(value: HashMap) -> Self { - Self::Map( - value - .into_iter() - .map(|(key, value)| (key.into(), value.into())) - .collect(), - ) + if let Some(&position) = self.schema_lookup.get(field) { + self.fields[position].1 = value.into() } } - /// Utility interface to build `Value::Record` objects. - #[derive(Debug, Clone)] - pub struct Record<'a> { - /// List of fields contained in the record. - /// Ordered according to the fields in the schema given to create this - /// `Record` object. Any unset field defaults to `Value::Null`. - pub fields: Vec<(String, Value)>, - schema_lookup: &'a BTreeMap, + /// Get the value for a given field name. + /// Returns `None` if the field is not present in the schema + pub fn get(&self, field: &str) -> Option<&Value> { + self.schema_lookup + .get(field) + .map(|&position| &self.fields[position].1) } +} - impl Record<'_> { - /// Create a `Record` given a `Schema`. - /// - /// If the `Schema` is not a `Schema::Record` variant, `None` will be returned. - pub fn new(schema: &Schema) -> Option> { - match *schema { - Schema::Record(RecordSchema { - fields: ref schema_fields, - lookup: ref schema_lookup, - .. - }) => { - let mut fields = Vec::with_capacity(schema_fields.len()); - for schema_field in schema_fields.iter() { - fields.push((schema_field.name.clone(), Value::Null)); - } +impl<'a> From> for Value { + fn from(value: Record<'a>) -> Self { + Self::Record(value.fields) + } +} - Some(Record { - fields, - schema_lookup, - }) +impl From for Value { + fn from(value: serde_json::Value) -> Self { + match value { + serde_json::Value::Null => Self::Null, + serde_json::Value::Bool(b) => b.into(), + serde_json::Value::Number(ref n) if n.is_i64() => { + let n = n.as_i64().unwrap(); + if n >= i32::MIN as i64 && n <= i32::MAX as i64 { + Value::Int(n as i32) + } else { + Value::Long(n) } - _ => None, } - } - - /// Put a compatible value (implementing the `ToAvro` trait) in the - /// `Record` for a given `field` name. - /// - /// **NOTE** Only ensure that the field name is present in the `Schema` given when creating - /// this `Record`. Does not perform any schema validation. - pub fn put(&mut self, field: &str, value: V) - where - V: Into, - { - if let Some(&position) = self.schema_lookup.get(field) { - self.fields[position].1 = value.into() + serde_json::Value::Number(ref n) if n.is_f64() => Value::Double(n.as_f64().unwrap()), + serde_json::Value::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great + serde_json::Value::String(s) => s.into(), + serde_json::Value::Array(items) => { + Value::Array(items.into_iter().map(Value::from).collect()) } - } - - /// Get the value for a given field name. - /// Returns `None` if the field is not present in the schema - pub fn get(&self, field: &str) -> Option<&Value> { - self.schema_lookup - .get(field) - .map(|&position| &self.fields[position].1) + serde_json::Value::Object(items) => Value::Map( + items + .into_iter() + .map(|(key, value)| (key, value.into())) + .collect(), + ), } } +} - impl<'a> From> for Value { - fn from(value: Record<'a>) -> Self { - Self::Record(value.fields) - } - } +#[synca::synca( + #[cfg(feature = "tokio")] + pub mod tokio { }, + #[cfg(feature = "sync")] + pub mod sync { + sync!(); + replace!( + crate::bigdecimal::tokio => crate::bigdecimal::sync, + crate::decode::tokio => crate::decode::sync, + crate::encode::tokio => crate::encode::sync, + crate::error::tokio => crate::error::sync, + crate::schema::tokio => crate::schema::sync, + crate::util::tokio => crate::util::sync, + crate::types::tokio => crate::types::sync, + crate::ser::tokio => crate::ser::sync, + crate::util::tokio => crate::util::sync, + #[tokio::test] => #[test] + ); + } +)] +mod types { - impl From for Value { - fn from(value: serde_json::Value) -> Self { - match value { - serde_json::Value::Null => Self::Null, - serde_json::Value::Bool(b) => b.into(), - serde_json::Value::Number(ref n) if n.is_i64() => { - let n = n.as_i64().unwrap(); - if n >= i32::MIN as i64 && n <= i32::MAX as i64 { - Value::Int(n as i32) - } else { - Value::Long(n) - } - } - serde_json::Value::Number(ref n) if n.is_f64() => { - Value::Double(n.as_f64().unwrap()) - } - serde_json::Value::Number(n) => Value::Long(n.as_u64().unwrap() as i64), // TODO: Not so great - serde_json::Value::String(s) => s.into(), - serde_json::Value::Array(items) => { - Value::Array(items.into_iter().map(Value::from).collect()) - } - serde_json::Value::Object(items) => Value::Map( - items - .into_iter() - .map(|(key, value)| (key, value.into())) - .collect(), - ), - } - } - } + use crate::AvroResult; + use crate::{ + Uuid, + bigdecimal::{ + BigDecimal, + tokio::{deserialize_big_decimal, serialize_big_decimal}, + }, + decimal::Decimal, + duration::Duration, + error::Details, + error::Error, + schema::{ + DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, + RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, + }, + types::Value, + }; + use log::{debug, error}; + use std::collections::HashMap; + use std::str::FromStr; /// Convert Avro values to Json values impl TryFrom for serde_json::Value { @@ -389,24 +363,53 @@ mod types { } } - impl Value { + /// Compute the maximum decimal value precision of a byte array of length `len` could hold. + fn max_prec_for_len(len: usize) -> Result { + let len = i32::try_from(len).map_err(|e| Details::ConvertLengthToI32(e, len))?; + Ok((2.0_f64.powi(8 * len - 1) - 1.0).log10().floor() as usize) + } + + macro_rules! to_value( + ($type:ty, $variant_constructor:expr) => ( + impl From<$type> for Value { + fn from(value: $type) -> Self { + $variant_constructor(value) + } + } + ); +); + + to_value!(bool, Value::Boolean); + to_value!(i32, Value::Int); + to_value!(i64, Value::Long); + to_value!(f32, Value::Float); + to_value!(f64, Value::Double); + to_value!(String, Value::String); + to_value!(Vec, Value::Bytes); + to_value!(uuid::Uuid, Value::Uuid); + to_value!(Decimal, Value::Decimal); + to_value!(BigDecimal, Value::BigDecimal); + to_value!(Duration, Value::Duration); + + pub struct ValueExt; + + impl ValueExt { /// Validate the value against the given [Schema](../schema/enum.Schema.html). /// /// See the [Avro specification](https://avro.apache.org/docs/current/specification) /// for the full set of rules of schema validation. - pub async fn validate(&self, schema: &Schema) -> bool { - self.validate_schemata(vec![schema]).await + pub async fn validate(value: &Value, schema: &Schema) -> bool { + Self::validate_schemata(value, vec![schema]).await } - pub async fn validate_schemata(&self, schemata: Vec<&Schema>) -> bool { + pub async fn validate_schemata(value: &Value, schemata: Vec<&Schema>) -> bool { let rs = ResolvedSchema::try_from(schemata.clone()) .expect("Schemata didn't successfully resolve"); let schemata_len = schemata.len(); for schema in schemata { let enclosing_namespace = schema.namespace(); - match self - .validate_internal(schema, rs.get_names(), &enclosing_namespace) + match Self::validate_internal(value, schema, rs.get_names(), &enclosing_namespace) .await { Some(reason) => { @@ -436,16 +439,16 @@ mod types { /// Validates the value against the provided schema. pub(crate) async fn validate_internal( - &self, + value: &Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, ) -> Option { - match (self, schema) { + match (value, schema) { (_, Schema::Ref { name }) => { let name = name.fully_qualified_name(enclosing_namespace); if let Some(s) = names.get(&name) { - Box::pin(self.validate_internal(s, names, &name.namespace)).await + Box::pin(Self::validate_internal(value, s, names, &name.namespace)).await } else { Some(format!( "Unresolved schema reference: '{:?}'. Parsed names: {:?}", @@ -691,8 +694,8 @@ mod types { /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) /// in the Avro specification for the full set of rules of schema /// resolution. - pub async fn resolve(self, schema: &Schema) -> AvroResult { - self.resolve_schemata(schema, Vec::with_capacity(0)).await + pub async fn resolve(value: Value, schema: &Schema) -> AvroResult { + Self::resolve_schemata(value, schema, Vec::with_capacity(0)).await } /// Attempt to perform schema resolution on the value, with the given @@ -702,7 +705,7 @@ mod types { /// in the Avro specification for the full set of rules of schema /// resolution. pub async fn resolve_schemata( - self, + value: Value, schema: &Schema, schemata: Vec<&Schema>, ) -> AvroResult { @@ -712,27 +715,26 @@ mod types { } else { ResolvedSchema::try_from(schemata)? }; - self.resolve_internal(schema, rs.get_names(), &enclosing_namespace, &None) - .await + Self::resolve_internal(value, schema, rs.get_names(), &enclosing_namespace, &None).await } pub(crate) async fn resolve_internal( - mut self, + value: &mut Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, ) -> AvroResult { // Check if this schema is a union, and if the reader schema is not. - if SchemaKind::from(&self) == SchemaKind::Union + if SchemaKind::from(value) == SchemaKind::Union && SchemaKind::from(schema) != SchemaKind::Union { // Pull out the Union, and attempt to resolve against it. - let v = match self { + let v = match *value { Value::Union(_i, b) => *b, _ => unreachable!(), }; - self = v; + value = v; } match *schema { Schema::Ref { ref name } => { @@ -1301,9 +1303,11 @@ mod types { use super::*; use crate::{ duration::{Days, Millis, Months}, - error::tokio::Details, - schema::tokio::RecordFieldOrder, + error::Details, + schema::RecordFieldOrder, + schema::tokio::SchemaExt, ser::tokio::Serializer, + types::{Record, Value}, }; use apache_avro_test_helper::{ TestResult, @@ -1315,7 +1319,7 @@ mod types { #[tokio::test] async fn avro_3809_validate_nested_records_with_implicit_namespace() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "name": "record_name", "namespace": "space", @@ -2054,7 +2058,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3621_resolve_to_nullable_union() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#"{ "type": "record", "name": "root", @@ -2287,7 +2291,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3433_recursive_resolves_record() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2326,7 +2330,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3433_recursive_resolves_array() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2377,7 +2381,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3433_recursive_resolves_map() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2425,7 +2429,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3433_recursive_resolves_record_wrapper() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2472,7 +2476,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3433_recursive_resolves_map_and_array() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2523,7 +2527,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3433_recursive_resolves_union() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2611,7 +2615,7 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -2703,7 +2707,7 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( "middle_field_1".into(), @@ -2796,7 +2800,7 @@ Field with name '"b"' is not a member of the map items"#, ] } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let inner_record = Value::Record(vec![("inner_field_1".into(), Value::Double(5.4))]); let middle_record_variation_1 = Value::Record(vec![( @@ -2847,7 +2851,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3460_validation_with_refs() -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -2927,7 +2931,7 @@ Field with name '"b"' is not a member of the map items"#, b: Option, // could be literally anything } - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type":"record", @@ -3034,7 +3038,7 @@ Field with name '"b"' is not a member of the map items"#, }, ); - let schema = Schema::parse_str(&schema_str).await?; + let schema = SchemaExt::parse_str(&schema_str).await?; #[derive(Serialize)] enum EnumType { @@ -3132,7 +3136,7 @@ Field with name '"b"' is not a member of the map items"#, field_b: Option, } - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; let msg = Message { field_a: Some(Inner { @@ -3187,7 +3191,7 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::from(value); - let schemas = Schema::parse_list([main_schema, referenced_schema]).await?; + let schemas = SchemaExt::parse_list([main_schema, referenced_schema]).await?; let main_schema = schemas.first().unwrap(); let schemata: Vec<_> = schemas.iter().skip(1).collect(); @@ -3231,7 +3235,7 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::from(value); let schemata = - Schema::parse_list([referenced_enum, referenced_record, main_schema]).await?; + SchemaExt::parse_list([referenced_enum, referenced_record, main_schema]).await?; let main_schema = schemata.last().unwrap(); let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); @@ -3262,7 +3266,7 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::Decimal(Decimal::from( BigInt::from(12345678u32).to_signed_bytes_be(), )); - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let resolve_result = avro_value.resolve(&schema).await; assert!( resolve_result.is_ok(), @@ -3278,7 +3282,7 @@ Field with name '"b"' is not a member of the map items"#, r#"{"name": "bigDecimalSchema", "logicalType": "big-decimal", "type": "bytes" }"#; let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let resolve_result: AvroResult = avro_value.resolve(&schema).await; assert!( resolve_result.is_ok(), @@ -3385,7 +3389,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "double"}"#).await?; + let schema = SchemaExt::parse_str(r#"{"type": "double"}"#).await?; let value = Value::String("unknown".to_owned()); match value.resolve(&schema).await.map_err(Error::into_details) { Err(err @ Details::GetDouble(_)) => { @@ -3403,7 +3407,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { - let schema = Schema::parse_str(r#"{"type": "float"}"#).await?; + let schema = SchemaExt::parse_str(r#"{"type": "float"}"#).await?; let value = Value::String("unknown".to_owned()); match value.resolve(&schema).await.map_err(Error::into_details) { Err(err @ Details::GetFloat(_)) => { @@ -3555,7 +3559,7 @@ Field with name '"b"' is not a member of the map items"#, ]; for (schema_str, value, expected_error) in data { - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; match value.resolve(&schema).await { Err(error) => { assert_eq!(format!("{error}"), expected_error); @@ -3588,7 +3592,7 @@ Field with name '"b"' is not a member of the map items"#, } "#; - let schema = Schema::parse_str(schema).await?; + let schema = SchemaExt::parse_str(schema).await?; let mut record = Record::new(&schema).unwrap(); record.put("foo", "hello"); record.put("bar", 123_i64); diff --git a/avro/src/util.rs b/avro/src/util.rs index 44e0ebb4..eb6b8999 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use crate::schema::Documentation; +use serde_json::{Map, Value}; use std::sync::atomic::Ordering; use std::sync::{ Once, @@ -68,6 +70,41 @@ pub fn set_serde_human_readable(human_readable: bool) { }); } +pub trait MapHelper { + fn string(&self, key: &str) -> Option; + + fn name(&self) -> Option { + self.string("name") + } + + fn doc(&self) -> Documentation { + self.string("doc") + } + + fn aliases(&self) -> Option>; +} + +impl MapHelper for Map { + fn string(&self, key: &str) -> Option { + self.get(key) + .and_then(|v| v.as_str()) + .map(|v| v.to_string()) + } + + fn aliases(&self) -> Option> { + // FIXME no warning when aliases aren't a json array of json strings + self.get("aliases") + .and_then(|aliases| aliases.as_array()) + .and_then(|aliases| { + aliases + .iter() + .map(|alias| alias.as_str()) + .map(|alias| alias.map(|a| a.to_string())) + .collect::>() + }) + } +} + #[synca::synca( #[cfg(feature = "tokio")] pub mod tokio { }, @@ -76,16 +113,13 @@ pub fn set_serde_human_readable(human_readable: bool) { sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, crate::schema::tokio => crate::schema::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } @@ -95,64 +129,30 @@ mod util { use futures::future::TryFutureExt; #[synca::cfg(sync)] use std::io::Read as AvroRead; + #[synca::cfg(sync)] + use std::io::Write as AvroWrite; #[synca::cfg(tokio)] use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] + use tokio::io::AsyncWrite as AvroWrite; + #[cfg(feature = "tokio")] + use tokio::io::AsyncWriteExt; + use crate::AvroResult; - use crate::error::tokio::Details; - use crate::schema::tokio::Documentation; - use serde_json::{Map, Value}; + use crate::error::Details; use std::io::Write; - pub trait MapHelper { - fn string(&self, key: &str) -> Option; - - fn name(&self) -> Option { - self.string("name") - } - - fn doc(&self) -> Documentation { - self.string("doc") - } - - fn aliases(&self) -> Option>; - } - - impl MapHelper for Map { - fn string(&self, key: &str) -> Option { - self.get(key) - .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - } - - fn aliases(&self) -> Option> { - // FIXME no warning when aliases aren't a json array of json strings - self.get("aliases") - .and_then(|aliases| aliases.as_array()) - .and_then(|aliases| { - aliases - .iter() - .map(|alias| alias.as_str()) - .map(|alias| alias.map(|a| a.to_string())) - .collect::>() - }) - } - } - pub async fn read_long(reader: &mut R) -> AvroResult { zag_i64(reader).await } - pub fn zig_i32(n: i32, buffer: W) -> AvroResult { + pub async fn zig_i32(n: i32, buffer: W) -> AvroResult { zig_i64(n as i64, buffer) } - pub fn zig_i64(n: i64, writer: W) -> AvroResult { + pub async fn zig_i64(n: i64, writer: W) -> AvroResult { encode_variable(((n << 1) ^ (n >> 63)) as u64, writer) } @@ -170,7 +170,7 @@ mod util { }) } - fn encode_variable(mut z: u64, mut writer: W) -> AvroResult { + async fn encode_variable(mut z: u64, mut writer: W) -> AvroResult { let mut buffer = [0u8; 10]; let mut i: usize = 0; loop { diff --git a/avro/src/validator.rs b/avro/src/validator.rs index 1f194ee7..cc22a8ca 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -15,337 +15,330 @@ // specific language governing permissions and limitations // under the License. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decimal::tokio => crate::decimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, - crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, - #[tokio::test] => #[test] - ); - } -)] -mod validator { - - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] - use crate::AvroResult; - use crate::{error::tokio::Details, schema::tokio::Namespace}; - use log::debug; - use regex_lite::Regex; - use std::sync::OnceLock; - - /// A validator that validates names and namespaces according to the Avro specification. - struct SpecificationValidator; - - /// A trait that validates schema names. - /// To register a custom one use [set_schema_name_validator]. - pub trait SchemaNameValidator: Send + Sync { - /// Returns the regex used to validate the schema name - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static SCHEMA_NAME_ONCE: OnceLock = OnceLock::new(); - SCHEMA_NAME_ONCE.get_or_init(|| { +// #[synca::synca( +// #[cfg(feature = "tokio")] +// pub mod tokio { }, +// #[cfg(feature = "sync")] +// pub mod sync { +// sync!(); +// replace!( +// crate::bigdecimal::tokio => crate::bigdecimal::sync, +// crate::decode::tokio => crate::decode::sync, +// crate::encode::tokio => crate::encode::sync, +// crate::error::tokio => crate::error::sync, +// crate::schema::tokio => crate::schema::sync, +// crate::util::tokio => crate::util::sync, +// crate::types::tokio => crate::types::sync, +// crate::schema_equality::tokio => crate::schema_equality::sync, +// crate::util::tokio => crate::util::sync, +// crate::validator::tokio => crate::validator::sync, +// #[tokio::test] => #[test] +// ); +// } +// )] +// mod validator { + +use crate::AvroResult; +use crate::{error::Details, schema::Namespace}; +use log::debug; +use regex_lite::Regex; +use std::sync::OnceLock; + +/// A validator that validates names and namespaces according to the Avro specification. +struct SpecificationValidator; + +/// A trait that validates schema names. +/// To register a custom one use [set_schema_name_validator]. +pub trait SchemaNameValidator: Send + Sync { + /// Returns the regex used to validate the schema name + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static SCHEMA_NAME_ONCE: OnceLock = OnceLock::new(); + SCHEMA_NAME_ONCE.get_or_init(|| { Regex::new( // An optional namespace (with optional dots) followed by a name without any dots in it. r"^((?P([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?)\.)?(?P[A-Za-z_][A-Za-z0-9_]*)$", ) .unwrap() }) - } - - /// Validates the schema name and returns the name and the optional namespace, - /// or [Details::InvalidSchemaName] if it is invalid. - fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)>; } - impl SchemaNameValidator for SpecificationValidator { - fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)> { - let regex = SchemaNameValidator::regex(self); - let caps = regex.captures(schema_name).ok_or_else(|| { - Details::InvalidSchemaName(schema_name.to_string(), regex.as_str()) - })?; - Ok(( - caps["name"].to_string(), - caps.name("namespace").map(|s| s.as_str().to_string()), - )) - } - } + /// Validates the schema name and returns the name and the optional namespace, + /// or [Details::InvalidSchemaName] if it is invalid. + fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)>; +} - static NAME_VALIDATOR_ONCE: OnceLock> = - OnceLock::new(); - - /// Sets a custom schema name validator. - /// - /// Returns a unit if the registration was successful or the already - /// registered validator if the registration failed. - /// - /// **Note**: This function must be called before parsing any schema because this will - /// register the default validator and the registration is one time only! - pub fn set_schema_name_validator( - validator: Box, - ) -> Result<(), Box> { - debug!("Setting a custom schema name validator."); - NAME_VALIDATOR_ONCE.set(validator) +impl SchemaNameValidator for SpecificationValidator { + fn validate(&self, schema_name: &str) -> AvroResult<(String, Namespace)> { + let regex = SchemaNameValidator::regex(self); + let caps = regex + .captures(schema_name) + .ok_or_else(|| Details::InvalidSchemaName(schema_name.to_string(), regex.as_str()))?; + Ok(( + caps["name"].to_string(), + caps.name("namespace").map(|s| s.as_str().to_string()), + )) } +} - pub(crate) fn validate_schema_name(schema_name: &str) -> AvroResult<(String, Namespace)> { - NAME_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default name validator."); - Box::new(SpecificationValidator) - }) - .validate(schema_name) - } +static NAME_VALIDATOR_ONCE: OnceLock> = OnceLock::new(); + +/// Sets a custom schema name validator. +/// +/// Returns a unit if the registration was successful or the already +/// registered validator if the registration failed. +/// +/// **Note**: This function must be called before parsing any schema because this will +/// register the default validator and the registration is one time only! +pub fn set_schema_name_validator( + validator: Box, +) -> Result<(), Box> { + debug!("Setting a custom schema name validator."); + NAME_VALIDATOR_ONCE.set(validator) +} - /// A trait that validates schema namespaces. - /// To register a custom one use [set_schema_namespace_validator]. - pub trait SchemaNamespaceValidator: Send + Sync { - /// Returns the regex used to validate the schema namespace - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static NAMESPACE_ONCE: OnceLock = OnceLock::new(); - NAMESPACE_ONCE.get_or_init(|| { - Regex::new(r"^([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?$").unwrap() - }) - } +pub(crate) fn validate_schema_name(schema_name: &str) -> AvroResult<(String, Namespace)> { + NAME_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default name validator."); + Box::new(SpecificationValidator) + }) + .validate(schema_name) +} - /// Validates the schema namespace or [Details::InvalidNamespace] if it is invalid. - fn validate(&self, namespace: &str) -> AvroResult<()>; +/// A trait that validates schema namespaces. +/// To register a custom one use [set_schema_namespace_validator]. +pub trait SchemaNamespaceValidator: Send + Sync { + /// Returns the regex used to validate the schema namespace + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static NAMESPACE_ONCE: OnceLock = OnceLock::new(); + NAMESPACE_ONCE.get_or_init(|| { + Regex::new(r"^([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?$").unwrap() + }) } - impl SchemaNamespaceValidator for SpecificationValidator { - fn validate(&self, ns: &str) -> AvroResult<()> { - let regex = SchemaNamespaceValidator::regex(self); - if !regex.is_match(ns) { - Err(Details::InvalidNamespace(ns.to_string(), regex.as_str()).into()) - } else { - Ok(()) - } - } - } + /// Validates the schema namespace or [Details::InvalidNamespace] if it is invalid. + fn validate(&self, namespace: &str) -> AvroResult<()>; +} - static NAMESPACE_VALIDATOR_ONCE: OnceLock> = - OnceLock::new(); - - /// Sets a custom schema namespace validator. - /// - /// Returns a unit if the registration was successful or the already - /// registered validator if the registration failed. - /// - /// **Note**: This function must be called before parsing any schema because this will - /// register the default validator and the registration is one time only! - pub fn set_schema_namespace_validator( - validator: Box, - ) -> Result<(), Box> { - NAMESPACE_VALIDATOR_ONCE.set(validator) +impl SchemaNamespaceValidator for SpecificationValidator { + fn validate(&self, ns: &str) -> AvroResult<()> { + let regex = SchemaNamespaceValidator::regex(self); + if !regex.is_match(ns) { + Err(Details::InvalidNamespace(ns.to_string(), regex.as_str()).into()) + } else { + Ok(()) + } } +} - pub(crate) fn validate_namespace(ns: &str) -> AvroResult<()> { - NAMESPACE_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default namespace validator."); - Box::new(SpecificationValidator) - }) - .validate(ns) - } +static NAMESPACE_VALIDATOR_ONCE: OnceLock> = + OnceLock::new(); + +/// Sets a custom schema namespace validator. +/// +/// Returns a unit if the registration was successful or the already +/// registered validator if the registration failed. +/// +/// **Note**: This function must be called before parsing any schema because this will +/// register the default validator and the registration is one time only! +pub fn set_schema_namespace_validator( + validator: Box, +) -> Result<(), Box> { + NAMESPACE_VALIDATOR_ONCE.set(validator) +} - /// A trait that validates enum symbol names. - /// To register a custom one use [set_enum_symbol_name_validator]. - pub trait EnumSymbolNameValidator: Send + Sync { - /// Returns the regex used to validate the symbols of enum schema - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static ENUM_SYMBOL_NAME_ONCE: OnceLock = OnceLock::new(); - ENUM_SYMBOL_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) - } +pub(crate) fn validate_namespace(ns: &str) -> AvroResult<()> { + NAMESPACE_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default namespace validator."); + Box::new(SpecificationValidator) + }) + .validate(ns) +} - /// Validates the symbols of an Enum schema name and returns nothing (unit), - /// or [Details::EnumSymbolName] if it is invalid. - fn validate(&self, name: &str) -> AvroResult<()>; +/// A trait that validates enum symbol names. +/// To register a custom one use [set_enum_symbol_name_validator]. +pub trait EnumSymbolNameValidator: Send + Sync { + /// Returns the regex used to validate the symbols of enum schema + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static ENUM_SYMBOL_NAME_ONCE: OnceLock = OnceLock::new(); + ENUM_SYMBOL_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) } - impl EnumSymbolNameValidator for SpecificationValidator { - fn validate(&self, symbol: &str) -> AvroResult<()> { - let regex = EnumSymbolNameValidator::regex(self); - if !regex.is_match(symbol) { - return Err(Details::EnumSymbolName(symbol.to_string()).into()); - } + /// Validates the symbols of an Enum schema name and returns nothing (unit), + /// or [Details::EnumSymbolName] if it is invalid. + fn validate(&self, name: &str) -> AvroResult<()>; +} - Ok(()) +impl EnumSymbolNameValidator for SpecificationValidator { + fn validate(&self, symbol: &str) -> AvroResult<()> { + let regex = EnumSymbolNameValidator::regex(self); + if !regex.is_match(symbol) { + return Err(Details::EnumSymbolName(symbol.to_string()).into()); } - } - static ENUM_SYMBOL_NAME_VALIDATOR_ONCE: OnceLock< - Box, - > = OnceLock::new(); - - /// Sets a custom enum symbol name validator. - /// - /// Returns a unit if the registration was successful or the already - /// registered validator if the registration failed. - /// - /// **Note**: This function must be called before parsing any schema because this will - /// register the default validator and the registration is one time only! - pub fn set_enum_symbol_name_validator( - validator: Box, - ) -> Result<(), Box> { - ENUM_SYMBOL_NAME_VALIDATOR_ONCE.set(validator) + Ok(()) } +} - pub(crate) fn validate_enum_symbol_name(symbol: &str) -> AvroResult<()> { - ENUM_SYMBOL_NAME_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default enum symbol name validator."); - Box::new(SpecificationValidator) - }) - .validate(symbol) - } +static ENUM_SYMBOL_NAME_VALIDATOR_ONCE: OnceLock> = + OnceLock::new(); + +/// Sets a custom enum symbol name validator. +/// +/// Returns a unit if the registration was successful or the already +/// registered validator if the registration failed. +/// +/// **Note**: This function must be called before parsing any schema because this will +/// register the default validator and the registration is one time only! +pub fn set_enum_symbol_name_validator( + validator: Box, +) -> Result<(), Box> { + ENUM_SYMBOL_NAME_VALIDATOR_ONCE.set(validator) +} - /// A trait that validates record field names. - /// To register a custom one use [set_record_field_name_validator]. - pub trait RecordFieldNameValidator: Send + Sync { - /// Returns the regex used to validate the record field names - /// according to the Avro specification. - fn regex(&self) -> &'static Regex { - static FIELD_NAME_ONCE: OnceLock = OnceLock::new(); - FIELD_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) - } +pub(crate) fn validate_enum_symbol_name(symbol: &str) -> AvroResult<()> { + ENUM_SYMBOL_NAME_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default enum symbol name validator."); + Box::new(SpecificationValidator) + }) + .validate(symbol) +} - /// Validates the record field's names and returns nothing (unit), - /// or [Details::FieldName] if it is invalid. - fn validate(&self, name: &str) -> AvroResult<()>; +/// A trait that validates record field names. +/// To register a custom one use [set_record_field_name_validator]. +pub trait RecordFieldNameValidator: Send + Sync { + /// Returns the regex used to validate the record field names + /// according to the Avro specification. + fn regex(&self) -> &'static Regex { + static FIELD_NAME_ONCE: OnceLock = OnceLock::new(); + FIELD_NAME_ONCE.get_or_init(|| Regex::new(r"^[A-Za-z_][A-Za-z0-9_]*$").unwrap()) } - impl RecordFieldNameValidator for SpecificationValidator { - fn validate(&self, field_name: &str) -> AvroResult<()> { - let regex = RecordFieldNameValidator::regex(self); - if !regex.is_match(field_name) { - return Err(Details::FieldName(field_name.to_string()).into()); - } + /// Validates the record field's names and returns nothing (unit), + /// or [Details::FieldName] if it is invalid. + fn validate(&self, name: &str) -> AvroResult<()>; +} - Ok(()) +impl RecordFieldNameValidator for SpecificationValidator { + fn validate(&self, field_name: &str) -> AvroResult<()> { + let regex = RecordFieldNameValidator::regex(self); + if !regex.is_match(field_name) { + return Err(Details::FieldName(field_name.to_string()).into()); } - } - static RECORD_FIELD_NAME_VALIDATOR_ONCE: OnceLock< - Box, - > = OnceLock::new(); - - /// Sets a custom record field name validator. - /// - /// Returns a unit if the registration was successful or the already - /// registered validator if the registration failed. - /// - /// **Note**: This function must be called before parsing any schema because this will - /// register the default validator and the registration is one time only! - pub fn set_record_field_name_validator( - validator: Box, - ) -> Result<(), Box> { - RECORD_FIELD_NAME_VALIDATOR_ONCE.set(validator) + Ok(()) } +} - pub(crate) fn validate_record_field_name(field_name: &str) -> AvroResult<()> { - RECORD_FIELD_NAME_VALIDATOR_ONCE - .get_or_init(|| { - debug!("Going to use the default record field name validator."); - Box::new(SpecificationValidator) - }) - .validate(field_name) - } +static RECORD_FIELD_NAME_VALIDATOR_ONCE: OnceLock> = + OnceLock::new(); + +/// Sets a custom record field name validator. +/// +/// Returns a unit if the registration was successful or the already +/// registered validator if the registration failed. +/// +/// **Note**: This function must be called before parsing any schema because this will +/// register the default validator and the registration is one time only! +pub fn set_record_field_name_validator( + validator: Box, +) -> Result<(), Box> { + RECORD_FIELD_NAME_VALIDATOR_ONCE.set(validator) +} - #[cfg(test)] - mod tests { - use super::*; - use crate::schema::tokio::Name; - use apache_avro_test_helper::TestResult; +pub(crate) fn validate_record_field_name(field_name: &str) -> AvroResult<()> { + RECORD_FIELD_NAME_VALIDATOR_ONCE + .get_or_init(|| { + debug!("Going to use the default record field name validator."); + Box::new(SpecificationValidator) + }) + .validate(field_name) +} - #[test] - fn avro_3900_default_name_validator_with_valid_ns() -> TestResult { - validate_schema_name("example")?; - Ok(()) - } +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::Name; + use apache_avro_test_helper::TestResult; - #[test] - fn avro_3900_default_name_validator_with_invalid_ns() -> TestResult { - assert!(validate_schema_name("com-example").is_err()); - Ok(()) - } + #[test] + fn avro_3900_default_name_validator_with_valid_ns() -> TestResult { + validate_schema_name("example")?; + Ok(()) + } - #[test] - fn test_avro_3897_disallow_invalid_namespaces_in_fully_qualified_name() -> TestResult { - let full_name = "ns.0.record1"; - let name = Name::new(full_name); - assert!(name.is_err()); - let validator = SpecificationValidator; - let expected = Details::InvalidSchemaName( - full_name.to_string(), - SchemaNameValidator::regex(&validator).as_str(), - ) - .to_string(); - let err = name.map_err(|e| e.to_string()).err().unwrap(); - pretty_assertions::assert_eq!(expected, err); - - let full_name = "ns..record1"; - let name = Name::new(full_name); - assert!(name.is_err()); - let expected = Details::InvalidSchemaName( - full_name.to_string(), - SchemaNameValidator::regex(&validator).as_str(), - ) - .to_string(); - let err = name.map_err(|e| e.to_string()).err().unwrap(); - pretty_assertions::assert_eq!(expected, err); - Ok(()) - } + #[test] + fn avro_3900_default_name_validator_with_invalid_ns() -> TestResult { + assert!(validate_schema_name("com-example").is_err()); + Ok(()) + } - #[test] - fn avro_3900_default_namespace_validator_with_valid_ns() -> TestResult { - validate_namespace("com.example")?; - Ok(()) - } + #[test] + fn test_avro_3897_disallow_invalid_namespaces_in_fully_qualified_name() -> TestResult { + let full_name = "ns.0.record1"; + let name = Name::new(full_name); + assert!(name.is_err()); + let validator = SpecificationValidator; + let expected = Details::InvalidSchemaName( + full_name.to_string(), + SchemaNameValidator::regex(&validator).as_str(), + ) + .to_string(); + let err = name.map_err(|e| e.to_string()).err().unwrap(); + pretty_assertions::assert_eq!(expected, err); + + let full_name = "ns..record1"; + let name = Name::new(full_name); + assert!(name.is_err()); + let expected = Details::InvalidSchemaName( + full_name.to_string(), + SchemaNameValidator::regex(&validator).as_str(), + ) + .to_string(); + let err = name.map_err(|e| e.to_string()).err().unwrap(); + pretty_assertions::assert_eq!(expected, err); + Ok(()) + } - #[test] - fn avro_3900_default_namespace_validator_with_invalid_ns() -> TestResult { - assert!(validate_namespace("com-example").is_err()); - Ok(()) - } + #[test] + fn avro_3900_default_namespace_validator_with_valid_ns() -> TestResult { + validate_namespace("com.example")?; + Ok(()) + } - #[test] - fn avro_3900_default_enum_symbol_validator_with_valid_symbol_name() -> TestResult { - validate_enum_symbol_name("spades")?; - Ok(()) - } + #[test] + fn avro_3900_default_namespace_validator_with_invalid_ns() -> TestResult { + assert!(validate_namespace("com-example").is_err()); + Ok(()) + } - #[test] - fn avro_3900_default_enum_symbol_validator_with_invalid_symbol_name() -> TestResult { - assert!(validate_enum_symbol_name("com-example").is_err()); - Ok(()) - } + #[test] + fn avro_3900_default_enum_symbol_validator_with_valid_symbol_name() -> TestResult { + validate_enum_symbol_name("spades")?; + Ok(()) + } - #[test] - fn avro_3900_default_record_field_validator_with_valid_name() -> TestResult { - validate_record_field_name("test")?; - Ok(()) - } + #[test] + fn avro_3900_default_enum_symbol_validator_with_invalid_symbol_name() -> TestResult { + assert!(validate_enum_symbol_name("com-example").is_err()); + Ok(()) + } - #[test] - fn avro_3900_default_record_field_validator_with_invalid_name() -> TestResult { - assert!(validate_record_field_name("com-example").is_err()); - Ok(()) - } + #[test] + fn avro_3900_default_record_field_validator_with_valid_name() -> TestResult { + validate_record_field_name("test")?; + Ok(()) + } + + #[test] + fn avro_3900_default_record_field_validator_with_invalid_name() -> TestResult { + assert!(validate_record_field_name("com-example").is_err()); + Ok(()) } } +// } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index abf155c7..deb854ec 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -26,7 +26,6 @@ replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, crate::codec::tokio => crate::codec::sync, - crate::decimal::tokio => crate::decimal::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -36,28 +35,24 @@ crate::reader::tokio => crate::reader::sync, crate::util::tokio => crate::util::sync, crate::types::tokio => crate::types::sync, - crate::schema_equality::tokio => crate::schema_equality::sync, crate::util::tokio => crate::util::sync, - crate::validator::tokio => crate::validator::sync, #[tokio::test] => #[test] ); } )] mod writer { - #[synca::cfg(tokio)] - use crate::AsyncAvroResult as AvroResult; - #[synca::cfg(sync)] use crate::AvroResult; use crate::{ codec::tokio::Codec, encode::tokio::{encode, encode_internal, encode_to_vec}, - error::tokio::Details, - error::tokio::Error, + error::Details, + error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, - schema::tokio::{AvroSchema, Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, + schema::tokio::AvroSchema, + schema::{Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, ser_schema::tokio::SchemaAwareWriteSerializer, - types::tokio::Value, + types::Value, }; use serde::Serialize; use std::{ @@ -440,7 +435,7 @@ mod writer { /// Append a raw Avro Value to the payload avoiding to encode it again. fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult { - self.append_bytes(encode_to_vec(value, schema)?.as_ref()) + self.append_bytes(encode_to_vec(value, schema).await?.as_ref()) } /// Append pure bytes to the payload. @@ -569,7 +564,7 @@ mod writer { { return Err(Details::Validation.into()); } - encode_internal(&avro, schema, names, &enclosing_namespace, buffer) + encode_internal(&avro, schema, names, &enclosing_namespace, buffer).await } /// Writer that encodes messages according to the single object encoding v1 spec @@ -722,13 +717,16 @@ mod writer { reason, } .into()), - None => encode_internal( - value, - schema, - resolved_schema.get_names(), - &schema.namespace(), - buffer, - ), + None => { + encode_internal( + value, + schema, + resolved_schema.get_names(), + &schema.namespace(), + buffer, + ) + .await + } } } @@ -759,7 +757,8 @@ mod writer { resolved_schema.get_names(), &root_schema.namespace(), buffer, - )?; + ) + .await?; Ok(()) } @@ -833,13 +832,14 @@ mod writer { use super::*; use crate::{ - decimal::tokio::Decimal, + decimal::Decimal, duration::{Days, Duration, Millis, Months}, headers::tokio::GlueSchemaUuidHeader, rabin::Rabin, reader::tokio::{Reader, from_avro_datum}, - schema::tokio::{DecimalSchema, FixedSchema, Name}, - types::tokio::Record, + schema::tokio::SchemaExt, + schema::{DecimalSchema, FixedSchema, Name}, + types::Record, util::tokio::zig_i64, }; #[synca::cfg(tokio)] @@ -848,7 +848,7 @@ mod writer { use serde::{Deserialize, Serialize}; use uuid::Uuid; - use crate::{codec::tokio::DeflateSettings, error::tokio::Details}; + use crate::{codec::tokio::DeflateSettings, error::Details}; use apache_avro_test_helper::TestResult; const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); @@ -875,7 +875,7 @@ mod writer { #[tokio::test] async fn test_to_avro_datum() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut record = Record::new(&schema).unwrap(); record.put("a", 27i64); record.put("b", "foo"); @@ -898,7 +898,7 @@ mod writer { b: String, } - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer: Vec = Vec::new(); let data = TestStruct { a: 27, @@ -920,7 +920,7 @@ mod writer { #[tokio::test] async fn avro_rs_220_flush_write_header() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; // By default flush should write the header even if nothing was added yet let mut writer = Writer::new(&schema, Vec::new()); @@ -943,7 +943,7 @@ mod writer { #[tokio::test] async fn test_union_not_null() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA).await?; + let schema = SchemaExt::parse_str(UNION_SCHEMA).await?; let union = Value::Union(1, Box::new(Value::Long(3))); let mut expected = Vec::new(); @@ -957,7 +957,7 @@ mod writer { #[tokio::test] async fn test_union_null() -> TestResult { - let schema = Schema::parse_str(UNION_SCHEMA).await?; + let schema = SchemaExt::parse_str(UNION_SCHEMA).await?; let union = Value::Union(0, Box::new(Value::Null)); let mut expected = Vec::new(); @@ -977,7 +977,7 @@ mod writer { raw_schema: &Schema, raw_value: T, ) -> TestResult { - let schema = Schema::parse_str(schema_str).await?; + let schema = SchemaExt::parse_str(schema_str).await?; assert_eq!(&schema, expected_schema); // The serialized format should be the same as the schema. let ser = to_avro_datum(&schema, value.clone()).await?; @@ -1120,7 +1120,7 @@ mod writer { #[tokio::test] async fn test_writer_append() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut record = Record::new(&schema).unwrap(); @@ -1154,7 +1154,7 @@ mod writer { #[tokio::test] async fn test_writer_extend() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut record = Record::new(&schema).unwrap(); @@ -1195,7 +1195,7 @@ mod writer { #[tokio::test] async fn test_writer_append_ser() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let record = TestSerdeSerialize { @@ -1228,7 +1228,7 @@ mod writer { #[tokio::test] async fn test_writer_extend_ser() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let record = TestSerdeSerialize { @@ -1312,14 +1312,14 @@ mod writer { #[tokio::test] async fn test_writer_with_codec() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let writer = make_writer_with_codec(&schema); check_writer(writer, &schema).await } #[tokio::test] async fn test_writer_with_builder() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let writer = make_writer_with_builder(&schema); check_writer(writer, &schema).await } @@ -1345,7 +1345,7 @@ mod writer { } "#; let codec = Codec::Deflate(DeflateSettings::default()); - let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA).await?; + let schema = SchemaExt::parse_str(LOGICAL_TYPE_SCHEMA).await?; let mut writer = Writer::builder() .schema(&schema) .codec(codec) @@ -1391,7 +1391,7 @@ mod writer { #[tokio::test] async fn test_avro_3405_writer_add_metadata_success() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); writer.add_user_metadata("stringKey".to_string(), String::from("stringValue"))?; @@ -1415,7 +1415,7 @@ mod writer { #[tokio::test] async fn test_avro_3881_metadata_empty_body() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); writer.add_user_metadata("a".to_string(), "b")?; let result = writer.into_inner()?; @@ -1431,7 +1431,7 @@ mod writer { #[tokio::test] async fn test_avro_3405_writer_add_metadata_failure() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let mut record = Record::new(&schema).unwrap(); @@ -1457,7 +1457,7 @@ mod writer { #[tokio::test] async fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let key = "avro.stringKey".to_string(); @@ -1488,7 +1488,7 @@ mod writer { #[tokio::test] async fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult { - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut user_meta_data: HashMap = HashMap::new(); user_meta_data.insert( @@ -1542,7 +1542,7 @@ mod writer { ] } "#; - Schema::parse_str(schema).await.unwrap() + SchemaExt::parse_str(schema).await.unwrap() } } @@ -1690,7 +1690,7 @@ mod writer { time: Some(1234567890), }; - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); let bytes = writer.append_ser(conf)?; @@ -1723,7 +1723,7 @@ mod writer { time: Some(12345678.90), }; - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); match writer.append_ser(conf) { @@ -1773,7 +1773,7 @@ mod writer { let buffered_writer = std::io::BufWriter::new(shared_buffer.clone()); - let schema = Schema::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, buffered_writer); diff --git a/avro/tests/append_to_existing.rs b/avro/tests/append_to_existing.rs index e4993b8c..c087cd3d 100644 --- a/avro/tests/append_to_existing.rs +++ b/avro/tests/append_to_existing.rs @@ -17,7 +17,7 @@ use apache_avro::{ AvroResult, Reader, Schema, Writer, read_marker, - types::sync::{Record, Value}, + types::{Record, Value}, }; use apache_avro_test_helper::TestResult; @@ -31,7 +31,7 @@ const SCHEMA: &str = r#"{ #[test] fn avro_3630_append_to_an_existing_file() -> TestResult { - let schema = Schema::parse_str(SCHEMA).expect("Cannot parse the schema"); + let schema = SchemaExt::parse_str(SCHEMA).expect("Cannot parse the schema"); let bytes = get_avro_bytes(&schema); @@ -57,7 +57,7 @@ fn avro_3630_append_to_an_existing_file() -> TestResult { #[test] fn avro_4031_append_to_file_using_multiple_writers() -> TestResult { - let schema = Schema::parse_str(SCHEMA).expect("Cannot parse the schema"); + let schema = SchemaExt::parse_str(SCHEMA).expect("Cannot parse the schema"); let mut first_writer = Writer::builder().schema(&schema).writer(Vec::new()).build(); first_writer.append(create_datum(&schema, -42))?; diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index 3b159ec6..f63e1dd3 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::schema::sync::Schema; -use apache_avro::types::sync::Value; +use apache_avro::schema::Schema; +use apache_avro::types::Value; use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; @@ -124,7 +124,7 @@ fn avro_3786_deserialize_union_with_different_enum_order() -> TestResult { ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_init: Bar::Bar1, bar_use_parent: Some(BarUseParent { bar_use: Bar::Bar1 }), @@ -136,7 +136,7 @@ fn avro_3786_deserialize_union_with_different_enum_order() -> TestResult { ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -249,7 +249,7 @@ fn avro_3786_deserialize_union_with_different_enum_order_defined_in_record() -> } ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_parent: Some(BarParent { bar: Bar::Bar0 }), }; @@ -260,7 +260,7 @@ fn avro_3786_deserialize_union_with_different_enum_order_defined_in_record() -> ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -362,7 +362,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ } ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_parent: Some(BarParent { bar: Bar::Bar1 }), }; @@ -373,7 +373,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -475,7 +475,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ } ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_parent: Some(BarParent { bar: Bar::Bar1 }), }; @@ -486,7 +486,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -588,7 +588,7 @@ fn deserialize_union_with_different_enum_order_defined_in_record() -> TestResult } ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_parent: Some(BarParent { bar: Bar::Bar2 }), }; @@ -599,7 +599,7 @@ fn deserialize_union_with_different_enum_order_defined_in_record() -> TestResult ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -854,7 +854,7 @@ fn deserialize_union_with_record_with_enum_defined_inline_reader_has_different_i } ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_init: Bar::Bar0, baz: Baz::Baz0, @@ -873,7 +873,7 @@ fn deserialize_union_with_record_with_enum_defined_inline_reader_has_different_i ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index 878fb5ca..4d87b71f 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -16,7 +16,7 @@ // under the License. use apache_avro::Schema; -use apache_avro::types::sync::Value; +use apache_avro::types::Value; use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; @@ -125,7 +125,7 @@ fn avro_3787_deserialize_union_with_unknown_symbol() -> TestResult { ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_init: Bar::Bar1, bar_use_parent: Some(BarUseParent { bar_use: Bar::Bar2 }), @@ -137,7 +137,7 @@ fn avro_3787_deserialize_union_with_unknown_symbol() -> TestResult { ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -253,7 +253,7 @@ fn avro_3787_deserialize_union_with_unknown_symbol_no_ref() -> TestResult { ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo2 = Foo { bar_parent: Some(BarParent { bar: Bar::Bar2 }), }; @@ -264,7 +264,7 @@ fn avro_3787_deserialize_union_with_unknown_symbol_no_ref() -> TestResult { ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { diff --git a/avro/tests/codecs.rs b/avro/tests/codecs.rs index 866eaf86..b4603bc0 100644 --- a/avro/tests/codecs.rs +++ b/avro/tests/codecs.rs @@ -17,7 +17,7 @@ use apache_avro::{ Codec, DeflateSettings, Reader, Schema, Writer, - types::sync::{Record, Value}, + types::{Record, Value}, }; use apache_avro_test_helper::TestResult; use miniz_oxide::deflate::CompressionLevel; @@ -60,7 +60,7 @@ fn avro_4032_zstandard_codec_settings() -> TestResult { } fn avro_4032_codec_settings(codec: Codec) -> TestResult { - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", diff --git a/avro/tests/io.rs b/avro/tests/io.rs index f29d8d9e..c1166506 100644 --- a/avro/tests/io.rs +++ b/avro/tests/io.rs @@ -16,7 +16,7 @@ // under the License. //! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py -use apache_avro::{Error, Schema, error::sync::Details, from_avro_datum, to_avro_datum, types::sync::Value}; +use apache_avro::{Error, Schema, error::Details, from_avro_datum, to_avro_datum, types::Value}; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use std::{io::Cursor, sync::OnceLock}; @@ -178,7 +178,7 @@ fn default_value_examples() -> &'static Vec<(&'static str, &'static str, Value)> fn long_record_schema() -> &'static Schema { static LONG_RECORD_SCHEMA_ONCE: OnceLock = OnceLock::new(); LONG_RECORD_SCHEMA_ONCE.get_or_init(|| { - Schema::parse_str( + SchemaExt::parse_str( r#" { "type": "record", @@ -217,7 +217,7 @@ fn long_record_datum() -> &'static Value { #[test] fn test_validate() -> TestResult { for (raw_schema, value) in schemas_to_validate().iter() { - let schema = Schema::parse_str(raw_schema)?; + let schema = SchemaExt::parse_str(raw_schema)?; assert!( value.validate(&schema), "value {value:?} does not validate schema: {raw_schema}" @@ -230,7 +230,7 @@ fn test_validate() -> TestResult { #[test] fn test_round_trip() -> TestResult { for (raw_schema, value) in schemas_to_validate().iter() { - let schema = Schema::parse_str(raw_schema)?; + let schema = SchemaExt::parse_str(raw_schema)?; let encoded = to_avro_datum(&schema, value.clone()).unwrap(); let decoded = from_avro_datum(&schema, &mut Cursor::new(encoded), None).unwrap(); assert_eq!(value, &decoded); @@ -271,10 +271,10 @@ fn test_schema_promotion() -> TestResult { Value::Double(219.0), ]; for (i, writer_raw_schema) in promotable_schemas.iter().enumerate() { - let writer_schema = Schema::parse_str(writer_raw_schema)?; + let writer_schema = SchemaExt::parse_str(writer_raw_schema)?; let original_value = &promotable_values[i]; for (j, reader_raw_schema) in promotable_schemas.iter().enumerate().skip(i + 1) { - let reader_schema = Schema::parse_str(reader_raw_schema)?; + let reader_schema = SchemaExt::parse_str(reader_raw_schema)?; let encoded = to_avro_datum(&writer_schema, original_value.clone())?; let decoded = from_avro_datum( &writer_schema, @@ -294,9 +294,9 @@ fn test_schema_promotion() -> TestResult { #[test] fn test_unknown_symbol() -> TestResult { let writer_schema = - Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["FOO", "BAR"]}"#)?; + SchemaExt::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["FOO", "BAR"]}"#)?; let reader_schema = - Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["BAR", "BAZ"]}"#)?; + SchemaExt::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["BAR", "BAZ"]}"#)?; let original_value = Value::Enum(0, "FOO".to_string()); let encoded = to_avro_datum(&writer_schema, original_value)?; let decoded = from_avro_datum( @@ -312,7 +312,7 @@ fn test_unknown_symbol() -> TestResult { #[test] fn test_default_value() -> TestResult { for (field_type, default_json, default_datum) in default_value_examples().iter() { - let reader_schema = Schema::parse_str(&format!( + let reader_schema = SchemaExt::parse_str(&format!( r#"{{ "type": "record", "name": "Test", @@ -370,7 +370,7 @@ fn test_default_value() -> TestResult { #[test] fn test_no_default_value() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#"{ "type": "record", "name": "Test", @@ -392,7 +392,7 @@ fn test_no_default_value() -> TestResult { #[test] fn test_projection() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" { "type": "record", @@ -421,7 +421,7 @@ fn test_projection() -> TestResult { #[test] fn test_field_order() -> TestResult { - let reader_schema = Schema::parse_str( + let reader_schema = SchemaExt::parse_str( r#" { "type": "record", @@ -450,7 +450,7 @@ fn test_field_order() -> TestResult { #[test] fn test_type_exception() -> Result<(), String> { - let writer_schema = Schema::parse_str( + let writer_schema = SchemaExt::parse_str( r#" { "type": "record", diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index cb49b270..3e04c7fa 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -22,11 +22,11 @@ use std::{ use apache_avro::{ Codec, Reader, Writer, - error::sync::{Details, Error}, + error::{Details, Error}, from_avro_datum, from_value, - schema::sync::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, + schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, to_avro_datum, to_value, - types::sync::{Record, Value}, + types::{Record, Value}, }; use apache_avro_test_helper::{ TestResult, @@ -56,7 +56,7 @@ fn test_correct_recursive_extraction() -> TestResult { } ] }"#; - let outer_schema = Schema::parse_str(raw_outer_schema)?; + let outer_schema = SchemaExt::parse_str(raw_outer_schema)?; if let Schema::Record(RecordSchema { fields: outer_fields, .. @@ -89,7 +89,7 @@ fn test_correct_recursive_extraction() -> TestResult { fn test_parse() -> TestResult { init(); for (raw_schema, valid) in examples().iter() { - let schema = Schema::parse_str(raw_schema); + let schema = SchemaExt::parse_str(raw_schema); if *valid { assert!( schema.is_ok(), @@ -109,7 +109,7 @@ fn test_parse() -> TestResult { fn test_3799_parse_reader() -> TestResult { init(); for (raw_schema, valid) in examples().iter() { - let schema = Schema::parse_reader(&mut Cursor::new(raw_schema)); + let schema = SchemaExt::parse_reader(&mut Cursor::new(raw_schema)); if *valid { assert!( schema.is_ok(), @@ -126,7 +126,7 @@ fn test_3799_parse_reader() -> TestResult { // Ensure it works for trait objects too. for (raw_schema, valid) in examples().iter() { let reader: &mut dyn Read = &mut Cursor::new(raw_schema); - let schema = Schema::parse_reader(reader); + let schema = SchemaExt::parse_reader(reader); if *valid { assert!( schema.is_ok(), @@ -147,7 +147,7 @@ fn test_3799_raise_io_error_from_parse_read() -> Result<(), String> { // 0xDF is invalid for UTF-8. let mut invalid_data = Cursor::new([0xDF]); - let error = Schema::parse_reader(&mut invalid_data) + let error = SchemaExt::parse_reader(&mut invalid_data) .unwrap_err() .into_details(); @@ -167,8 +167,8 @@ fn test_3799_raise_io_error_from_parse_read() -> Result<(), String> { fn test_valid_cast_to_string_after_parse() -> TestResult { init(); for (raw_schema, _) in valid_examples().iter() { - let schema = Schema::parse_str(raw_schema)?; - Schema::parse_str(schema.canonical_form().as_str())?; + let schema = SchemaExt::parse_str(raw_schema)?; + SchemaExt::parse_str(schema.canonical_form().as_str())?; } Ok(()) } @@ -191,10 +191,10 @@ fn test_parse_list_without_cross_deps() -> TestResult { "size": 16 }"#; let schema_strs = [schema_str_1, schema_str_2]; - let schemas = Schema::parse_list(schema_strs)?; + let schemas = SchemaExt::parse_list(schema_strs)?; for schema_str in &schema_strs { - let parsed = Schema::parse_str(schema_str)?; + let parsed = SchemaExt::parse_str(schema_str)?; assert!(schemas.contains(&parsed)); } Ok(()) @@ -224,8 +224,8 @@ fn test_parse_list_with_cross_deps_basic() -> TestResult { let schema_strs_first = [schema_a_str, schema_b_str]; let schema_strs_second = [schema_b_str, schema_a_str]; - let schemas_first = Schema::parse_list(schema_strs_first)?; - let schemas_second = Schema::parse_list(schema_strs_second)?; + let schemas_first = SchemaExt::parse_list(schema_strs_first)?; + let schemas_second = SchemaExt::parse_list(schema_strs_second)?; assert_eq!(schemas_first[0], schemas_second[1]); assert_eq!(schemas_first[1], schemas_second[0]); @@ -253,8 +253,8 @@ fn test_parse_list_recursive_type() -> TestResult { }"#; let schema_strs_first = [schema_str_1, schema_str_2]; let schema_strs_second = [schema_str_2, schema_str_1]; - let _ = Schema::parse_list(schema_strs_first)?; - let _ = Schema::parse_list(schema_strs_second)?; + let _ = SchemaExt::parse_list(schema_strs_first)?; + let _ = SchemaExt::parse_list(schema_strs_second)?; Ok(()) } @@ -278,8 +278,8 @@ fn test_parse_list_with_cross_deps_and_namespaces() -> TestResult { ] }"#; - let schemas_first = Schema::parse_list([schema_a_str, schema_b_str])?; - let schemas_second = Schema::parse_list([schema_b_str, schema_a_str])?; + let schemas_first = SchemaExt::parse_list([schema_a_str, schema_b_str])?; + let schemas_second = SchemaExt::parse_list([schema_b_str, schema_a_str])?; assert_eq!(schemas_first[0], schemas_second[1]); assert_eq!(schemas_first[1], schemas_second[0]); @@ -309,8 +309,8 @@ fn test_parse_list_with_cross_deps_and_namespaces_error() -> TestResult { let schema_strs_first = [schema_str_1, schema_str_2]; let schema_strs_second = [schema_str_2, schema_str_1]; - let _ = Schema::parse_list(schema_strs_first).expect_err("Test failed"); - let _ = Schema::parse_list(schema_strs_second).expect_err("Test failed"); + let _ = SchemaExt::parse_list(schema_strs_first).expect_err("Test failed"); + let _ = SchemaExt::parse_list(schema_strs_second).expect_err("Test failed"); Ok(()) } @@ -354,7 +354,7 @@ fn test_parse_reused_record_schema_by_fullname() -> TestResult { } "#; - let schema = Schema::parse_str(schema_str); + let schema = SchemaExt::parse_str(schema_str); assert!(schema.is_ok()); match schema? { Schema::Record(RecordSchema { @@ -457,11 +457,11 @@ fn test_parse_list_multiple_dependencies() -> TestResult { ] }"#; - let parsed = Schema::parse_list([schema_a_str, schema_b_str, schema_c_str])?; + let parsed = SchemaExt::parse_list([schema_a_str, schema_b_str, schema_c_str])?; let schema_strs = vec![schema_a_str, schema_b_str, schema_c_str]; for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); - let schemas = Schema::parse_list(&schema_str_perm)?; + let schemas = SchemaExt::parse_list(&schema_str_perm)?; assert_eq!(schemas.len(), 3); for parsed_schema in &parsed { assert!(schemas.contains(parsed_schema)); @@ -497,11 +497,11 @@ fn test_parse_list_shared_dependency() -> TestResult { ] }"#; - let parsed = Schema::parse_list([schema_a_str, schema_b_str, schema_c_str])?; + let parsed = SchemaExt::parse_list([schema_a_str, schema_b_str, schema_c_str])?; let schema_strs = vec![schema_a_str, schema_b_str, schema_c_str]; for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); - let schemas = Schema::parse_list(&schema_str_perm)?; + let schemas = SchemaExt::parse_list(&schema_str_perm)?; assert_eq!(schemas.len(), 3); for parsed_schema in &parsed { assert!(schemas.contains(parsed_schema)); @@ -530,7 +530,7 @@ fn test_name_collision_error() -> TestResult { ] }"#; - let _ = Schema::parse_list([schema_str_1, schema_str_2]).expect_err("Test failed"); + let _ = SchemaExt::parse_list([schema_str_1, schema_str_2]).expect_err("Test failed"); Ok(()) } @@ -554,9 +554,9 @@ fn test_namespace_prevents_collisions() -> TestResult { ] }"#; - let parsed = Schema::parse_list([schema_str_1, schema_str_2])?; - let parsed_1 = Schema::parse_str(schema_str_1)?; - let parsed_2 = Schema::parse_str(schema_str_2)?; + let parsed = SchemaExt::parse_list([schema_str_1, schema_str_2])?; + let parsed_1 = SchemaExt::parse_str(schema_str_1)?; + let parsed_2 = SchemaExt::parse_str(schema_str_2)?; assert_eq!(parsed, vec!(parsed_1, parsed_2)); Ok(()) } @@ -697,7 +697,7 @@ fn test_doc_attributes() -> TestResult { } for (raw_schema, _) in DOC_EXAMPLES.iter() { - let original_schema = Schema::parse_str(raw_schema)?; + let original_schema = SchemaExt::parse_str(raw_schema)?; assert_doc(&original_schema); if let Schema::Record(RecordSchema { fields, .. }) = original_schema { for f in fields { @@ -723,7 +723,7 @@ fn test_other_attributes() { } for (raw_schema, _) in OTHER_ATTRIBUTES_EXAMPLES.iter() { - let schema = Schema::parse_str(raw_schema)?; + let schema = SchemaExt::parse_str(raw_schema)?; // all inputs have at least some user-defined attributes assert!(schema.other_attributes.is_some()); for prop in schema.other_attributes?.iter() { @@ -746,7 +746,7 @@ fn test_other_attributes() { fn test_root_error_is_not_swallowed_on_parse_error() -> Result<(), String> { init(); let raw_schema = r#"/not/a/real/file"#; - let error = Schema::parse_str(raw_schema).unwrap_err().into_details(); + let error = SchemaExt::parse_str(raw_schema).unwrap_err().into_details(); if let Details::ParseSchemaJson(e) = error { assert!( @@ -764,7 +764,7 @@ fn test_root_error_is_not_swallowed_on_parse_error() -> Result<(), String> { #[test] fn test_record_schema_with_cyclic_references() -> TestResult { init(); - let schema = Schema::parse_str( + let schema = SchemaExt::parse_str( r#" { "type": "record", @@ -833,12 +833,12 @@ fn test_record_schema_with_cyclic_references() -> TestResult { #[test] fn test_decimal_valid_type_attributes() { init(); - let fixed_decimal = Schema::parse_str(DECIMAL_LOGICAL_TYPE_ATTRIBUTES[0])?; + let fixed_decimal = SchemaExt::parse_str(DECIMAL_LOGICAL_TYPE_ATTRIBUTES[0])?; assert_eq!(4, fixed_decimal.get_attribute("precision")); assert_eq!(2, fixed_decimal.get_attribute("scale")); assert_eq!(2, fixed_decimal.get_attribute("size")); - let bytes_decimal = Schema::parse_str(DECIMAL_LOGICAL_TYPE_ATTRIBUTES[1])?; + let bytes_decimal = SchemaExt::parse_str(DECIMAL_LOGICAL_TYPE_ATTRIBUTES[1])?; assert_eq!(4, bytes_decimal.get_attribute("precision")); assert_eq!(0, bytes_decimal.get_attribute("scale")); } @@ -857,7 +857,7 @@ fn avro_old_issue_47() -> TestResult { {"name": "b", "type": "string"} ] }"#; - let schema = Schema::parse_str(schema_str)?; + let schema = SchemaExt::parse_str(schema_str)?; use serde::{Deserialize, Serialize}; @@ -988,7 +988,7 @@ fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_ ] }"#; - let writer_schema = Schema::parse_str(writer_schema)?; + let writer_schema = SchemaExt::parse_str(writer_schema)?; let foo1 = Foo { bar_init: Bar::Bar0, bar_use_parent: Some(BarUseParent { bar_use: Bar::Bar1 }), @@ -1000,7 +1000,7 @@ fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_ ); let datum = to_avro_datum(&writer_schema, avro_value)?; let mut x = &datum[..]; - let reader_schema = Schema::parse_str(reader_schema)?; + let reader_schema = SchemaExt::parse_str(reader_schema)?; let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema))?; match deser_value { Value::Record(fields) => { @@ -1036,7 +1036,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); @@ -1069,7 +1069,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1109,7 +1109,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Enum(1, "b".to_string())); @@ -1135,7 +1135,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1169,7 +1169,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Fixed(3, vec![0, 1, 2])); @@ -1195,7 +1195,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1240,7 +1240,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> Test ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); @@ -1274,7 +1274,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> Test ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1315,7 +1315,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> Test ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Enum(1, "b".to_string())); @@ -1342,7 +1342,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> Test ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1377,7 +1377,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> Test ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Fixed(3, vec![0, 1, 2])); @@ -1404,7 +1404,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> Test ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1449,7 +1449,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); @@ -1483,7 +1483,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1524,7 +1524,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Enum(1, "b".to_string())); @@ -1551,7 +1551,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1586,7 +1586,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Fixed(3, vec![0, 1, 2])); @@ -1613,7 +1613,7 @@ fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1647,7 +1647,7 @@ fn write_schema_for_default_value_test() -> apache_avro::AvroResult> { ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()) .ok_or("Expected Some(Record), but got None") @@ -1677,7 +1677,7 @@ fn test_avro_3851_read_default_value_for_simple_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = write_schema_for_default_value_test()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1724,7 +1724,7 @@ fn test_avro_3851_read_default_value_for_nested_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = write_schema_for_default_value_test()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1767,7 +1767,7 @@ fn test_avro_3851_read_default_value_for_enum_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = write_schema_for_default_value_test()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1807,7 +1807,7 @@ fn test_avro_3851_read_default_value_for_fixed_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = write_schema_for_default_value_test()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1844,7 +1844,7 @@ fn test_avro_3851_read_default_value_for_array_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = write_schema_for_default_value_test()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1884,7 +1884,7 @@ fn test_avro_3851_read_default_value_for_map_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = write_schema_for_default_value_test()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1930,7 +1930,7 @@ fn test_avro_3851_read_default_value_for_ref_record_field() -> TestResult { ] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); @@ -1962,7 +1962,7 @@ fn test_avro_3851_read_default_value_for_ref_record_field() -> TestResult { ] } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -1995,7 +1995,7 @@ fn test_avro_3851_read_default_value_for_enum() -> TestResult { "symbols": ["a", "b", "c"] } "#; - let writer_schema = Schema::parse_str(writer_schema_str)?; + let writer_schema = SchemaExt::parse_str(writer_schema_str)?; let mut writer = Writer::new(&writer_schema, Vec::new()); writer.append("c")?; @@ -2008,7 +2008,7 @@ fn test_avro_3851_read_default_value_for_enum() -> TestResult { "default": "a" } "#; - let reader_schema = Schema::parse_str(reader_schema_str)?; + let reader_schema = SchemaExt::parse_str(reader_schema_str)?; let input = writer.into_inner()?; let reader = Reader::with_schema(&reader_schema, &input[..])?; let result = reader.collect::, _>>()?; @@ -2092,7 +2092,7 @@ fn avro_rs_66_test_independent_canonical_form_primitives() -> TestResult { ] }"#; - let independent_schema = Schema::parse_str(record_with_no_dependencies)?; + let independent_schema = SchemaExt::parse_str(record_with_no_dependencies)?; let schema_strs = [ fixed_primitive, enum_primitive, @@ -2102,7 +2102,7 @@ fn avro_rs_66_test_independent_canonical_form_primitives() -> TestResult { for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); - let schemata = Schema::parse_list(&schema_str_perm)?; + let schemata = SchemaExt::parse_list(&schema_str_perm)?; assert_eq!(schemata.len(), schema_strs.len()); let test_schema = schemata .iter() @@ -2220,31 +2220,31 @@ fn avro_rs_66_test_independent_canonical_form_usages() -> TestResult { for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); - let schemata = Schema::parse_list(&schema_str_perm)?; + let schemata = SchemaExt::parse_list(&schema_str_perm)?; for schema in &schemata { match schema.name().unwrap().to_string().as_str() { "RecUsage" => { assert_eq!( schema.independent_canonical_form(&schemata)?, - Schema::parse_str(record_usage_independent)?.canonical_form() + SchemaExt::parse_str(record_usage_independent)?.canonical_form() ); } "ArrayUsage" => { assert_eq!( schema.independent_canonical_form(&schemata)?, - Schema::parse_str(array_usage_independent)?.canonical_form() + SchemaExt::parse_str(array_usage_independent)?.canonical_form() ); } "UnionUsage" => { assert_eq!( schema.independent_canonical_form(&schemata)?, - Schema::parse_str(union_usage_independent)?.canonical_form() + SchemaExt::parse_str(union_usage_independent)?.canonical_form() ); } "MapUsage" => { assert_eq!( schema.independent_canonical_form(&schemata)?, - Schema::parse_str(map_usage_independent)?.canonical_form() + SchemaExt::parse_str(map_usage_independent)?.canonical_form() ); } "ns.Rec" => { @@ -2315,14 +2315,14 @@ fn avro_rs_66_test_independent_canonical_form_deep_recursion() -> TestResult { for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); - let schemata = Schema::parse_list(&schema_str_perm)?; + let schemata = SchemaExt::parse_list(&schema_str_perm)?; let ruu = schemata .iter() .find(|s| s.name().unwrap().to_string().as_str() == "RecUsageUsage") .unwrap(); assert_eq!( ruu.independent_canonical_form(&schemata)?, - Schema::parse_str(record_usage_usage_independent)?.canonical_form() + SchemaExt::parse_str(record_usage_usage_independent)?.canonical_form() ); } Ok(()) @@ -2349,7 +2349,7 @@ fn avro_rs_66_test_independent_canonical_form_missing_ref() -> TestResult { }"#; let schema_strs = [record_primitive, record_usage]; - let schemata = Schema::parse_list(schema_strs)?; + let schemata = SchemaExt::parse_list(schema_strs)?; assert!(matches!( schemata[1] .independent_canonical_form(&Vec::with_capacity(0)) @@ -2362,7 +2362,7 @@ fn avro_rs_66_test_independent_canonical_form_missing_ref() -> TestResult { #[test] fn avro_rs_181_single_null_record() -> TestResult { let mut buff = Cursor::new(Vec::new()); - let schema = Schema::parse_str(r#""null""#)?; + let schema = SchemaExt::parse_str(r#""null""#)?; let mut writer = Writer::new(&schema, &mut buff); writer.append(serde_json::Value::Null)?; writer.into_inner()?; diff --git a/avro/tests/shared.rs b/avro/tests/shared.rs index c7954a8d..5d598a1d 100644 --- a/avro/tests/shared.rs +++ b/avro/tests/shared.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, Reader, Schema, Writer, types::sync::Value}; +use apache_avro::{Codec, Reader, Schema, Writer, types::Value}; use apache_avro_test_helper::TestResult; use std::{ fmt, @@ -100,7 +100,7 @@ fn test_folder(folder: &str) -> Result<(), ErrorsDesc> { let file_name = folder.to_owned() + "/schema.json"; let content = std::fs::read_to_string(file_name).expect("Unable to find schema.json file"); - let schema: Schema = Schema::parse_str(content.as_str()).expect("Can't read schema"); + let schema: Schema = SchemaExt::parse_str(content.as_str()).expect("Can't read schema"); let data_file_name = folder.to_owned() + "/data.avro"; let data_path: &Path = Path::new(data_file_name.as_str()); diff --git a/avro/tests/to_from_avro_datum_schemata.rs b/avro/tests/to_from_avro_datum_schemata.rs index 7ab6f217..546ecd0c 100644 --- a/avro/tests/to_from_avro_datum_schemata.rs +++ b/avro/tests/to_from_avro_datum_schemata.rs @@ -17,7 +17,7 @@ use apache_avro::{ Codec, Reader, Schema, Writer, from_avro_datum_reader_schemata, from_avro_datum_schemata, - to_avro_datum_schemata, types::sync::Value, + to_avro_datum_schemata, types::Value, }; use apache_avro_test_helper::{TestResult, init}; @@ -46,7 +46,7 @@ fn test_avro_3683_multiple_schemata_to_from_avro_datum() -> TestResult { Value::Record(vec![(String::from("field_a"), Value::Float(1.0))]), )]); - let schemata: Vec = Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR])?; + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR])?; let schemata: Vec<&Schema> = schemata.iter().collect(); // this is the Schema we want to use for write/read @@ -70,7 +70,7 @@ fn avro_rs_106_test_multiple_schemata_to_from_avro_datum_with_resolution() -> Te Value::Record(vec![(String::from("field_a"), Value::Float(1.0))]), )]); - let schemata: Vec = Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR])?; + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR])?; let schemata: Vec<&Schema> = schemata.iter().collect(); // this is the Schema we want to use for write/read @@ -100,7 +100,7 @@ fn test_avro_3683_multiple_schemata_writer_reader() -> TestResult { Value::Record(vec![(String::from("field_a"), Value::Float(1.0))]), )]); - let schemata: Vec = Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR])?; + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR])?; let schemata: Vec<&Schema> = schemata.iter().collect(); // this is the Schema we want to use for write/read diff --git a/avro/tests/union_schema.rs b/avro/tests/union_schema.rs index b9c22ff5..e3401f94 100644 --- a/avro/tests/union_schema.rs +++ b/avro/tests/union_schema.rs @@ -82,8 +82,8 @@ where #[test] fn test_avro_3901_union_schema_round_trip_no_null() -> AvroResult<()> { - let schemata: Vec = - Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_C_STR]).expect("parsing schemata"); + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_C_STR]) + .expect("parsing schemata"); let input = C { field_union: (UnionAB::A(A { field_a: 45.5 })), @@ -126,8 +126,8 @@ struct D { #[test] fn test_avro_3901_union_schema_round_trip_null_at_start() -> AvroResult<()> { - let schemata: Vec = - Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_D_STR]).expect("parsing schemata"); + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_D_STR]) + .expect("parsing schemata"); let input = D { field_union: UnionNoneAB::A(A { field_a: 54.25 }), @@ -177,8 +177,8 @@ struct E { #[test] fn test_avro_3901_union_schema_round_trip_with_out_of_order_null() -> AvroResult<()> { - let schemata: Vec = - Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_E_STR]).expect("parsing schemata"); + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_E_STR]) + .expect("parsing schemata"); let input = E { field_union: UnionANoneB::A(A { field_a: 23.75 }), @@ -228,8 +228,8 @@ struct F { #[test] fn test_avro_3901_union_schema_round_trip_with_end_null() -> AvroResult<()> { - let schemata: Vec = - Schema::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_F_STR]).expect("parsing schemata"); + let schemata: Vec = SchemaExt::parse_list([SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_F_STR]) + .expect("parsing schemata"); let input = F { field_union: UnionABNone::A(A { field_a: 23.75 }), @@ -276,7 +276,7 @@ struct G { #[test] fn test_avro_3901_union_schema_as_optional_2() -> AvroResult<()> { let schemata: Vec = - Schema::parse_list(&[SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_G_STR]).expect("parsing schemata"); + SchemaExt::parse_list(&[SCHEMA_A_STR, SCHEMA_B_STR, SCHEMA_G_STR]).expect("parsing schemata"); let input = G { field_union: Some(UnionAB::A(A { field_a: 32.25 })), @@ -319,7 +319,7 @@ struct H { #[test] fn test_avro_3901_union_schema_as_optional() -> AvroResult<()> { - let schemata: Vec = Schema::parse_list([SCHEMA_H_STR]).expect("parsing schemata"); + let schemata: Vec = SchemaExt::parse_list([SCHEMA_H_STR]).expect("parsing schemata"); let input = H { field_union: Some(23), diff --git a/avro/tests/validators.rs b/avro/tests/validators.rs index cd24d24d..8bfd738f 100644 --- a/avro/tests/validators.rs +++ b/avro/tests/validators.rs @@ -17,8 +17,8 @@ use apache_avro::{ AvroResult, - schema::sync::Namespace, - validator::sync::{ + schema::Namespace, + validator::{ EnumSymbolNameValidator, RecordFieldNameValidator, SchemaNameValidator, SchemaNamespaceValidator, set_enum_symbol_name_validator, set_record_field_name_validator, set_schema_name_validator, set_schema_namespace_validator, @@ -79,7 +79,7 @@ fn avro_3900_custom_validator_with_spec_invalid_names() -> TestResult { ] }"#; - apache_avro::Schema::parse_str(invalid_schema)?; + apache_avro::SchemaExt::parse_str(invalid_schema)?; Ok(()) } diff --git a/avro_derive/tests/derive.rs b/avro_derive/tests/derive.rs index 6ac95570..0730bf3e 100644 --- a/avro_derive/tests/derive.rs +++ b/avro_derive/tests/derive.rs @@ -102,7 +102,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestBasic::get_schema()); let test = TestBasic { a, @@ -136,7 +136,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestBasicNamespace::get_schema()); if let Schema::Record(RecordSchema { name, .. }) = TestBasicNamespace::get_schema() { assert_eq!("com.testing.namespace".to_owned(), name.namespace.unwrap()) @@ -183,7 +183,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestComplexNamespace::get_schema()); if let Schema::Record(RecordSchema { name, fields, .. }) = TestComplexNamespace::get_schema() @@ -235,7 +235,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestNamedRecord::get_schema()); if let Schema::Record(RecordSchema { name, .. }) = TestNamedRecord::get_schema() { assert_eq!("Other", name.name.as_str()); @@ -311,7 +311,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestAllSupportedBaseTypes::get_schema()); let all_basic = TestAllSupportedBaseTypes { a, @@ -398,7 +398,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestNested::get_schema()); let all_basic = TestAllSupportedBaseTypes { a, @@ -439,7 +439,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestOptional::get_schema()); let optional_field = TestOptional { a: Some(a) }; serde_assert(optional_field); @@ -488,7 +488,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestGeneric::::get_schema()); let test_generic = TestGeneric:: { a, @@ -572,7 +572,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!( schema, TestGeneric::::get_schema() @@ -648,7 +648,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestAllowedEnumNested::get_schema()); let enum_included = TestAllowedEnumNested { a: TestAllowedEnum::B, @@ -697,7 +697,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestNamedEnumNested::get_schema()); let enum_included = TestNamedEnumNested { a: TestNamedEnum::B, @@ -730,7 +730,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, ConsList::get_schema()); let list = ConsList { value: 34, @@ -783,7 +783,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!( schema, ConsListGeneric::::get_schema() @@ -827,7 +827,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestSimpleArray::get_schema()); let test = TestSimpleArray { a }; serde_assert(test) @@ -868,7 +868,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestComplexArray::::get_schema()); let test = TestComplexArray:: { a: [ @@ -934,7 +934,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestSmartPointers::get_schema()); let test = TestSmartPointers { a: "hey".into(), @@ -981,7 +981,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); assert_eq!(schema, TestReference::get_schema()); // let a = vec![34]; // let c = 4.55555555_f64; @@ -1023,7 +1023,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Record(RecordSchema { name, doc, .. }) = TestBasicWithAttributes::get_schema() { @@ -1066,7 +1066,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); let derived_schema = TestBasicWithOuterDocAttributes::get_schema(); assert_eq!(&schema, &derived_schema); if let Schema::Record(RecordSchema { name, doc, .. }) = derived_schema { @@ -1110,7 +1110,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Record(RecordSchema { name, doc, .. }) = TestBasicWithLargeDoc::get_schema() { assert_eq!("com.testing.namespace".to_owned(), name.namespace.unwrap()); @@ -1149,7 +1149,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); let derived_schema = TestBasicWithBool::get_schema(); if let Schema::Record(RecordSchema { name, .. }) = derived_schema { @@ -1182,7 +1182,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Record(RecordSchema { name, .. }) = TestBasicWithU32::get_schema() { assert_eq!("TestBasicWithU32", name.fullname(None)) } else { @@ -1214,7 +1214,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Record(RecordSchema { name, aliases, .. }) = TestBasicStructWithAliases::get_schema() { @@ -1258,7 +1258,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Record(RecordSchema { name, aliases, .. }) = TestBasicStructWithAliases2::get_schema() { @@ -1299,7 +1299,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Enum(EnumSchema { name, aliases, .. }) = TestBasicEnumWithAliases::get_schema() { @@ -1342,7 +1342,7 @@ mod test_derive { ] } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Enum(EnumSchema { name, aliases, .. }) = TestBasicEnumWithAliases2::get_schema() { @@ -1448,7 +1448,7 @@ mod test_derive { } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); if let Schema::Record(RecordSchema { name, fields, .. }) = TestBasicStructWithDefaultValues::get_schema() { @@ -1545,7 +1545,7 @@ mod test_derive { } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); let derived_schema = TestBasicStructWithSkipAttribute::get_schema(); if let Schema::Record(RecordSchema { name, fields, .. }) = &derived_schema { assert_eq!("TestBasicStructWithSkipAttribute", name.fullname(None)); @@ -1612,7 +1612,7 @@ mod test_derive { } "#; - let schema = Schema::parse_str(schema).unwrap(); + let schema = SchemaExt::parse_str(schema).unwrap(); let derived_schema = TestBasicStructWithRenameAttribute::get_schema(); if let Schema::Record(RecordSchema { name, fields, .. }) = &derived_schema { assert_eq!("TestBasicStructWithRenameAttribute", name.fullname(None)); diff --git a/wasm-demo/tests/demos.rs b/wasm-demo/tests/demos.rs index 30bbe9c3..050bf8be 100644 --- a/wasm-demo/tests/demos.rs +++ b/wasm-demo/tests/demos.rs @@ -58,7 +58,7 @@ fn write_read() { {"name": "b", "type": "string"} ] }"#; - let schema = Schema::parse_str(schema_str).unwrap(); + let schema = SchemaExt::parse_str(schema_str).unwrap(); let mut record = Record::new(&schema).unwrap(); record.put("a", 12_i32); From e24d2158092b11738358dde6f57bc143c8c7f2f8 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 12 Aug 2025 22:26:50 +0300 Subject: [PATCH 20/47] WIP: More async for encode.rs & writer.rs Revert async for ser_schema.rs Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 14 +- avro/src/bytes.rs | 2 +- avro/src/decode.rs | 29 +-- avro/src/encode.rs | 46 ++-- avro/src/reader.rs | 16 +- avro/src/schema.rs | 31 ++- avro/src/ser_schema.rs | 94 +++----- avro/src/types.rs | 509 +++++++++++++++++++---------------------- avro/src/util.rs | 64 +++--- avro/src/writer.rs | 209 +++++++++-------- 10 files changed, 484 insertions(+), 530 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 1ab48739..83a309f0 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -53,19 +53,19 @@ mod bigdecimal { let mut buffer: Vec = Vec::new(); let (big_int, exponent): (BigInt, i64) = decimal.as_bigint_and_exponent(); let big_endian_value: Vec = big_int.to_signed_bytes_be(); - encode_bytes(&big_endian_value, &mut buffer)?; - encode_long(exponent, &mut buffer)?; + encode_bytes(&big_endian_value, &mut buffer).await?; + encode_long(exponent, &mut buffer).await?; Ok(buffer) } pub(crate) async fn serialize_big_decimal(decimal: &BigDecimal) -> AvroResult> { // encode big decimal, without global size - let buffer = big_decimal_as_bytes(decimal)?; + let buffer = big_decimal_as_bytes(decimal).await?; // encode global size and content let mut final_buffer: Vec = Vec::new(); - encode_bytes(&buffer, &mut final_buffer)?; + encode_bytes(&buffer, &mut final_buffer).await?; Ok(final_buffer) } @@ -85,7 +85,7 @@ mod bigdecimal { use tokio::io::AsyncReadExt; bytes - .read_exact(&mut big_decimal_buffer[..]) + .read_exact(&mut big_decimal_buffer[..]).await .map_err(Details::ReadDouble)?; match decode_long(&mut bytes).await { @@ -130,7 +130,7 @@ mod bigdecimal { let mut current: BigDecimal = BigDecimal::one(); for iter in 1..180 { - let buffer: Vec = serialize_big_decimal(¤t)?; + let buffer: Vec = serialize_big_decimal(¤t).await?; let mut as_slice = buffer.as_slice(); decode_long(&mut as_slice).await?; @@ -148,7 +148,7 @@ mod bigdecimal { current = current.mul(&value); } - let buffer: Vec = serialize_big_decimal(&BigDecimal::zero())?; + let buffer: Vec = serialize_big_decimal(&BigDecimal::zero()).await?; let mut as_slice = buffer.as_slice(); decode_long(&mut as_slice).await?; diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 0e9af644..d10e36db 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -375,7 +375,7 @@ mod tests { }"#, ) .await?; - assert!(value.validate(&schema).await); + assert!(ValueExt::validate(&value, &schema).await); Ok(()) } diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 46d64906..5b9db3d2 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -399,6 +399,7 @@ mod decode { schema::{DecimalSchema, FixedSchema, Name, Schema}, types::Value, }; + use crate::schema::tokio::SchemaExt; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use std::collections::HashMap; @@ -468,7 +469,7 @@ mod decode { let value = Value::Decimal(Decimal::from(bigint.to_signed_bytes_be())); let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); let mut bytes = &buffer[..]; let result = decode(&schema, &mut bytes).await?; @@ -498,7 +499,7 @@ mod decode { )); let mut buffer = Vec::::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); let mut bytes: &[u8] = &buffer[..]; let result = decode(&schema, &mut bytes).await?; assert_eq!(result, value); @@ -542,7 +543,7 @@ mod decode { ("b".into(), inner_value2.clone()), ]); let mut buf = Vec::new(); - encode(&outer_value1, &schema, &mut buf).expect(&success(&outer_value1, &schema)); + encode(&outer_value1, &schema, &mut buf).await.expect(&success(&outer_value1, &schema)); assert!(!buf.is_empty()); let mut bytes = &buf[..]; assert_eq!( @@ -558,7 +559,7 @@ mod decode { ("a".into(), Value::Union(0, Box::new(Value::Null))), ("b".into(), inner_value2), ]); - encode(&outer_value2, &schema, &mut buf).expect(&success(&outer_value2, &schema)); + encode(&outer_value2, &schema, &mut buf).await.expect(&success(&outer_value2, &schema)); let mut bytes = &buf[..]; assert_eq!( outer_value2, @@ -609,7 +610,7 @@ mod decode { ("b".into(), inner_value2), ]); let mut buf = Vec::new(); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf).await.expect(&success(&outer_value, &schema)); let mut bytes = &buf[..]; assert_eq!( outer_value, @@ -663,7 +664,7 @@ mod decode { ("b".into(), inner_value2), ]); let mut buf = Vec::new(); - encode(&outer_value, &schema, &mut buf).expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf).await.expect(&success(&outer_value, &schema)); let mut bytes = &buf[..]; assert_eq!( outer_value, @@ -754,7 +755,7 @@ mod decode { ]); let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) + encode(&outer_record_variation_1, &schema, &mut buf).await .expect(&success(&outer_record_variation_1, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -766,7 +767,7 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_2, &schema, &mut buf) + encode(&outer_record_variation_2, &schema, &mut buf).await .expect(&success(&outer_record_variation_2, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -778,7 +779,7 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_3, &schema, &mut buf) + encode(&outer_record_variation_3, &schema, &mut buf).await .expect(&success(&outer_record_variation_3, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -871,7 +872,7 @@ mod decode { ]); let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf) + encode(&outer_record_variation_1, &schema, &mut buf).await .expect(&success(&outer_record_variation_1, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -883,7 +884,7 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_2, &schema, &mut buf) + encode(&outer_record_variation_2, &schema, &mut buf).await .expect(&success(&outer_record_variation_2, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -895,7 +896,7 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_3, &schema, &mut buf) + encode(&outer_record_variation_3, &schema, &mut buf).await .expect(&success(&outer_record_variation_3, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -915,7 +916,7 @@ mod decode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); let result = decode(&Schema::Uuid, &mut &buffer[..]).await?; assert_eq!(result, value); @@ -936,7 +937,7 @@ mod decode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); let result = decode(&Schema::Uuid, &mut &buffer[..]).await?; assert_eq!(result, value); diff --git a/avro/src/encode.rs b/avro/src/encode.rs index d44a660e..7f7639b3 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -41,7 +41,6 @@ mod encode { use crate::{ bigdecimal::tokio::serialize_big_decimal, error::Details, - schema::tokio::SchemaExt, schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, SchemaKind, UnionSchema, @@ -50,6 +49,7 @@ mod encode { }; #[synca::cfg(sync)] use std::io::Write as AvroWrite; + use std::marker::Unpin; #[synca::cfg(tokio)] use tokio::io::AsyncWrite as AvroWrite; #[cfg(feature = "tokio")] @@ -63,7 +63,7 @@ mod encode { /// **NOTE** This will not perform schema validation. The value is assumed to /// be valid with regards to the schema. Schema are needed only to guide the /// encoding for complex type values. - pub async fn encode( + pub async fn encode( value: &Value, schema: &Schema, writer: &mut W, @@ -72,27 +72,27 @@ mod encode { encode_internal(value, schema, rs.get_names(), &None, writer).await } - pub(crate) async fn encode_bytes + ?Sized, W: AvroWrite>( + pub(crate) async fn encode_bytes + ?Sized, W: AvroWrite + Unpin>( s: &B, mut writer: W, ) -> AvroResult { let bytes = s.as_ref(); - encode_long(bytes.len() as i64, &mut writer)?; + encode_long(bytes.len() as i64, &mut writer).await?; writer .write(bytes) .await .map_err(|e| Details::WriteBytes(e).into()) } - pub(crate) async fn encode_long(i: i64, writer: W) -> AvroResult { + pub(crate) async fn encode_long(i: i64, writer: W) -> AvroResult { zig_i64(i, writer).await } - pub(crate) async fn encode_int(i: i32, writer: W) -> AvroResult { + pub(crate) async fn encode_int(i: i32, writer: W) -> AvroResult { zig_i32(i, writer).await } - pub(crate) async fn encode_internal>( + pub(crate) async fn encode_internal>( value: &Value, schema: &Schema, names: &HashMap, @@ -130,7 +130,7 @@ mod encode { } } Value::Boolean(b) => writer - .write(&[u8::from(*b)]) + .write(&[u8::from(*b)]).await .map_err(|e| Details::WriteBytes(e).into()), // Pattern | Pattern here to signify that these _must_ have the same encoding. Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => encode_int(*i, writer).await, @@ -143,10 +143,10 @@ mod encode { | Value::LocalTimestampNanos(i) | Value::TimeMicros(i) => encode_long(*i, writer).await, Value::Float(x) => writer - .write(&x.to_le_bytes()) + .write(&x.to_le_bytes()).await .map_err(|e| Details::WriteBytes(e).into()), Value::Double(x) => writer - .write(&x.to_le_bytes()) + .write(&x.to_le_bytes()).await .map_err(|e| Details::WriteBytes(e).into()), Value::Decimal(decimal) => match schema { Schema::Decimal(DecimalSchema { inner, .. }) => match *inner.clone() { @@ -156,9 +156,9 @@ mod encode { if num_bytes != size { return Err(Details::EncodeDecimalAsFixedError(num_bytes, size).into()); } - encode(&Value::Fixed(size, bytes), inner, writer) + encode(&Value::Fixed(size, bytes), inner, writer).await } - Schema::Bytes => encode(&Value::Bytes(decimal.try_into()?), inner, writer), + Schema::Bytes => encode(&Value::Bytes(decimal.try_into()?), inner, writer).await, _ => { Err(Details::ResolveDecimalSchema(SchemaKind::from(*inner.clone())).into()) } @@ -172,7 +172,7 @@ mod encode { &Value::Duration(duration) => { let slice: [u8; 12] = duration.into(); writer - .write(&slice) + .write(&slice).await .map_err(|e| Details::WriteBytes(e).into()) } Value::Uuid(uuid) => match *schema { @@ -200,15 +200,15 @@ mod encode { .into()), }, Value::BigDecimal(bg) => { - let buf: Vec = serialize_big_decimal(bg)?; + let buf: Vec = serialize_big_decimal(bg).await?; writer - .write(buf.as_slice()) + .write(buf.as_slice()).await .map_err(|e| Details::WriteBytes(e).into()) } Value::Bytes(bytes) => match *schema { Schema::Bytes => encode_bytes(bytes, writer).await, Schema::Fixed { .. } => writer - .write(bytes.as_slice()) + .write(bytes.as_slice()).await .map_err(|e| Details::WriteBytes(e).into()), _ => Err(Details::EncodeValueAsSchemaError { value_kind: ValueKind::Bytes, @@ -233,7 +233,7 @@ mod encode { .into()), }, Value::Fixed(_, bytes) => writer - .write(bytes.as_slice()) + .write(bytes.as_slice()).await .map_err(|e| Details::WriteBytes(e).into()), Value::Enum(i, _) => encode_int(*i as i32, writer).await, Value::Union(idx, item) => { @@ -363,14 +363,14 @@ mod encode { } else if let Schema::Union(UnionSchema { schemas, .. }) = schema { let mut union_buffer: Vec = Vec::new(); for (index, schema) in schemas.iter().enumerate() { - encode_long(index as i64, &mut union_buffer)?; - let encode_res = encode_internal( + encode_long(index as i64, &mut union_buffer).await?; + let encode_res = Box::pin(encode_internal( value, schema, names, enclosing_namespace, &mut union_buffer, - ); + )).await; match encode_res { Ok(_) => { return writer @@ -411,6 +411,7 @@ mod encode { pub(crate) mod tests { use super::*; use crate::error::{Details, Error}; + use crate::schema::tokio::SchemaExt; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use uuid::Uuid; @@ -1029,7 +1030,7 @@ mod encode { let value = Value::String(String::from("00000000-0000-0000-0000-000000000000")); let schema = Schema::Uuid; let mut buffer = Vec::new(); - let encoded = encode(&value, &schema, &mut buffer); + let encoded = encode(&value, &schema, &mut buffer).await; assert!(encoded.is_ok()); assert!(!buffer.is_empty()); } @@ -1047,9 +1048,8 @@ mod encode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - match encode(&value, &schema, &mut buffer) + match encode(&value, &schema, &mut buffer).await .map_err(Error::into_details) - .await { Err(Details::ConvertFixedToUuid(actual)) => { assert_eq!(actual, 15); diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 6d5ffb97..fdcbda6c 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -63,7 +63,7 @@ mod reader { Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, resolve_names_with_schemata, }, - types::Value, + types::{Value, tokio::ValueExt}, util::tokio::read_long, }; use log::warn; @@ -242,7 +242,7 @@ mod reader { ) .await?; let item = match read_schema { - Some(schema) => item.resolve(schema).await?, + Some(schema) => ValueExt::resolve(item, schema).await?, None => item, }; @@ -547,7 +547,7 @@ mod reader { ) -> AvroResult { let value = decode(writer_schema, reader).await?; match reader_schema { - Some(schema) => value.resolve(schema).await, + Some(schema) => ValueExt::resolve(value, schema).await, None => Ok(value), } } @@ -592,9 +592,9 @@ mod reader { match reader_schema { Some(schema) => { if reader_schemata.is_empty() { - value.resolve(schema).await + ValueExt::resolve(value, schema).await } else { - value.resolve_schemata(schema, reader_schemata).await + ValueExt::resolve_schemata(value, schema, reader_schemata).await } } None => Ok(value), @@ -1052,7 +1052,7 @@ mod reader { &obj.clone().into(), &TestSingleObjectReader::get_schema().await, &mut to_read, - ) + ).await .expect("Encode should succeed"); let mut to_read = &to_read[..]; let generic_reader = @@ -1089,7 +1089,7 @@ mod reader { &obj.clone().into(), &TestSingleObjectReader::get_schema().await, &mut to_read_3, - ) + ).await .expect("Encode should succeed"); let mut to_read = (&to_read_1[..]).chain(&to_read_2[..]).chain(&to_read_3[..]); let generic_reader = @@ -1125,7 +1125,7 @@ mod reader { &obj.clone().into(), &TestSingleObjectReader::get_schema().await, &mut to_read, - ) + ).await .expect("Encode should succeed"); let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 4c7d8e84..d372457b 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -854,7 +854,7 @@ impl Schema { fn pcf_array(arr: &[serde_json::Value], defined_names: &mut HashSet) -> String { let inter = arr .iter() - .map(|a| parsing_canonical_form(a, defined_names)) + .map(|a| Self::parsing_canonical_form(a, defined_names)) .collect::>() .join(","); format!("[{inter}]") @@ -1377,7 +1377,6 @@ mod schema { validate_schema_name, }, }; - use digest::Digest; use log::{debug, error, warn}; use serde::{ Deserialize, Serialize, Serializer, @@ -1389,13 +1388,13 @@ mod schema { collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, hash::Hash, - io::Read, str::FromStr, }; #[synca::cfg(tokio)] use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; + use crate::types::tokio::ValueExt; pub struct RecordFieldExt; @@ -1473,9 +1472,8 @@ mod schema { let mut resolved = false; for schema in schemas { - if avro_value - .to_owned() - .resolve_internal(schema, names, &schema.namespace(), &None) + if ValueExt::resolve_internal(&avro_value + .to_owned(), schema, names, &schema.namespace(), &None) .await .is_ok() { @@ -1497,8 +1495,7 @@ mod schema { } } _ => { - let resolved = avro_value - .resolve_internal(field_schema, names, &field_schema.namespace(), &None) + let resolved = ValueExt::resolve_internal(&avro_value, field_schema, names, &field_schema.namespace(), &None) .await .is_ok(); @@ -1584,9 +1581,7 @@ mod schema { collected_names.extend(resolved_names.clone()); let namespace = &schema.namespace().or_else(|| enclosing_namespace.clone()); - if value - .clone() - .resolve_internal(schema, &collected_names, namespace, &None) + if ValueExt::resolve_internal(&value, schema, &collected_names, namespace, &None) .await .is_ok() { @@ -1742,7 +1737,7 @@ mod schema { /// Create a `Schema` from a reader which implements [`Read`]. pub async fn parse_reader( - reader: &mut (impl AvroRead + ?Sized), + reader: &mut (impl AvroRead + ?Sized + Unpin), ) -> AvroResult { let mut buf = String::new(); match reader.read_to_string(&mut buf).await { @@ -2384,8 +2379,8 @@ mod schema { } if let Some(ref value) = default { - let resolved = Value::from(value.clone()) - .resolve_enum(&symbols, &Some(value.to_string()), &None) + let value = Value::from(value.clone()); + let resolved = ValueExt::resolve_enum(&value, &symbols, &Some(value.to_string()), &None) .is_ok(); if !resolved { return Err(Details::GetEnumDefault { @@ -5439,7 +5434,7 @@ mod schema { }; let avro_value = to_value(foo)?; - assert!(avro_value.validate(&schema).await); + assert!(ValueExt::validate(&avro_value, &schema).await); let mut writer = Writer::new(&schema, Vec::new()); @@ -5538,7 +5533,7 @@ mod schema { }; let avro_value = to_value(foo)?; assert!( - avro_value.validate(&writer_schema).await, + ValueExt::validate(&avro_value, &writer_schema).await, "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value).await?; @@ -6125,7 +6120,7 @@ mod schema { let writer_schema = SchemaExt::parse(&writer_schema).await?; let avro_value = to_value(s)?; assert!( - avro_value.validate(&writer_schema).await, + ValueExt::validate(&avro_value, &writer_schema).await, "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value).await?; @@ -6136,7 +6131,7 @@ mod schema { // Deserialization should succeed and we should be able to resolve the schema. let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema)).await?; - assert!(deser_value.validate(&reader_schema).await); + assert!(ValueExt::validate(&deser_value, &reader_schema).await); // Verify that we can read a field from the record. let d: MyRecordReader = from_value(&deser_value)?; diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 98725f51..416132f8 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -18,27 +18,6 @@ //! Logic for serde-compatible schema-aware serialization //! which writes directly to a `Write` stream -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::util::tokio => crate::util::sync, - #[tokio::test] => #[test] - ); - } -)] -mod bigdecimal { - use crate::{ bigdecimal::tokio::big_decimal_as_bytes, encode::tokio::{encode_int, encode_long}, @@ -2031,7 +2010,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_record() -> TestResult { + fn test_serialize_record() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "record", @@ -2042,7 +2021,7 @@ mod bigdecimal { ] }"#, ) - .await?; + ?; #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -2084,7 +2063,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_empty_record() -> TestResult { + fn test_serialize_empty_record() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "record", @@ -2092,7 +2071,7 @@ mod bigdecimal { "fields": [] }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2137,7 +2116,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_enum() -> TestResult { + fn test_serialize_enum() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "enum", @@ -2145,7 +2124,7 @@ mod bigdecimal { "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] }"#, ) - .await?; + ?; #[derive(Serialize)] enum Suit { @@ -2186,14 +2165,14 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_array() -> TestResult { + fn test_serialize_array() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "array", "items": "long" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2225,14 +2204,14 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_map() -> TestResult { + fn test_serialize_map() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "map", "values": "long" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2272,13 +2251,13 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_nullable_union() -> TestResult { + fn test_serialize_nullable_union() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": ["null", "long"] }"#, ) - .await?; + ?; #[derive(Serialize)] enum NullableLong { @@ -2321,13 +2300,13 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_union() -> TestResult { + fn test_serialize_union() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": ["null", "long", "string"] }"#, ) - .await?; + ?; #[derive(Serialize)] enum LongOrString { @@ -2373,7 +2352,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_fixed() -> TestResult { + fn test_serialize_fixed() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "fixed", @@ -2381,7 +2360,7 @@ mod bigdecimal { "name": "LongVal" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2450,7 +2429,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_decimal_bytes() -> TestResult { + fn test_serialize_decimal_bytes() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "bytes", @@ -2459,7 +2438,7 @@ mod bigdecimal { "scale": 2 }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2488,7 +2467,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_decimal_fixed() -> TestResult { + fn test_serialize_decimal_fixed() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "fixed", @@ -2499,7 +2478,7 @@ mod bigdecimal { "scale": 2 }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2529,14 +2508,14 @@ mod bigdecimal { #[tokio::test] #[serial(serde_is_human_readable)] - async fn test_serialize_bigdecimal() -> TestResult { + fn test_serialize_bigdecimal() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "bytes", "logicalType": "big-decimal" }"#, ) - .await?; + ?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); @@ -2554,14 +2533,14 @@ mod bigdecimal { #[tokio::test] #[serial(serde_is_human_readable)] - async fn test_serialize_uuid() -> TestResult { + fn test_serialize_uuid() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "string", "logicalType": "uuid" }"#, ) - .await?; + ?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); @@ -2599,14 +2578,14 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_date() -> TestResult { + fn test_serialize_date() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "int", "logicalType": "date" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2644,14 +2623,14 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_time_millis() -> TestResult { + fn test_serialize_time_millis() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "int", "logicalType": "time-millis" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2689,14 +2668,14 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_time_micros() -> TestResult { + fn test_serialize_time_micros() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "long", "logicalType": "time-micros" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2737,7 +2716,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_timestamp() -> TestResult { + fn test_serialize_timestamp() -> TestResult { for precision in ["millis", "micros", "nanos"] { let schema = SchemaExt::parse_str(&format!( r#"{{ @@ -2745,7 +2724,7 @@ mod bigdecimal { "logicalType": "timestamp-{precision}" }}"# )) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2796,7 +2775,7 @@ mod bigdecimal { } #[tokio::test] - async fn test_serialize_duration() -> TestResult { + fn test_serialize_duration() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "fixed", @@ -2805,7 +2784,7 @@ mod bigdecimal { "logicalType": "duration" }"#, ) - .await?; + ?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2843,7 +2822,7 @@ mod bigdecimal { #[tokio::test] #[serial(serde_is_human_readable)] // for BigDecimal and Uuid - async fn test_serialize_recursive_record() -> TestResult { + fn test_serialize_recursive_record() -> TestResult { let schema = SchemaExt::parse_str( r#"{ "type": "record", @@ -2856,7 +2835,7 @@ mod bigdecimal { {"name": "innerRecord", "type": ["null", "TestRecord"]} ] }"#, - ).await?; + )?; #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -2905,4 +2884,3 @@ mod bigdecimal { Ok(()) } } -} diff --git a/avro/src/types.rs b/avro/src/types.rs index 612e2f33..ca250d59 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -414,7 +414,7 @@ mod types { { Some(reason) => { let log_message = format!( - "Invalid value: {self:?} for schema: {schema:?}. Reason: {reason}" + "Invalid value: {value:?} for schema: {schema:?}. Reason: {reason}" ); if schemata_len == 1 { error!("{log_message}"); @@ -548,7 +548,7 @@ mod types { // (&Value::Union(None), &Schema::Union(_)) => None, (&Value::Union(i, ref value), Schema::Union(inner)) => { if let Some(schema) = inner.variants().get(i as usize) { - Box::pin(value.validate_internal(schema, names, enclosing_namespace)).await + Box::pin(ValueExt::validate_internal(&value, schema, names, enclosing_namespace)).await } else { Some(format!("No schema in the union at position '{i}'")) } @@ -567,8 +567,7 @@ mod types { for item in items.iter() { acc = Value::accumulate( acc, - Box::pin(item.validate_internal( - &inner.items, + Box::pin(ValueExt::validate_internal(&item, &inner.items, names, enclosing_namespace, )) @@ -588,8 +587,8 @@ mod types { for (_, value) in items.iter() { acc = Value::accumulate( acc, - Box::pin(value.validate_internal( - &inner.types, + Box::pin(ValueExt::validate_internal(&value, + &inner.types, names, enclosing_namespace, )) @@ -643,8 +642,8 @@ mod types { let field = &fields[*idx]; Value::accumulate( acc, - Box::pin(record_field.validate_internal( - &field.schema, + Box::pin(ValueExt::validate_internal(&record_field, + &field.schema, names, record_namespace, )) @@ -663,7 +662,8 @@ mod types { let mut acc = None; for field in fields.iter() { if let Some(item) = items.get(&field.name) { - let res = Box::pin(item.validate_internal( + let res = Box::pin(ValueExt::validate_internal( + &item, &field.schema, names, enclosing_namespace, @@ -694,7 +694,7 @@ mod types { /// See [Schema Resolution](https://avro.apache.org/docs/current/specification/#schema-resolution) /// in the Avro specification for the full set of rules of schema /// resolution. - pub async fn resolve(value: Value, schema: &Schema) -> AvroResult { + pub async fn resolve(value: Value, schema: &Schema) -> AvroResult { Self::resolve_schemata(value, schema, Vec::with_capacity(0)).await } @@ -708,14 +708,14 @@ mod types { value: Value, schema: &Schema, schemata: Vec<&Schema>, - ) -> AvroResult { + ) -> AvroResult { let enclosing_namespace = schema.namespace(); let rs = if schemata.is_empty() { ResolvedSchema::try_from(schema)? } else { ResolvedSchema::try_from(schemata)? }; - Self::resolve_internal(value, schema, rs.get_names(), &enclosing_namespace, &None).await + ValueExt::resolve_internal(value.clone(), schema, rs.get_names(), &enclosing_namespace, &None).await } pub(crate) async fn resolve_internal( @@ -724,7 +724,7 @@ mod types { names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, - ) -> AvroResult { + ) -> AvroResult { // Check if this schema is a union, and if the reader schema is not. if SchemaKind::from(value) == SchemaKind::Union && SchemaKind::from(schema) != SchemaKind::Union @@ -742,7 +742,8 @@ mod types { if let Some(resolved) = names.get(&name) { debug!("Resolved {name:?}"); - Box::pin(self.resolve_internal( + Box::pin(ValueExt::resolve_internal( + value, resolved, names, &name.namespace, @@ -754,55 +755,55 @@ mod types { Err(Details::SchemaResolutionError(name.clone()).into()) } } - Schema::Null => self.resolve_null(), - Schema::Boolean => self.resolve_boolean(), - Schema::Int => self.resolve_int(), - Schema::Long => self.resolve_long(), - Schema::Float => self.resolve_float(), - Schema::Double => self.resolve_double(), - Schema::Bytes => self.resolve_bytes().await, - Schema::String => self.resolve_string(), - Schema::Fixed(FixedSchema { size, .. }) => self.resolve_fixed(size), + Schema::Null => ValueExt::resolve_null(value), + Schema::Boolean => ValueExt::resolve_boolean(value), + Schema::Int => ValueExt::resolve_int(value), + Schema::Long => ValueExt::resolve_long(value), + Schema::Float => ValueExt::resolve_float(value), + Schema::Double => ValueExt::resolve_double(value), + Schema::Bytes => ValueExt::resolve_bytes(value).await, + Schema::String => ValueExt::resolve_string(value), + Schema::Fixed(FixedSchema { size, .. }) => ValueExt::resolve_fixed(value, size), Schema::Union(ref inner) => { - Box::pin(self.resolve_union(inner, names, enclosing_namespace, field_default)) + Box::pin(ValueExt::resolve_union(value, inner, names, enclosing_namespace, field_default)) .await } Schema::Enum(EnumSchema { ref symbols, ref default, .. - }) => self.resolve_enum(symbols, default, field_default), + }) => ValueExt::resolve_enum(value, symbols, default, field_default), Schema::Array(ref inner) => { - Box::pin(self.resolve_array(&inner.items, names, enclosing_namespace)).await + Box::pin(ValueExt::resolve_array(value, &inner.items, names, enclosing_namespace)).await } Schema::Map(ref inner) => { - Box::pin(self.resolve_map(&inner.types, names, enclosing_namespace)).await + Box::pin(ValueExt::resolve_map(value, &inner.types, names, enclosing_namespace)).await } Schema::Record(RecordSchema { ref fields, .. }) => { - Box::pin(self.resolve_record(fields, names, enclosing_namespace)).await + Box::pin(ValueExt::resolve_record(value, fields, names, enclosing_namespace)).await } Schema::Decimal(DecimalSchema { scale, precision, ref inner, - }) => self.resolve_decimal(precision, scale, inner), - Schema::BigDecimal => self.resolve_bigdecimal().await, - Schema::Date => self.resolve_date(), - Schema::TimeMillis => self.resolve_time_millis(), - Schema::TimeMicros => self.resolve_time_micros(), - Schema::TimestampMillis => self.resolve_timestamp_millis(), - Schema::TimestampMicros => self.resolve_timestamp_micros(), - Schema::TimestampNanos => self.resolve_timestamp_nanos(), - Schema::LocalTimestampMillis => self.resolve_local_timestamp_millis(), - Schema::LocalTimestampMicros => self.resolve_local_timestamp_micros(), - Schema::LocalTimestampNanos => self.resolve_local_timestamp_nanos(), - Schema::Duration => self.resolve_duration(), - Schema::Uuid => self.resolve_uuid(), + }) => ValueExt::resolve_decimal(value, precision, scale, inner), + Schema::BigDecimal => ValueExt::resolve_bigdecimal(value).await, + Schema::Date => ValueExt::resolve_date(value), + Schema::TimeMillis => ValueExt::resolve_time_millis(value), + Schema::TimeMicros => ValueExt::resolve_time_micros(value), + Schema::TimestampMillis => ValueExt::resolve_timestamp_millis(value), + Schema::TimestampMicros => ValueExt::resolve_timestamp_micros(value), + Schema::TimestampNanos => ValueExt::resolve_timestamp_nanos(value), + Schema::LocalTimestampMillis => ValueExt::resolve_local_timestamp_millis(value), + Schema::LocalTimestampMicros => ValueExt::resolve_local_timestamp_micros(value), + Schema::LocalTimestampNanos => ValueExt::resolve_local_timestamp_nanos(value), + Schema::Duration => ValueExt::resolve_duration(value), + Schema::Uuid => ValueExt::resolve_uuid(value), } } - fn resolve_uuid(self) -> Result { - Ok(match self { + fn resolve_uuid(value: Value) -> Result { + Ok(match value { uuid @ Value::Uuid(_) => uuid, Value::String(ref string) => { Value::Uuid(Uuid::from_str(string).map_err(Details::ConvertStrToUuid)?) @@ -811,16 +812,16 @@ mod types { }) } - async fn resolve_bigdecimal(self) -> Result { - Ok(match self { + async fn resolve_bigdecimal(value: Value) -> Result { + Ok(match value { bg @ Value::BigDecimal(_) => bg, Value::Bytes(b) => Value::BigDecimal(deserialize_big_decimal(&b).await.unwrap()), other => return Err(Details::GetBigDecimal(other).into()), }) } - fn resolve_duration(self) -> Result { - Ok(match self { + fn resolve_duration(value: Value) -> Result { + Ok(match value { duration @ Value::Duration { .. } => duration, Value::Fixed(size, bytes) => { if size != 12 { @@ -836,11 +837,11 @@ mod types { } fn resolve_decimal( - self, + value: &Value, precision: Precision, scale: Scale, inner: &Schema, - ) -> Result { + ) -> Result { if scale > precision { return Err(Details::GetScaleAndPrecision { scale, precision }.into()); } @@ -853,7 +854,7 @@ mod types { Schema::Bytes => (), _ => return Err(Details::ResolveDecimalSchema(inner.into()).into()), }; - match self { + match value { Value::Decimal(num) => { let num_bytes = num.len(); if max_prec_for_len(num_bytes)? < precision { @@ -883,54 +884,54 @@ mod types { } } - fn resolve_date(self) -> Result { - match self { + fn resolve_date(value: Value) -> AvroResult { + match value { Value::Date(d) | Value::Int(d) => Ok(Value::Date(d)), other => Err(Details::GetDate(other).into()), } } - fn resolve_time_millis(self) -> Result { - match self { + fn resolve_time_millis(value: Value) -> AvroResult { + match value { Value::TimeMillis(t) | Value::Int(t) => Ok(Value::TimeMillis(t)), other => Err(Details::GetTimeMillis(other).into()), } } - fn resolve_time_micros(self) -> Result { - match self { + fn resolve_time_micros(value: Value) -> AvroResult { + match value { Value::TimeMicros(t) | Value::Long(t) => Ok(Value::TimeMicros(t)), Value::Int(t) => Ok(Value::TimeMicros(i64::from(t))), other => Err(Details::GetTimeMicros(other).into()), } } - fn resolve_timestamp_millis(self) -> Result { - match self { + fn resolve_timestamp_millis(value: Value) -> AvroResult { + match value { Value::TimestampMillis(ts) | Value::Long(ts) => Ok(Value::TimestampMillis(ts)), Value::Int(ts) => Ok(Value::TimestampMillis(i64::from(ts))), other => Err(Details::GetTimestampMillis(other).into()), } } - fn resolve_timestamp_micros(self) -> Result { - match self { + fn resolve_timestamp_micros(value: Value) -> AvroResult { + match value { Value::TimestampMicros(ts) | Value::Long(ts) => Ok(Value::TimestampMicros(ts)), Value::Int(ts) => Ok(Value::TimestampMicros(i64::from(ts))), other => Err(Details::GetTimestampMicros(other).into()), } } - fn resolve_timestamp_nanos(self) -> Result { - match self { + fn resolve_timestamp_nanos(value: Value) -> AvroResult { + match value { Value::TimestampNanos(ts) | Value::Long(ts) => Ok(Value::TimestampNanos(ts)), Value::Int(ts) => Ok(Value::TimestampNanos(i64::from(ts))), other => Err(Details::GetTimestampNanos(other).into()), } } - fn resolve_local_timestamp_millis(self) -> Result { - match self { + fn resolve_local_timestamp_millis(value: Value) -> AvroResult { + match value { Value::LocalTimestampMillis(ts) | Value::Long(ts) => { Ok(Value::LocalTimestampMillis(ts)) } @@ -939,8 +940,8 @@ mod types { } } - fn resolve_local_timestamp_micros(self) -> Result { - match self { + fn resolve_local_timestamp_micros(value: Value) -> AvroResult { + match value { Value::LocalTimestampMicros(ts) | Value::Long(ts) => { Ok(Value::LocalTimestampMicros(ts)) } @@ -949,8 +950,8 @@ mod types { } } - fn resolve_local_timestamp_nanos(self) -> Result { - match self { + fn resolve_local_timestamp_nanos(value: Value) -> AvroResult { + match value { Value::LocalTimestampNanos(ts) | Value::Long(ts) => { Ok(Value::LocalTimestampNanos(ts)) } @@ -959,59 +960,59 @@ mod types { } } - fn resolve_null(self) -> Result { - match self { + fn resolve_null(value: Value) -> AvroResult { + match value { Value::Null => Ok(Value::Null), other => Err(Details::GetNull(other).into()), } } - fn resolve_boolean(self) -> Result { - match self { + fn resolve_boolean(value: Value) -> AvroResult { + match value { Value::Boolean(b) => Ok(Value::Boolean(b)), other => Err(Details::GetBoolean(other).into()), } } - fn resolve_int(self) -> Result { - match self { + fn resolve_int(value: Value) -> AvroResult { + match value { Value::Int(n) => Ok(Value::Int(n)), Value::Long(n) => Ok(Value::Int(n as i32)), other => Err(Details::GetInt(other).into()), } } - fn resolve_long(self) -> Result { - match self { + fn resolve_long(value: Value) -> AvroResult { + match value { Value::Int(n) => Ok(Value::Long(i64::from(n))), Value::Long(n) => Ok(Value::Long(n)), other => Err(Details::GetLong(other).into()), } } - fn resolve_float(self) -> Result { - match self { + fn resolve_float(value: Value) -> AvroResult { + match value { Value::Int(n) => Ok(Value::Float(n as f32)), Value::Long(n) => Ok(Value::Float(n as f32)), Value::Float(x) => Ok(Value::Float(x)), Value::Double(x) => Ok(Value::Float(x as f32)), Value::String(ref x) => match Self::parse_special_float(x) { Some(f) => Ok(Value::Float(f)), - None => Err(Details::GetFloat(self).into()), + None => Err(Details::GetFloat(value).into()), }, other => Err(Details::GetFloat(other).into()), } } - fn resolve_double(self) -> Result { - match self { + fn resolve_double(value: Value) -> AvroResult { + match value { Value::Int(n) => Ok(Value::Double(f64::from(n))), Value::Long(n) => Ok(Value::Double(n as f64)), Value::Float(x) => Ok(Value::Double(f64::from(x))), Value::Double(x) => Ok(Value::Double(x)), Value::String(ref x) => match Self::parse_special_float(x) { Some(f) => Ok(Value::Double(f64::from(f))), - None => Err(Details::GetDouble(self).into()), + None => Err(Details::GetDouble(value).into()), }, other => Err(Details::GetDouble(other).into()), } @@ -1028,14 +1029,14 @@ mod types { } } - async fn resolve_bytes(self) -> Result { - match self { + async fn resolve_bytes(value: Value) -> AvroResult { + match value { Value::Bytes(bytes) => Ok(Value::Bytes(bytes)), Value::String(s) => Ok(Value::Bytes(s.into_bytes())), Value::Array(items) => { let mut us = Vec::with_capacity(items.len()); for i in items.into_iter() { - let u = Box::pin(Value::try_u8(i)).await?; + let u = Box::pin(ValueExt::try_u8(i)).await?; us.push(u); } Ok(Value::Bytes(us)) @@ -1044,8 +1045,8 @@ mod types { } } - fn resolve_string(self) -> Result { - match self { + fn resolve_string(value: Value) -> AvroResult { + match value { Value::String(s) => Ok(Value::String(s)), Value::Bytes(bytes) | Value::Fixed(_, bytes) => Ok(Value::String( String::from_utf8(bytes).map_err(Details::ConvertToUtf8)?, @@ -1054,8 +1055,8 @@ mod types { } } - fn resolve_fixed(self, size: usize) -> Result { - match self { + fn resolve_fixed(value: Value, size: usize) -> AvroResult { + match value { Value::Fixed(n, bytes) => { if n == size { Ok(Value::Fixed(n, bytes)) @@ -1076,11 +1077,11 @@ mod types { } pub(crate) fn resolve_enum( - self, + value: Value, symbols: &[String], enum_default: &Option, _field_default: &Option, - ) -> Result { + ) -> AvroResult { let validate_symbol = |symbol: String, symbols: &[String]| { if let Some(index) = symbols.iter().position(|item| item == &symbol) { Ok(Value::Enum(index as u32, symbol)) @@ -1106,7 +1107,7 @@ mod types { } }; - match self { + match value { Value::Enum(_raw_index, s) => validate_symbol(s, symbols), Value::String(s) => validate_symbol(s, symbols), other => Err(Details::GetEnum(other).into()), @@ -1114,13 +1115,13 @@ mod types { } async fn resolve_union( - self, + value: Value, schema: &UnionSchema, names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, - ) -> Result { - let v = match self { + ) -> AvroResult { + let v = match value { // Both are unions case. Value::Union(_i, v) => *v, // Reader is a union, but writer is not. @@ -1137,23 +1138,24 @@ mod types { Ok(Value::Union( i as u32, Box::new( - v.resolve_internal(inner, names, enclosing_namespace, field_default) + ValueExt::resolve_internal(&v, inner, names, enclosing_namespace, field_default) .await?, ), )) } async fn resolve_array( - self, + value: Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, - ) -> Result { - match self { + ) -> AvroResult { + match value { Value::Array(items) => { let mut resolved_values = Vec::with_capacity(items.len()); for item in items.into_iter() { - let resolved = Box::pin(item.resolve_internal( + let resolved = Box::pin(ValueExt::resolve_internal( + item, schema, names, enclosing_namespace, @@ -1173,16 +1175,17 @@ mod types { } async fn resolve_map( - self, + value: Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, - ) -> Result { - match self { + ) -> AvroResult { + match value { Value::Map(items) => { let mut resolved = HashMap::with_capacity(items.len()); for (key, value) in items.into_iter() { - let v = Box::pin(value.resolve_internal( + let v = Box::pin(ValueExt::resolve_internal( + value, schema, names, enclosing_namespace, @@ -1217,7 +1220,7 @@ mod types { fields: &[RecordField], names: &HashMap, enclosing_namespace: &Namespace, - ) -> Result { + ) -> Result { let mut items = match self { Value::Map(items) => Ok(items), Value::Record(fields) => Ok(fields.into_iter().collect::>()), @@ -1240,7 +1243,8 @@ mod types { ref symbols, ref default, .. - }) => Value::from(value.clone()).resolve_enum( + }) => ValueExt::resolve_enum( + &Value::from(value.clone()), symbols, default, &field.default.clone(), @@ -1254,7 +1258,8 @@ mod types { _ => Value::Union( 0, Box::new( - Box::pin(Value::from(value.clone()).resolve_internal( + Box::pin(ValueExt::resolve_internal( + Value::from(value.clone()), first, names, enclosing_namespace, @@ -1273,7 +1278,8 @@ mod types { }, }; - let v = Box::pin(value.resolve_internal( + let v = Box::pin(ValueExt::resolve_internal( + value, &field.schema, names, enclosing_namespace, @@ -1286,8 +1292,8 @@ mod types { Ok(Value::Record(new_fields)) } - async fn try_u8(self) -> AvroResult { - let int = self.resolve(&Schema::Int).await?; + async fn try_u8(value: &Value) -> AvroResult { + let int = ValueExt::resolve(value, &Schema::Int).await?; if let Value::Int(n) = int { if n >= 0 && n <= i32::from(u8::MAX) { return Ok(n as u8); @@ -1364,7 +1370,7 @@ mod types { ]), )]); - assert!(value.validate(&schema).await); + assert!(ValueExt::validate(&value, &schema).await); Ok(()) } @@ -1504,8 +1510,7 @@ mod types { ]; for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { - let err_message = value - .validate_internal(&schema, &HashMap::default(), &None) + let err_message = ValueExt::validate_internal(&value, &schema, &HashMap::default(), &None) .await; assert_eq!(valid, err_message.is_none()); if !valid { @@ -1531,9 +1536,9 @@ mod types { attributes: Default::default(), }); - assert!(Value::Fixed(4, vec![0, 0, 0, 0]).validate(&schema).await); + assert!(ValueExt::validate(&Value::Fixed(4, vec![0, 0, 0, 0]), &schema).await); let value = Value::Fixed(5, vec![0, 0, 0, 0, 0]); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1542,9 +1547,9 @@ mod types { .as_str(), ); - assert!(Value::Bytes(vec![0, 0, 0, 0]).validate(&schema).await); + assert!(ValueExt::validate(&Value::Bytes(vec![0, 0, 0, 0]), &schema).await); let value = Value::Bytes(vec![0, 0, 0, 0, 0]); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1572,11 +1577,11 @@ mod types { attributes: Default::default(), }); - assert!(Value::Enum(0, "spades".to_string()).validate(&schema).await); - assert!(Value::String("spades".to_string()).validate(&schema).await); + assert!(ValueExt::validate(&Value::Enum(0, "spades".to_string()), &schema).await); + assert!(ValueExt::validate(&Value::String("spades".to_string()), &schema).await); let value = Value::Enum(1, "spades".to_string()); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1586,7 +1591,7 @@ mod types { ); let value = Value::Enum(1000, "spades".to_string()); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1596,7 +1601,7 @@ mod types { ); let value = Value::String("lorem".to_string()); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1620,7 +1625,7 @@ mod types { }); let value = Value::Enum(0, "spades".to_string()); - assert!(!value.validate(&other_schema).await); + assert!(!ValueExt::validate(&value, &other_schema).await); assert_logged( format!( "Invalid value: {:?} for schema: {:?}. Reason: {}", @@ -1693,12 +1698,12 @@ mod types { attributes: Default::default(), }); + let value = Value::Record(vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ]); assert!( - Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ]) - .validate(&schema) + ValueExt::validate(&value, &schema) .await ); @@ -1706,13 +1711,13 @@ mod types { ("b".to_string(), Value::String("foo".to_string())), ("a".to_string(), Value::Long(42i64)), ]); - assert!(value.validate(&schema).await); + assert!(ValueExt::validate(&value, &schema).await); let value = Value::Record(vec![ ("a".to_string(), Value::Boolean(false)), ("b".to_string(), Value::String("foo".to_string())), ]); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( r#"Invalid value: Record([("a", Boolean(false)), ("b", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Unsupported value-schema combination! Value: Boolean(false), schema: Long"#, ); @@ -1721,7 +1726,7 @@ mod types { ("a".to_string(), Value::Long(42i64)), ("c".to_string(), Value::String("foo".to_string())), ]); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( r#"Invalid value: Record([("a", Long(42)), ("c", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Could not find matching type in union"#, ); @@ -1733,7 +1738,7 @@ mod types { ("a".to_string(), Value::Long(42i64)), ("d".to_string(), Value::String("foo".to_string())), ]); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( r#"Invalid value: Record([("a", Long(42)), ("d", String("foo"))]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: There is no schema field for field 'd'"#, ); @@ -1744,31 +1749,31 @@ mod types { ("c".to_string(), Value::Null), ("d".to_string(), Value::Null), ]); - assert!(!value.validate(&schema).await); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")), ("c", Null), ("d", Null)]) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: The value's records length (4) is greater than the schema's (3 fields)"#, ); - assert!( - Value::Map( - vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ] + let value = Value::Map( + vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ] .into_iter() .collect() - ) - .validate(&schema) + ); + assert!( + ValueExt::validate(&value, &schema) .await ); + let value = Value::Map( + vec![("d".to_string(), Value::Long(123_i64)),] + .into_iter() + .collect() + ); assert!( - !Value::Map( - vec![("d".to_string(), Value::Long(123_i64)),] - .into_iter() - .collect() - ) - .validate(&schema) + !ValueExt::validate(&value, &schema) .await ); assert_logged( @@ -1778,15 +1783,15 @@ Field with name '"b"' is not a member of the map items"#, let union_schema = Schema::Union(UnionSchema::new(vec![Schema::Null, schema])?); + let value = Value::Union( + 1, + Box::new(Value::Record(vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ])) + ); assert!( - Value::Union( - 1, - Box::new(Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ])) - ) - .validate(&union_schema) + ValueExt::validate(&value, &union_schema) .await ); @@ -1813,7 +1818,7 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_bytes_ok() -> TestResult { let value = Value::Array(vec![Value::Int(0), Value::Int(42)]); assert_eq!( - value.resolve(&Schema::Bytes).await?, + ValueExt::resolve(value, &Schema::Bytes).await?, Value::Bytes(vec![0u8, 42u8]) ); @@ -1824,7 +1829,7 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_string_from_bytes() -> TestResult { let value = Value::Bytes(vec![97, 98, 99]); assert_eq!( - value.resolve(&Schema::String).await?, + ValueExt::resolve(value, &Schema::String).await?, Value::String("abc".to_string()) ); @@ -1835,7 +1840,7 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_string_from_fixed() -> TestResult { let value = Value::Fixed(3, vec![97, 98, 99]); assert_eq!( - value.resolve(&Schema::String).await?, + ValueExt::resolve(value, &Schema::String).await?, Value::String("abc".to_string()) ); @@ -1845,21 +1850,19 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn resolve_bytes_failure() { let value = Value::Array(vec![Value::Int(2000), Value::Int(-42)]); - assert!(value.resolve(&Schema::Bytes).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::Bytes).await.is_err()); } #[tokio::test] async fn resolve_decimal_bytes() -> TestResult { let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); - value - .clone() - .resolve(&Schema::Decimal(DecimalSchema { + ValueExt::resolve(value.clone(), &Schema::Decimal(DecimalSchema { precision: 10, scale: 4, inner: Box::new(Schema::Bytes), })) .await?; - assert!(value.resolve(&Schema::String).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::String).await.is_err()); Ok(()) } @@ -1868,8 +1871,7 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_decimal_invalid_scale() { let value = Value::Decimal(Decimal::from(vec![1, 2])); assert!( - value - .resolve(&Schema::Decimal(DecimalSchema { + ValueExt::resolve(value, &Schema::Decimal(DecimalSchema { precision: 2, scale: 3, inner: Box::new(Schema::Bytes), @@ -1883,8 +1885,7 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_decimal_invalid_precision_for_length() { let value = Value::Decimal(Decimal::from((1u8..=8u8).rev().collect::>())); assert!( - value - .resolve(&Schema::Decimal(DecimalSchema { + ValueExt::resolve(value, &Schema::Decimal(DecimalSchema { precision: 1, scale: 0, inner: Box::new(Schema::Bytes), @@ -1898,9 +1899,7 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_decimal_fixed() { let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); assert!( - value - .clone() - .resolve(&Schema::Decimal(DecimalSchema { + ValueExt::resolve(value.clone(), &Schema::Decimal(DecimalSchema { precision: 10, scale: 1, inner: Box::new(Schema::Fixed(FixedSchema { @@ -1915,70 +1914,67 @@ Field with name '"b"' is not a member of the map items"#, .await .is_ok() ); - assert!(value.resolve(&Schema::String).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::String).await.is_err()); } #[tokio::test] async fn resolve_date() { let value = Value::Date(2345); - assert!(value.clone().resolve(&Schema::Date).await.is_ok()); - assert!(value.resolve(&Schema::String).await.is_err()); + assert!(ValueExt::resolve(value.clone(), &Schema::Date).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::String).await.is_err()); } #[tokio::test] async fn resolve_time_millis() { let value = Value::TimeMillis(10); - assert!(value.clone().resolve(&Schema::TimeMillis).await.is_ok()); - assert!(value.resolve(&Schema::TimeMicros).await.is_err()); + assert!(ValueExt::resolve(value.clone(), &Schema::TimeMillis).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::TimeMicros).await.is_err()); } #[tokio::test] async fn resolve_time_micros() { let value = Value::TimeMicros(10); - assert!(value.clone().resolve(&Schema::TimeMicros).await.is_ok()); - assert!(value.resolve(&Schema::TimeMillis).await.is_err()); + assert!(ValueExt::resolve(value.clone(), &Schema::TimeMicros).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::TimeMillis).await.is_err()); } #[tokio::test] async fn resolve_timestamp_millis() { let value = Value::TimestampMillis(10); assert!( - value - .clone() - .resolve(&Schema::TimestampMillis) + ValueExt::resolve(value.clone(), &Schema::TimestampMillis) .await .is_ok() ); - assert!(value.resolve(&Schema::Float).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::Float).await.is_err()); let value = Value::Float(10.0f32); - assert!(value.resolve(&Schema::TimestampMillis).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::TimestampMillis).await.is_err()); } #[tokio::test] async fn resolve_timestamp_micros() { let value = Value::TimestampMicros(10); assert!( - value - .clone() - .resolve(&Schema::TimestampMicros) + ValueExt::resolve(value + .clone(), &Schema::TimestampMicros) .await .is_ok() ); - assert!(value.resolve(&Schema::Int).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::TimestampMicros).await.is_err()); } #[tokio::test] async fn test_avro_3914_resolve_timestamp_nanos() { let value = Value::TimestampNanos(10); - assert!(value.clone().resolve(&Schema::TimestampNanos).await.is_ok()); - assert!(value.resolve(&Schema::Int).await.is_err()); + assert!(ValueExt::resolve(value.clone(), &Schema::TimestampNanos).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::TimestampNanos).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::TimestampNanos).await.is_err()); } #[tokio::test] @@ -1991,42 +1987,38 @@ Field with name '"b"' is not a member of the map items"#, .await .is_ok() ); - assert!(value.resolve(&Schema::Float).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::Float).await.is_err()); let value = Value::Float(10.0f32); - assert!(value.resolve(&Schema::LocalTimestampMillis).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::LocalTimestampMillis).await.is_err()); } #[tokio::test] async fn test_avro_3853_resolve_timestamp_micros() { let value = Value::LocalTimestampMicros(10); assert!( - value - .clone() - .resolve(&Schema::LocalTimestampMicros) + ValueExt::resolve(value.clone(), &Schema::LocalTimestampMicros) .await .is_ok() ); - assert!(value.resolve(&Schema::Int).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::LocalTimestampMicros).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::LocalTimestampMicros).await.is_err()); } #[tokio::test] async fn test_avro_3916_resolve_timestamp_nanos() { let value = Value::LocalTimestampNanos(10); assert!( - value - .clone() - .resolve(&Schema::LocalTimestampNanos) + ValueExt::resolve(value.clone(), &Schema::LocalTimestampNanos) .await .is_ok() ); - assert!(value.resolve(&Schema::Int).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(value.resolve(&Schema::LocalTimestampNanos).await.is_err()); + assert!(ValueExt::resolve(value, &Schema::LocalTimestampNanos).await.is_err()); } #[tokio::test] @@ -2036,16 +2028,16 @@ Field with name '"b"' is not a member of the map items"#, Days::new(5), Millis::new(3000), )); - assert!(value.clone().resolve(&Schema::Duration).await.is_ok()); - assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); - assert!(Value::Long(1i64).resolve(&Schema::Duration).await.is_err()); + assert!(ValueExt::resolve(value.clone(), &Schema::Duration).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::TimestampMicros).await.is_err()); + assert!(ValueExt::resolve(Value::Long(1i64), &Schema::Duration).await.is_err()); } #[tokio::test] async fn resolve_uuid() -> TestResult { let value = Value::Uuid(Uuid::parse_str("1481531d-ccc9-46d9-a56f-5b67459c0537")?); - assert!(value.clone().resolve(&Schema::Uuid).await.is_ok()); - assert!(value.resolve(&Schema::TimestampMicros).await.is_err()); + assert!(ValueExt::resolve(value.clone(), &Schema::Uuid).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::TimestampMicros).await.is_err()); Ok(()) } @@ -2053,7 +2045,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn avro_3678_resolve_float_to_double() { let value = Value::Float(2345.1); - assert!(value.resolve(&Schema::Double).await.is_ok()); + assert!(ValueExt::resolve(value, &Schema::Double).await.is_ok()); } #[tokio::test] @@ -2097,13 +2089,13 @@ Field with name '"b"' is not a member of the map items"#, "event".to_string(), Value::Record(vec![("amount".to_string(), Value::Int(200))]), )]); - assert!(value.resolve(&schema).await.is_ok()); + assert!(ValueExt::resolve(value, &schema).await.is_ok()); let value = Value::Record(vec![( "event".to_string(), Value::Record(vec![("size".to_string(), Value::Int(1))]), )]); - assert!(value.resolve(&schema).await.is_err()); + assert!(ValueExt::resolve(value, &schema).await.is_err()); Ok(()) } @@ -2320,8 +2312,7 @@ Field with name '"b"' is not a member of the map items"#, let inner_value1 = Value::Record(vec![("z".into(), Value::Int(3))]); let inner_value2 = Value::Record(vec![("z".into(), Value::Int(6))]); let outer = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - outer - .resolve(&schema) + ValueExt::resolve(outer, &schema) .await .expect("Record definition defined in one field must be available in other field"); @@ -2371,8 +2362,7 @@ Field with name '"b"' is not a member of the map items"#, Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), ), ]); - outer_value - .resolve(&schema) + ValueExt::resolve(outer_value, &schema) .await .expect("Record defined in array definition must be resolvable from map"); @@ -2419,8 +2409,7 @@ Field with name '"b"' is not a member of the map items"#, Value::Map(vec![("akey".into(), inner_value2)].into_iter().collect()), ), ]); - outer_value - .resolve(&schema) + ValueExt::resolve(outer_value, &schema) .await .expect("Record defined in record field must be resolvable from map field"); @@ -2469,7 +2458,7 @@ Field with name '"b"' is not a member of the map items"#, )]); let outer_value = Value::Record(vec![("a".into(), inner_value1), ("b".into(), inner_value2)]); - outer_value.resolve(&schema).await.expect("Record schema defined in field must be resolvable in Record schema defined in other field"); + ValueExt::resolve(outer_value, &schema).await.expect("Record schema defined in field must be resolvable in Record schema defined in other field"); Ok(()) } @@ -2517,8 +2506,7 @@ Field with name '"b"' is not a member of the map items"#, ), ("b".into(), Value::Array(vec![inner_value1])), ]); - outer_value - .resolve(&schema) + ValueExt::resolve(outer_value, &schema) .await .expect("Record defined in map definition must be resolvable from array"); @@ -2559,13 +2547,11 @@ Field with name '"b"' is not a member of the map items"#, ("a".into(), inner_value1), ("b".into(), inner_value2.clone()), ]); - outer1 - .resolve(&schema) + ValueExt::resolve(outer1, &schema) .await .expect("Record definition defined in union must be resolved in other field"); let outer2 = Value::Record(vec![("a".into(), Value::Null), ("b".into(), inner_value2)]); - outer2 - .resolve(&schema) + ValueExt::resolve(outer2, &schema) .await .expect("Record definition defined in union must be resolved in other field"); @@ -2647,16 +2633,13 @@ Field with name '"b"' is not a member of the map items"#, ("outer_field_2".into(), inner_record), ]); - outer_record_variation_1 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_1, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_2 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_2, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_3 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_3, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); @@ -2739,16 +2722,13 @@ Field with name '"b"' is not a member of the map items"#, ("outer_field_2".into(), inner_record), ]); - outer_record_variation_1 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_1, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_2 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_2, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_3 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_3, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); @@ -2833,16 +2813,13 @@ Field with name '"b"' is not a member of the map items"#, ("outer_field_2".into(), inner_record), ]); - outer_record_variation_1 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_1, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_2 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_2, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); - outer_record_variation_3 - .resolve(&schema) + ValueExt::resolve(outer_record_variation_3, &schema) .await .expect("Should be able to resolve value to the schema that is it's definition"); @@ -2892,11 +2869,11 @@ Field with name '"b"' is not a member of the map items"#, ]); assert!( - !outer1.validate(&schema).await, + !ValueExt::validate(&outer1, &schema).await, "field b record is invalid against the schema" ); // this should pass, but doesn't assert!( - !outer2.validate(&schema).await, + !ValueExt::validate(&outer2, &schema).await, "field b record is invalid against the schema" ); // this should pass, but doesn't @@ -2979,15 +2956,15 @@ Field with name '"b"' is not a member of the map items"#, let test_outer3: Value = test_outer3.serialize(&mut ser)?; assert!( - !test_outer1.validate(&schema).await, + !ValueExt::validate(&test_outer1, &schema).await, "field b record is invalid against the schema" ); assert!( - !test_outer2.validate(&schema).await, + !ValueExt::validate(&test_outer2, &schema).await, "field b record is invalid against the schema" ); assert!( - !test_outer3.validate(&schema).await, + !ValueExt::validate(&test_outer3, &schema).await, "field b record is invalid against the schema" ); @@ -3069,11 +3046,11 @@ Field with name '"b"' is not a member of the map items"#, let mut ser = Serializer::default(); let test_value: Value = msg.serialize(&mut ser)?; assert!( - test_value.validate(&schema).await, + ValueExt::validate(&test_value, &schema).await, "test_value should validate" ); assert!( - test_value.resolve(&schema).await.is_ok(), + ValueExt::resolve(test_value, &schema).await.is_ok(), "test_value should resolve" ); @@ -3154,11 +3131,11 @@ Field with name '"b"' is not a member of the map items"#, let mut ser = Serializer::default(); let test_value: Value = msg.serialize(&mut ser)?; assert!( - test_value.validate(&schema).await, + ValueExt::validate(&test_value, &schema).await, "test_value should validate" ); assert!( - test_value.resolve(&schema).await.is_ok(), + ValueExt::resolve(test_value, &schema).await.is_ok(), "test_value should resolve" ); @@ -3196,9 +3173,8 @@ Field with name '"b"' is not a member of the map items"#, let main_schema = schemas.first().unwrap(); let schemata: Vec<_> = schemas.iter().skip(1).collect(); - let resolve_result = avro_value - .clone() - .resolve_schemata(main_schema, schemata) + let resolve_result = + ValueExt::resolve_schemata(avro_value.clone(), main_schema, schemata) .await; assert!( @@ -3206,7 +3182,7 @@ Field with name '"b"' is not a member of the map items"#, "result of resolving with schemata should be ok, got: {resolve_result:?}" ); - let resolve_result = avro_value.resolve(main_schema).await; + let resolve_result = ValueExt::resolve(avro_value, main_schema).await; assert!( resolve_result.is_err(), "result of resolving without schemata should be err, got: {resolve_result:?}" @@ -3240,8 +3216,7 @@ Field with name '"b"' is not a member of the map items"#, let main_schema = schemata.last().unwrap(); let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); - let resolve_result = avro_value - .resolve_schemata(main_schema, other_schemata) + let resolve_result = ValueExt::resolve_schemata(avro_value, main_schema, other_schemata) .await; assert!( @@ -3250,8 +3225,7 @@ Field with name '"b"' is not a member of the map items"#, ); assert!( - resolve_result? - .validate_schemata(schemata.iter().collect()) + ValueExt::validate_schemata(&resolve_result?, schemata.iter().collect()) .await, "result of validation with schemata should be true" ); @@ -3267,7 +3241,7 @@ Field with name '"b"' is not a member of the map items"#, BigInt::from(12345678u32).to_signed_bytes_be(), )); let schema = SchemaExt::parse_str(schema).await?; - let resolve_result = avro_value.resolve(&schema).await; + let resolve_result = ValueExt::resolve(avro_value, &schema).await; assert!( resolve_result.is_ok(), "resolve result must be ok, got: {resolve_result:?}" @@ -3283,7 +3257,7 @@ Field with name '"b"' is not a member of the map items"#, let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); let schema = SchemaExt::parse_str(schema).await?; - let resolve_result: AvroResult = avro_value.resolve(&schema).await; + let resolve_result: AvroResult = ValueExt::resolve(avro_value, &schema).await; assert!( resolve_result.is_ok(), "resolve result must be ok, got: {resolve_result:?}" @@ -3296,8 +3270,7 @@ Field with name '"b"' is not a member of the map items"#, async fn test_avro_3892_resolve_fixed_from_bytes() -> TestResult { let value = Value::Bytes(vec![97, 98, 99]); assert_eq!( - value - .resolve(&Schema::Fixed(FixedSchema { + ValueExt::resolve(value, &Schema::Fixed(FixedSchema { name: "test".into(), aliases: None, doc: None, @@ -3311,8 +3284,7 @@ Field with name '"b"' is not a member of the map items"#, let value = Value::Bytes(vec![97, 99]); assert!( - value - .resolve(&Schema::Fixed(FixedSchema { + ValueExt::resolve(value, &Schema::Fixed(FixedSchema { name: "test".into(), aliases: None, doc: None, @@ -3326,8 +3298,7 @@ Field with name '"b"' is not a member of the map items"#, let value = Value::Bytes(vec![97, 98, 99, 100]); assert!( - value - .resolve(&Schema::Fixed(FixedSchema { + ValueExt::resolve(value, &Schema::Fixed(FixedSchema { name: "test".into(), aliases: None, doc: None, @@ -3391,7 +3362,7 @@ Field with name '"b"' is not a member of the map items"#, async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { let schema = SchemaExt::parse_str(r#"{"type": "double"}"#).await?; let value = Value::String("unknown".to_owned()); - match value.resolve(&schema).await.map_err(Error::into_details) { + match ValueExt::resolve(value, &schema).await.map_err(Error::into_details) { Err(err @ Details::GetDouble(_)) => { assert_eq!( format!("{err:?}"), @@ -3409,7 +3380,7 @@ Field with name '"b"' is not a member of the map items"#, async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { let schema = SchemaExt::parse_str(r#"{"type": "float"}"#).await?; let value = Value::String("unknown".to_owned()); - match value.resolve(&schema).await.map_err(Error::into_details) { + match ValueExt::resolve(value, &schema).await.map_err(Error::into_details) { Err(err @ Details::GetFloat(_)) => { assert_eq!( format!("{err:?}"), @@ -3560,7 +3531,7 @@ Field with name '"b"' is not a member of the map items"#, for (schema_str, value, expected_error) in data { let schema = SchemaExt::parse_str(schema_str).await?; - match value.resolve(&schema).await { + match ValueExt::resolve(value, &schema).await { Err(error) => { assert_eq!(format!("{error}"), expected_error); } diff --git a/avro/src/util.rs b/avro/src/util.rs index eb6b8999..5f3d62ff 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -142,18 +142,18 @@ mod util { use crate::AvroResult; use crate::error::Details; - use std::io::Write; + use std::marker::Unpin; pub async fn read_long(reader: &mut R) -> AvroResult { zag_i64(reader).await } - pub async fn zig_i32(n: i32, buffer: W) -> AvroResult { - zig_i64(n as i64, buffer) + pub async fn zig_i32(n: i32, buffer: W) -> AvroResult { + zig_i64(n as i64, buffer).await } - pub async fn zig_i64(n: i64, writer: W) -> AvroResult { - encode_variable(((n << 1) ^ (n >> 63)) as u64, writer) + pub async fn zig_i64(n: i64, writer: W) -> AvroResult { + encode_variable(((n << 1) ^ (n >> 63)) as u64, writer).await } pub async fn zag_i32(reader: &mut R) -> AvroResult { @@ -170,7 +170,7 @@ mod util { }) } - async fn encode_variable(mut z: u64, mut writer: W) -> AvroResult { + async fn encode_variable(mut z: u64, mut writer: W) -> AvroResult { let mut buffer = [0u8; 10]; let mut i: usize = 0; loop { @@ -185,7 +185,7 @@ mod util { } } writer - .write(&buffer[..i]) + .write(&buffer[..i]).await .map_err(|e| Details::WriteBytes(e).into()) } @@ -236,87 +236,87 @@ mod util { use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; - #[test] - fn test_zigzag() { + #[tokio::test] + async fn test_zigzag() { let mut a = Vec::new(); let mut b = Vec::new(); - zig_i32(42i32, &mut a).unwrap(); - zig_i64(42i64, &mut b).unwrap(); + zig_i32(42i32, &mut a).await.unwrap(); + zig_i64(42i64, &mut b).await.unwrap(); assert_eq!(a, b); } - #[test] - fn test_zig_i64() { + #[tokio::test] + async fn test_zig_i64() { let mut s = Vec::new(); - zig_i64(0, &mut s).unwrap(); + zig_i64(0, &mut s).await.unwrap(); assert_eq!(s, [0]); s.clear(); - zig_i64(-1, &mut s).unwrap(); + zig_i64(-1, &mut s).await.unwrap(); assert_eq!(s, [1]); s.clear(); - zig_i64(1, &mut s).unwrap(); + zig_i64(1, &mut s).await.unwrap(); assert_eq!(s, [2]); s.clear(); - zig_i64(-64, &mut s).unwrap(); + zig_i64(-64, &mut s).await.unwrap(); assert_eq!(s, [127]); s.clear(); - zig_i64(64, &mut s).unwrap(); + zig_i64(64, &mut s).await.unwrap(); assert_eq!(s, [128, 1]); s.clear(); - zig_i64(i32::MAX as i64, &mut s).unwrap(); + zig_i64(i32::MAX as i64, &mut s).await.unwrap(); assert_eq!(s, [254, 255, 255, 255, 15]); s.clear(); - zig_i64(i32::MAX as i64 + 1, &mut s).unwrap(); + zig_i64(i32::MAX as i64 + 1, &mut s).await.unwrap(); assert_eq!(s, [128, 128, 128, 128, 16]); s.clear(); - zig_i64(i32::MIN as i64, &mut s).unwrap(); + zig_i64(i32::MIN as i64, &mut s).await.unwrap(); assert_eq!(s, [255, 255, 255, 255, 15]); s.clear(); - zig_i64(i32::MIN as i64 - 1, &mut s).unwrap(); + zig_i64(i32::MIN as i64 - 1, &mut s).await.unwrap(); assert_eq!(s, [129, 128, 128, 128, 16]); s.clear(); - zig_i64(i64::MAX, &mut s).unwrap(); + zig_i64(i64::MAX, &mut s).await.unwrap(); assert_eq!(s, [254, 255, 255, 255, 255, 255, 255, 255, 255, 1]); s.clear(); - zig_i64(i64::MIN, &mut s).unwrap(); + zig_i64(i64::MIN, &mut s).await.unwrap(); assert_eq!(s, [255, 255, 255, 255, 255, 255, 255, 255, 255, 1]); } - #[test] - fn test_zig_i32() { + #[tokio::test] + async fn test_zig_i32() { let mut s = Vec::new(); - zig_i32(i32::MAX / 2, &mut s).unwrap(); + zig_i32(i32::MAX / 2, &mut s).await.unwrap(); assert_eq!(s, [254, 255, 255, 255, 7]); s.clear(); - zig_i32(i32::MIN / 2, &mut s).unwrap(); + zig_i32(i32::MIN / 2, &mut s).await.unwrap(); assert_eq!(s, [255, 255, 255, 255, 7]); s.clear(); - zig_i32(-(i32::MIN / 2), &mut s).unwrap(); + zig_i32(-(i32::MIN / 2), &mut s).await.unwrap(); assert_eq!(s, [128, 128, 128, 128, 8]); s.clear(); - zig_i32(i32::MIN / 2 - 1, &mut s).unwrap(); + zig_i32(i32::MIN / 2 - 1, &mut s).await.unwrap(); assert_eq!(s, [129, 128, 128, 128, 8]); s.clear(); - zig_i32(i32::MAX, &mut s).unwrap(); + zig_i32(i32::MAX, &mut s).await.unwrap(); assert_eq!(s, [254, 255, 255, 255, 15]); s.clear(); - zig_i32(i32::MIN, &mut s).unwrap(); + zig_i32(i32::MIN, &mut s).await.unwrap(); assert_eq!(s, [255, 255, 255, 255, 15]); } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index deb854ec..d506e8c6 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -42,6 +42,14 @@ )] mod writer { + #[synca::cfg(sync)] + use std::io::Write as AvroWrite; + use std::marker::Unpin; + #[synca::cfg(tokio)] + use tokio::io::AsyncWrite as AvroWrite; + #[cfg(feature = "tokio")] + use tokio::io::AsyncWriteExt; + use crate::AvroResult; use crate::{ codec::tokio::Codec, @@ -51,14 +59,15 @@ mod writer { headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, schema::tokio::AvroSchema, schema::{Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, - ser_schema::tokio::SchemaAwareWriteSerializer, + ser_schema::SchemaAwareWriteSerializer, types::Value, }; use serde::Serialize; use std::{ - collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, + collections::HashMap, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive, }; + use crate::types::tokio::ValueExt; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; @@ -69,7 +78,7 @@ mod writer { /// the contents of the buffer, any errors that happen in the process of dropping will be ignored. /// Calling flush ensures that the buffer is empty and thus dropping will not even attempt file operations. #[derive(bon::Builder)] - pub struct Writer<'a, W: Write> { + pub struct Writer<'a, W: AvroWrite + Unpin> { schema: &'a Schema, writer: W, #[builder(skip)] @@ -93,7 +102,7 @@ mod writer { user_metadata: HashMap, } - impl<'a, W: Write> Writer<'a, W> { + impl<'a, W: AvroWrite + Unpin> Writer<'a, W> { /// Creates a `Writer` given a `Schema` and something implementing the `io::Write` trait to write /// to. /// No compression `Codec` will be used. @@ -206,7 +215,7 @@ mod writer { /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). pub async fn append_value_ref(&mut self, value: &Value) -> AvroResult { - let n = self.maybe_write_header()?; + let n = self.maybe_write_header().await?; // Lazy init for users using the builder pattern with error throwing match self.resolved_schema { @@ -215,7 +224,7 @@ mod writer { self.num_values += 1; if self.buffer.len() >= self.block_size { - return self.flush().map(|b| b + n); + return self.flush().await.map(|b| b + n); } Ok(n) @@ -237,8 +246,8 @@ mod writer { /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). - pub fn append_ser(&mut self, value: S) -> AvroResult { - let n = self.maybe_write_header()?; + pub async fn append_ser(&mut self, value: S) -> AvroResult { + let n = self.maybe_write_header().await?; match self.resolved_schema { Some(ref rs) => { @@ -252,7 +261,7 @@ mod writer { self.num_values += 1; if self.buffer.len() >= self.block_size { - return self.flush().map(|b| b + n); + return self.flush().await.map(|b| b + n); } Ok(n) @@ -294,7 +303,7 @@ mod writer { for value in values { num_bytes += self.append(value).await?; } - num_bytes += self.flush()?; + num_bytes += self.flush().await?; Ok(num_bytes) } @@ -307,7 +316,7 @@ mod writer { /// /// **NOTE**: This function forces the written data to be flushed (an implicit /// call to [`flush`](Writer::flush) is performed). - pub fn extend_ser(&mut self, values: I) -> AvroResult + pub async fn extend_ser(&mut self, values: I) -> AvroResult where I: IntoIterator, { @@ -329,7 +338,7 @@ mod writer { for value in values { num_bytes += self.append_ser(value)?; } - num_bytes += self.flush()?; + num_bytes += self.flush().await?; Ok(num_bytes) } @@ -346,7 +355,7 @@ mod writer { for value in values { num_bytes += self.append_value_ref(value).await?; } - num_bytes += self.flush()?; + num_bytes += self.flush().await?; Ok(num_bytes) } @@ -358,8 +367,8 @@ mod writer { /// [`WriterBuilder::has_header`]. /// /// Returns the number of bytes written. - pub fn flush(&mut self) -> AvroResult { - let mut num_bytes = self.maybe_write_header()?; + pub async fn flush(&mut self) -> AvroResult { + let mut num_bytes = self.maybe_write_header().await?; if self.num_values == 0 { return Ok(num_bytes); } @@ -369,18 +378,18 @@ mod writer { let num_values = self.num_values; let stream_len = self.buffer.len(); - num_bytes += self.append_raw(&num_values.into(), &Schema::Long)? - + self.append_raw(&stream_len.into(), &Schema::Long)? + num_bytes += self.append_raw(&num_values.into(), &Schema::Long).await? + + self.append_raw(&stream_len.into(), &Schema::Long).await? + self .writer - .write(self.buffer.as_ref()) + .write(self.buffer.as_ref()).await .map_err(Details::WriteBytes)? - + self.append_marker()?; + + self.append_marker().await?; self.buffer.clear(); self.num_values = 0; - self.writer.flush().map_err(Details::FlushWriter)?; + self.writer.flush().await.map_err(Details::FlushWriter)?; Ok(num_bytes) } @@ -389,9 +398,9 @@ mod writer { /// /// **NOTE**: This function forces the written data to be flushed (an implicit /// call to [`flush`](Writer::flush) is performed). - pub fn into_inner(mut self) -> AvroResult { - self.maybe_write_header()?; - self.flush()?; + pub async fn into_inner(mut self) -> AvroResult { + self.maybe_write_header().await?; + self.flush().await?; let mut this = ManuallyDrop::new(self); @@ -425,23 +434,23 @@ mod writer { } /// Generate and append synchronization marker to the payload. - fn append_marker(&mut self) -> AvroResult { + async fn append_marker(&mut self) -> AvroResult { // using .writer.write directly to avoid mutable borrow of self // with ref borrowing of self.marker self.writer - .write(&self.marker) + .write(&self.marker).await .map_err(|e| Details::WriteMarker(e).into()) } /// Append a raw Avro Value to the payload avoiding to encode it again. - fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult { - self.append_bytes(encode_to_vec(value, schema).await?.as_ref()) + async fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult { + self.append_bytes(encode_to_vec(value, schema).await?.as_ref()).await } /// Append pure bytes to the payload. - fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult { + async fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult { self.writer - .write(bytes) + .write(bytes).await .map_err(|e| Details::WriteBytes(e).into()) } @@ -465,7 +474,7 @@ mod writer { } /// Create an Avro header based on schema, codec and sync marker. - fn header(&self) -> Result, Error> { + async fn header(&self) -> Result, Error> { let schema_bytes = serde_json::to_string(self.schema) .map_err(Details::ConvertJsonToString)? .into_bytes(); @@ -504,16 +513,16 @@ mod writer { let mut header = Vec::new(); header.extend_from_slice(AVRO_OBJECT_HEADER); - encode(&metadata.into(), &Schema::map(Schema::Bytes), &mut header)?; + encode(&metadata.into(), &Schema::map(Schema::Bytes), &mut header).await?; header.extend_from_slice(&self.marker); Ok(header) } - fn maybe_write_header(&mut self) -> AvroResult { + async fn maybe_write_header(&mut self) -> AvroResult { if !self.has_header { - let header = self.header()?; - let n = self.append_bytes(header.as_ref())?; + let header = self.header().await?; + let n = self.append_bytes(header.as_ref()).await?; self.has_header = true; Ok(n) } else { @@ -522,7 +531,7 @@ mod writer { } } - impl Drop for Writer<'_, W> { + impl Drop for Writer<'_, W> { /// Drop the writer, will try to flush ignoring any errors. fn drop(&mut self) { let _ = self.maybe_write_header(); @@ -535,16 +544,16 @@ mod writer { /// /// This is an internal function which gets the bytes buffer where to write as parameter instead of /// creating a new one like `to_avro_datum`. - async fn write_avro_datum, W: Write>( + async fn write_avro_datum, W: AvroWrite + Unpin>( schema: &Schema, value: T, writer: &mut W, ) -> Result<(), Error> { let avro = value.into(); - if !avro.validate(schema).await { + if !ValueExt::validate(&avro, schema).await { return Err(Details::Validation.into()); } - encode(&avro, schema, writer)?; + encode(&avro, schema, writer).await?; Ok(()) } @@ -558,8 +567,8 @@ mod writer { let rs = ResolvedSchema::try_from(schemata)?; let names = rs.get_names(); let enclosing_namespace = schema.namespace(); - if let Some(_err) = avro - .validate_internal(schema, names, &enclosing_namespace) + if let Some(_err) = + ValueExt::validate_internal(&avro, schema, names, &enclosing_namespace) .await { return Err(Details::Validation.into()); @@ -602,7 +611,7 @@ mod writer { const HEADER_LENGTH_RANGE: RangeInclusive = 10_usize..=20_usize; /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header - pub async fn write_value_ref( + pub async fn write_value_ref( &mut self, v: &Value, writer: &mut W, @@ -613,7 +622,7 @@ mod writer { } else { write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer).await?; writer - .write_all(&self.buffer) + .write_all(&self.buffer).await .map_err(Details::WriteBytes)?; let len = self.buffer.len(); self.buffer.truncate(original_length); @@ -622,7 +631,7 @@ mod writer { } /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header - pub async fn write_value( + pub async fn write_value( &mut self, v: Value, writer: &mut W, @@ -663,7 +672,7 @@ mod writer { { /// Write the `Into` to the provided Write object. Returns a result with the number /// of bytes written including the header - pub async fn write_value( + pub async fn write_value( &mut self, data: T, writer: &mut W, @@ -679,12 +688,12 @@ mod writer { { /// Write the referenced `Serialize` object to the provided `Write` object. Returns a result with /// the number of bytes written including the header - pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { + pub async fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { let mut bytes_written: usize = 0; if !self.header_written { bytes_written += writer - .write(self.inner.buffer.as_slice()) + .write(self.inner.buffer.as_slice()).await .map_err(Details::WriteBytes)?; self.header_written = true; } @@ -696,8 +705,8 @@ mod writer { /// Write the Serialize object to the provided Write object. Returns a result with the number /// of bytes written including the header - pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { - self.write_ref(&data, writer) + pub async fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + self.write_ref(&data, writer).await } } @@ -707,8 +716,7 @@ mod writer { value: &Value, buffer: &mut Vec, ) -> AvroResult { - match value - .validate_internal(schema, resolved_schema.get_names(), &schema.namespace()) + match ValueExt::validate_internal(&value, schema, resolved_schema.get_names(), &schema.namespace()) .await { Some(reason) => Err(Details::ValidationWithReason { @@ -736,8 +744,8 @@ mod writer { buffer: &mut Vec, ) -> AvroResult<()> { let root_schema = resolved_schema.get_root_schema(); - if let Some(reason) = value - .validate_internal( + if let Some(reason) = ValueExt::validate_internal( + &value, root_schema, resolved_schema.get_names(), &root_schema.namespace(), @@ -780,7 +788,7 @@ mod writer { /// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync /// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible /// if you don't know what you are doing, instead. - pub fn write_avro_datum_ref( + pub fn write_avro_datum_ref( schema: &Schema, data: &T, writer: &mut W, @@ -881,8 +889,8 @@ mod writer { record.put("b", "foo"); let mut expected = Vec::new(); - zig_i64(27, &mut expected)?; - zig_i64(3, &mut expected)?; + zig_i64(27, &mut expected).await?; + zig_i64(3, &mut expected).await?; expected.extend([b'f', b'o', b'o']); assert_eq!(to_avro_datum(&schema, record).await?, expected); @@ -906,8 +914,8 @@ mod writer { }; let mut expected = Vec::new(); - zig_i64(27, &mut expected)?; - zig_i64(3, &mut expected)?; + zig_i64(27, &mut expected).await?; + zig_i64(3, &mut expected).await?; expected.extend([b'f', b'o', b'o']); let bytes = write_avro_datum_ref(&schema, &data, &mut writer)?; @@ -924,8 +932,8 @@ mod writer { // By default flush should write the header even if nothing was added yet let mut writer = Writer::new(&schema, Vec::new()); - writer.flush()?; - let result = writer.into_inner()?; + writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(result.len(), 163); // Unless the user indicates via the builder that the header has already been written @@ -934,8 +942,8 @@ mod writer { .schema(&schema) .writer(Vec::new()) .build(); - writer.flush()?; - let result = writer.into_inner()?; + writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(result.len(), 0); Ok(()) @@ -947,8 +955,8 @@ mod writer { let union = Value::Union(1, Box::new(Value::Long(3))); let mut expected = Vec::new(); - zig_i64(1, &mut expected)?; - zig_i64(3, &mut expected)?; + zig_i64(1, &mut expected).await?; + zig_i64(3, &mut expected).await?; assert_eq!(to_avro_datum(&schema, union).await?, expected); @@ -961,7 +969,7 @@ mod writer { let union = Value::Union(0, Box::new(Value::Null)); let mut expected = Vec::new(); - zig_i64(0, &mut expected)?; + zig_i64(0, &mut expected).await?; assert_eq!(to_avro_datum(&schema, union).await?, expected); @@ -1129,14 +1137,14 @@ mod writer { let n1 = writer.append(record.clone()).await?; let n2 = writer.append(record.clone()).await?; - let n3 = writer.flush()?; - let result = writer.into_inner()?; + let n3 = writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(n1 + n2 + n3, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; + zig_i64(27, &mut data).await?; + zig_i64(3, &mut data).await?; data.extend(b"foo"); data.extend(data.clone()); @@ -1164,14 +1172,14 @@ mod writer { let records = vec![record, record_copy]; let n1 = writer.extend(records).await?; - let n2 = writer.flush()?; - let result = writer.into_inner()?; + let n2 = writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(n1 + n2, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; + zig_i64(27, &mut data).await?; + zig_i64(3, &mut data).await?; data.extend(b"foo"); data.extend(data.clone()); @@ -1204,14 +1212,14 @@ mod writer { }; let n1 = writer.append_ser(record)?; - let n2 = writer.flush()?; - let result = writer.into_inner()?; + let n2 = writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(n1 + n2, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; + zig_i64(27, &mut data).await?; + zig_i64(3, &mut data).await?; data.extend(b"foo"); // starts with magic @@ -1238,15 +1246,15 @@ mod writer { let record_copy = record.clone(); let records = vec![record, record_copy]; - let n1 = writer.extend_ser(records)?; - let n2 = writer.flush()?; - let result = writer.into_inner()?; + let n1 = writer.extend_ser(records).await?; + let n2 = writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(n1 + n2, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; + zig_i64(27, &mut data).await?; + zig_i64(3, &mut data).await?; data.extend(b"foo"); data.extend(data.clone()); @@ -1286,14 +1294,14 @@ mod writer { let n1 = writer.append(record.clone()).await?; let n2 = writer.append(record.clone()).await?; - let n3 = writer.flush()?; - let result = writer.into_inner()?; + let n3 = writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(n1 + n2 + n3, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data)?; - zig_i64(3, &mut data)?; + zig_i64(27, &mut data).await?; + zig_i64(3, &mut data).await?; data.extend(b"foo"); data.extend(data.clone()); Codec::Deflate(DeflateSettings::default()).compress(&mut data)?; @@ -1363,18 +1371,18 @@ mod writer { let n1 = writer.append(record1).await?; let n2 = writer.append(record2).await?; - let n3 = writer.flush()?; - let result = writer.into_inner()?; + let n3 = writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(n1 + n2 + n3, result.len()); let mut data = Vec::new(); // byte indicating not null - zig_i64(1, &mut data)?; - zig_i64(1234, &mut data)?; + zig_i64(1, &mut data).await?; + zig_i64(1234, &mut data).await?; // byte indicating null - zig_i64(0, &mut data)?; + zig_i64(0, &mut data).await?; codec.compress(&mut data)?; // starts with magic @@ -1405,8 +1413,8 @@ mod writer { writer.append(record.clone()).await?; writer.append(record.clone()).await?; - writer.flush()?; - let result = writer.into_inner()?; + writer.flush().await?; + let result = writer.into_inner().await?; assert_eq!(result.len(), 260); @@ -1418,7 +1426,7 @@ mod writer { let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); writer.add_user_metadata("a".to_string(), "b")?; - let result = writer.into_inner()?; + let result = writer.into_inner().await?; let mut reader = Reader::with_schema(&schema, &result[..]).await?; let mut expected = HashMap::new(); @@ -1594,7 +1602,7 @@ mod writer { &value, &TestSingleObjectWriter::get_schema().await, &mut msg_binary, - ) + ).await .expect("encode should have failed by here as a dependency of any writing"); assert_eq!(&buf[10..], &msg_binary[..]); @@ -1651,7 +1659,7 @@ mod writer { .await .expect("Resolved should pass"); specific_writer - .write(obj1.clone(), &mut buf1) + .write(obj1.clone(), &mut buf1).await .expect("Serialization expected"); specific_writer .write_value(obj1.clone(), &mut buf2) @@ -1738,8 +1746,9 @@ mod writer { Ok(()) } - #[tokio::test] - async fn avro_4063_flush_applies_to_inner_writer() -> TestResult { + #[synca::cfg(sync)] + #[test] + fn avro_4063_flush_applies_to_inner_writer() -> TestResult { const SCHEMA: &str = r#" { "type": "record", @@ -1759,9 +1768,9 @@ mod writer { } } - impl Write for TestBuffer { + impl std::io::Write for TestBuffer { fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.borrow_mut().write(buf) + self.0.borrow_mut().write(buf).await } fn flush(&mut self) -> std::io::Result<()> { @@ -1781,7 +1790,7 @@ mod writer { record.put("exampleField", "value"); writer.append(record).await?; - writer.flush()?; + writer.flush().await?; assert_eq!( shared_buffer.len(), From 2a2d2d6d848c2637000dd8c88f03a8cd367875cd Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 00:36:54 +0300 Subject: [PATCH 21/47] WIP: More fixes in types::Value:: resolve() and validate() Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 6 +- avro/src/decode.rs | 2 - avro/src/encode.rs | 8 +- avro/src/lib.rs | 255 ++++++++++++++-------------- avro/src/reader.rs | 4 +- avro/src/schema.rs | 95 ++++++----- avro/src/ser_schema.rs | 42 ++--- avro/src/types.rs | 365 +++++++++++++++++++++-------------------- avro/src/writer.rs | 91 +++++----- 9 files changed, 443 insertions(+), 425 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 83a309f0..5bc93d9d 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -102,7 +102,7 @@ mod bigdecimal { mod tests { use super::*; use crate::{ - codec::tokio::Codec, error::Error, reader::tokio::Reader, schema::Schema, + codec::tokio::Codec, error::Error, reader::tokio::Reader, schema::tokio::SchemaExt, types::Record, writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; @@ -201,10 +201,10 @@ mod bigdecimal { .build(); writer.append(record.clone()).await?; - writer.flush()?; + writer.flush().await?; // read record - let wrote_data = writer.into_inner()?; + let wrote_data = writer.into_inner().await?; let mut reader = Reader::new(&wrote_data[..]).await?; let value = reader.next().await.unwrap()?; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 5b9db3d2..0198401c 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -48,10 +48,8 @@ mod decode { bigdecimal::tokio::deserialize_big_decimal, decimal::Decimal, duration::Duration, - encode::tokio::encode_long, error::Details, error::Error, - schema::tokio::SchemaExt, schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, Schema, diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 7f7639b3..bbd2bd5f 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -58,18 +58,14 @@ mod encode { use log::error; use std::{borrow::Borrow, collections::HashMap}; - /// Encode a `Value` into avro format. - /// - /// **NOTE** This will not perform schema validation. The value is assumed to - /// be valid with regards to the schema. Schema are needed only to guide the - /// encoding for complex type values. + pub async fn encode( value: &Value, schema: &Schema, writer: &mut W, ) -> AvroResult { let rs = ResolvedSchema::try_from(schema)?; - encode_internal(value, schema, rs.get_names(), &None, writer).await + Box::pin(encode_internal(value, schema, rs.get_names(), &None, writer)).await } pub(crate) async fn encode_bytes + ?Sized, W: AvroWrite + Unpin>( diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 12f50236..975f92ca 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -116,6 +116,7 @@ //! //! ``` //! use apache_avro::Schema; +//! use apache_avro::schema::sync::SchemaExt; //! //! let raw_schema = r#" //! { @@ -129,7 +130,7 @@ //! "#; //! //! // if the schema is not valid, this function will return an error -//! let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); //! //! // schemas can be printed for debugging //! println!("{:?}", schema); @@ -140,6 +141,7 @@ //! //! ``` //! use apache_avro::Schema; +//! use apache_avro::schema::sync::SchemaExt; //! //! let raw_schema_1 = r#"{ //! "name": "A", @@ -192,21 +194,22 @@ //! associated type provided by the library to specify the data we want to serialize: //! //! ``` -//! # use apache_avro::Schema; +//! use apache_avro::Schema; //! use apache_avro::types::Record; //! use apache_avro::Writer; -//! # -//! # let raw_schema = r#" -//! # { -//! # "type": "record", -//! # "name": "test", -//! # "fields": [ -//! # {"name": "a", "type": "long", "default": 42}, -//! # {"name": "b", "type": "string"} -//! # ] -//! # } -//! # "#; -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); +//! use apache_avro::schema::sync::SchemaExt; +//! +//! let raw_schema = r#" +//! { +//! "type": "record", +//! "name": "test", +//! "fields": [ +//! {"name": "a", "type": "long", "default": 42}, +//! {"name": "b", "type": "string"} +//! ] +//! } +//!"#; +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); //! // a writer needs a schema and something to write to //! let mut writer = Writer::new(&schema, Vec::new()); //! @@ -241,9 +244,10 @@ //! deriving `Serialize` to model our data: //! //! ``` -//! # use apache_avro::Schema; -//! # use serde::Serialize; +//! use apache_avro::Schema; +//! use serde::Serialize; //! use apache_avro::Writer; +//! use apache_avro::schema::sync::SchemaExt; //! //! #[derive(Debug, Serialize)] //! struct Test { @@ -251,17 +255,17 @@ //! b: String, //! } //! -//! # let raw_schema = r#" -//! # { -//! # "type": "record", -//! # "name": "test", -//! # "fields": [ -//! # {"name": "a", "type": "long", "default": 42}, -//! # {"name": "b", "type": "string"} -//! # ] -//! # } -//! # "#; -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); +//! let raw_schema = r#" +//! { +//! "type": "record", +//! "name": "test", +//! "fields": [ +//! {"name": "a", "type": "long", "default": 42}, +//! {"name": "b", "type": "string"} +//! ] +//! } +//! "#; +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); //! // a writer needs a schema and something to write to //! let mut writer = Writer::new(&schema, Vec::new()); //! @@ -315,18 +319,19 @@ //! To specify a codec to use to compress data, just specify it while creating a `Writer`: //! ``` //! use apache_avro::{Codec, DeflateSettings, Schema, Writer}; -//! # -//! # let raw_schema = r#" -//! # { -//! # "type": "record", -//! # "name": "test", -//! # "fields": [ -//! # {"name": "a", "type": "long", "default": 42}, -//! # {"name": "b", "type": "string"} -//! # ] -//! # } -//! # "#; -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); +//! use apache_avro::schema::sync::SchemaExt; +//! +//! let raw_schema = r#" +//! { +//! "type": "record", +//! "name": "test", +//! "fields": [ +//! {"name": "a", "type": "long", "default": 42}, +//! {"name": "b", "type": "string"} +//! ] +//! } +//!"#; +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); //! let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate(DeflateSettings::default())); //! ``` //! @@ -338,27 +343,28 @@ //! //! ``` //! use apache_avro::Reader; -//! # use apache_avro::Schema; -//! # use apache_avro::types::Record; -//! # use apache_avro::Writer; -//! # -//! # let raw_schema = r#" -//! # { -//! # "type": "record", -//! # "name": "test", -//! # "fields": [ -//! # {"name": "a", "type": "long", "default": 42}, -//! # {"name": "b", "type": "string"} -//! # ] -//! # } -//! # "#; -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); -//! # let mut writer = Writer::new(&schema, Vec::new()); -//! # let mut record = Record::new(writer.schema()).unwrap(); -//! # record.put("a", 27i64); -//! # record.put("b", "foo"); -//! # writer.append(record).unwrap(); -//! # let input = writer.into_inner().unwrap(); +//! use apache_avro::Schema; +//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::types::Record; +//! use apache_avro::Writer; +//! +//! let raw_schema = r#" +//! { +//! "type": "record", +//! "name": "test", +//! "fields": [ +//! {"name": "a", "type": "long", "default": 42}, +//! {"name": "b", "type": "string"} +//! ] +//! } +//! "#; +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); +//! let mut writer = Writer::new(&schema, Vec::new()); +//! let mut record = Record::new(writer.schema()).unwrap(); +//! record.put("a", 27i64); +//! record.put("b", "foo"); +//! writer.append(record).unwrap(); +//! let input = writer.into_inner().unwrap(); //! // reader creation can fail in case the input to read from is not Avro-compatible or malformed //! let reader = Reader::new(&input[..]).unwrap(); //! ``` @@ -368,26 +374,27 @@ //! ``` //! use apache_avro::Schema; //! use apache_avro::Reader; -//! # use apache_avro::types::Record; -//! # use apache_avro::Writer; -//! # -//! # let writer_raw_schema = r#" -//! # { -//! # "type": "record", -//! # "name": "test", -//! # "fields": [ -//! # {"name": "a", "type": "long", "default": 42}, -//! # {"name": "b", "type": "string"} -//! # ] -//! # } -//! # "#; -//! # let writer_schema = SchemaExt::parse_str(writer_raw_schema).await.unwrap(); -//! # let mut writer = Writer::new(&writer_schema, Vec::new()); -//! # let mut record = Record::new(writer.schema()).unwrap(); -//! # record.put("a", 27i64); -//! # record.put("b", "foo"); -//! # writer.append(record).unwrap(); -//! # let input = writer.into_inner().unwrap(); +//! use apache_avro::types::Record; +//! use apache_avro::Writer; +//! use apache_avro::schema::sync::SchemaExt; +//! +//! let writer_raw_schema = r#" +//! { +//! "type": "record", +//! "name": "test", +//! "fields": [ +//! {"name": "a", "type": "long", "default": 42}, +//! {"name": "b", "type": "string"} +//! ] +//! } +//! "#; +//! let writer_schema = SchemaExt::parse_str(writer_raw_schema).unwrap(); +//! let mut writer = Writer::new(&writer_schema, Vec::new()); +//! let mut record = Record::new(writer.schema()).unwrap(); +//! record.put("a", 27i64); +//! record.put("b", "foo"); +//! writer.append(record).unwrap(); +//! let input = writer.into_inner().unwrap(); //! //! let reader_raw_schema = r#" //! { @@ -401,7 +408,7 @@ //! } //! "#; //! -//! let reader_schema = SchemaExt::parse_str(reader_raw_schema).await.unwrap(); +//! let reader_schema = SchemaExt::parse_str(reader_raw_schema).unwrap(); //! //! // reader creation can fail in case the input to read from is not Avro-compatible or malformed //! let reader = Reader::with_schema(&reader_schema, &input[..]).unwrap(); @@ -425,29 +432,30 @@ //! We can just read directly instances of `Value` out of the `Reader` iterator: //! //! ``` -//! # use apache_avro::Schema; -//! # use apache_avro::types::Record; -//! # use apache_avro::Writer; +//! use apache_avro::Schema; +//! use apache_avro::types::Record; +//! use apache_avro::Writer; //! use apache_avro::Reader; -//! # -//! # let raw_schema = r#" -//! # { -//! # "type": "record", -//! # "name": "test", -//! # "fields": [ -//! # {"name": "a", "type": "long", "default": 42}, -//! # {"name": "b", "type": "string"} -//! # ] -//! # } -//! # "#; -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); -//! # let mut writer = Writer::new(&schema, Vec::new()); -//! # let mut record = Record::new(writer.schema()).unwrap(); -//! # record.put("a", 27i64); -//! # record.put("b", "foo"); -//! # writer.append(record).unwrap(); -//! # let input = writer.into_inner().unwrap(); +//! use apache_avro::schema::sync::SchemaExt; +//! +//! let raw_schema = r#" +//! { +//! "type": "record", +//! "name": "test", +//! "fields": [ +//! {"name": "a", "type": "long", "default": 42}, +//! {"name": "b", "type": "string"} +//! ] +//! } +//! "#; +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); +//! let schema = SchemaExt::parse_str(raw_schema).unwrap(); +//! let mut writer = Writer::new(&schema, Vec::new()); +//! let mut record = Record::new(writer.schema()).unwrap(); +//! record.put("a", 27i64); +//! record.put("b", "foo"); +//! writer.append(record).unwrap(); +//! let input = writer.into_inner().unwrap(); //! let reader = Reader::new(&input[..]).unwrap(); //! //! // value is a Result of an Avro Value in case the read operation fails @@ -467,6 +475,7 @@ //! # use apache_avro::Writer; //! # use serde::{Deserialize, Serialize}; //! use apache_avro::Reader; +//! use apache_avro::schema::sync::SchemaExt; //! use apache_avro::from_value; //! //! # #[derive(Serialize)] @@ -486,7 +495,7 @@ //! # ] //! # } //! # "#; -//! # let schema = SchemaExt::parse_str(raw_schema).await.unwrap(); +//! # let schema = SchemaExt::parse_str(raw_schema).unwrap(); //! # let mut writer = Writer::new(&schema, Vec::new()); //! # let test = Test { //! # a: 27, @@ -509,6 +518,7 @@ //! //! ``` //! use apache_avro::{Codec, DeflateSettings, Reader, Schema, Writer, from_value, types::Record, Error}; +//! use apache_avro::schema::sync::SchemaExt; //! use serde::{Deserialize, Serialize}; //! //! #[derive(Debug, Deserialize, Serialize)] @@ -529,7 +539,7 @@ //! } //! "#; //! -//! let schema = SchemaExt::parse_str(raw_schema).await?; +//! let schema = SchemaExt::parse_str(raw_schema)?; //! //! println!("{:?}", schema); //! @@ -576,6 +586,7 @@ //! types::Record, types::Value, Codec, Days, Decimal, DeflateSettings, Duration, Millis, Months, Reader, Schema, //! Writer, Error, //! }; +//! use apache_avro::schema::sync::SchemaExt; //! use num_bigint::ToBigInt; //! //! fn main() -> Result<(), Error> { @@ -655,7 +666,7 @@ //! } //! "#; //! -//! let schema = SchemaExt::parse_str(raw_schema).await?; +//! let schema = SchemaExt::parse_str(raw_schema)?; //! //! println!("{:?}", schema); //! @@ -701,6 +712,7 @@ //! ```rust //! use apache_avro::rabin::Rabin; //! use apache_avro::{Schema, Error}; +//! use apache_avro::schema::sync::SchemaExt; //! use md5::Md5; //! use sha2::Sha256; //! @@ -715,7 +727,7 @@ //! ] //! } //! "#; -//! let schema = SchemaExt::parse_str(raw_schema).await?; +//! let schema = SchemaExt::parse_str(raw_schema)?; //! println!("{}", schema.fingerprint::()); //! println!("{}", schema.fingerprint::()); //! println!("{}", schema.fingerprint::()); @@ -764,10 +776,10 @@ //! (32bit signed integer) fits into a long (64bit signed integer) //! //! ```rust -//! use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; +//! use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; //! -//! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); -//! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); +//! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); +//! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); //! assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_ok()); //! ``` //! @@ -777,10 +789,10 @@ //! long (64bit signed integer) does not fit into an int (32bit signed integer) //! //! ```rust -//! use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; +//! use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; //! -//! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).await.unwrap(); -//! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).await.unwrap(); +//! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); +//! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); //! assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_err()); //! ``` //! ## Custom names validators @@ -920,16 +932,16 @@ pub use reader::tokio::{ pub use schema::Schema; #[cfg(feature = "sync")] pub use schema::sync::AvroSchema; -#[cfg(feature = "sync")] -use schema::sync::SchemaExt; #[cfg(feature = "tokio")] pub use schema::tokio::AvroSchema as AsyncAvroSchema; -#[cfg(feature = "tokio")] -use schema::tokio::SchemaExt as AsyncSchemaExt; #[cfg(feature = "sync")] pub use ser::sync::to_value; #[cfg(feature = "tokio")] pub use ser::tokio::to_value as async_to_value; +#[cfg(feature = "sync")] +pub use schema::sync::SchemaExt; +#[cfg(feature = "tokio")] +pub use schema::tokio::SchemaExt as AsyncSchemaExt; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; #[cfg(feature = "sync")] @@ -943,7 +955,6 @@ pub use writer::tokio::{ SpecificSingleObjectWriter as AsyncSpecificSingleObjectWriter, Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, to_avro_datum as async_to_avro_datum, to_avro_datum_schemata as async_to_avro_datum_schemata, - write_avro_datum_ref as async_write_avro_datum_ref, }; #[cfg(feature = "derive")] @@ -972,7 +983,7 @@ mod tests { use crate::{ codec::tokio::Codec, reader::tokio::{Reader, from_avro_datum}, - schema::Schema, + schema::tokio::SchemaExt, types::{Record, Value}, writer::tokio::Writer, }; @@ -1019,7 +1030,7 @@ mod tests { record.put("a", 27i64); record.put("b", "foo"); writer.append(record).await.unwrap(); - let input = writer.into_inner().unwrap(); + let input = writer.into_inner().await.unwrap(); let mut reader = Reader::with_schema(&reader_schema, &input[..]) .await .unwrap(); @@ -1063,7 +1074,7 @@ mod tests { record.put("b", "foo"); record.put("c", "clubs"); writer.append(record).await.unwrap(); - let input = writer.into_inner().unwrap(); + let input = writer.into_inner().await.unwrap(); let mut reader = Reader::with_schema(&schema, &input[..]).await.unwrap(); assert_eq!( reader.next().await.unwrap().unwrap(), @@ -1105,7 +1116,7 @@ mod tests { record.put("b", "foo"); record.put("c", "clubs"); writer.append(record).await.unwrap(); - let input = writer.into_inner().unwrap(); + let input = writer.into_inner().await.unwrap(); let mut reader = Reader::new(&input[..]).await.unwrap(); assert_eq!( reader.next().await.unwrap().unwrap(), diff --git a/avro/src/reader.rs b/avro/src/reader.rs index fdcbda6c..51613ea2 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -943,8 +943,8 @@ mod reader { writer.append(record.clone()).await?; writer.append(record.clone()).await?; - writer.flush()?; - let result = writer.into_inner()?; + writer.flush().await?; + let result = writer.into_inner().await?; let reader = Reader::new(&result[..]).await?; assert_eq!(reader.user_metadata(), &user_meta_data); diff --git a/avro/src/schema.rs b/avro/src/schema.rs index d372457b..6f65a0e1 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -630,6 +630,24 @@ pub struct SchemaFingerprint { pub bytes: Vec, } +const RESERVED_FIELDS: &[&str] = &[ + "name", + "type", + "fields", + "symbols", + "items", + "values", + "size", + "logicalType", + "order", + "doc", + "aliases", + "default", + "precision", + "scale", +]; + + impl std::fmt::Display for SchemaFingerprint { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( @@ -864,26 +882,9 @@ impl Schema { format!("\"{s}\"") } - const RESERVED_FIELDS: &[&str] = &[ - "name", - "type", - "fields", - "symbols", - "items", - "values", - "size", - "logicalType", - "order", - "doc", - "aliases", - "default", - "precision", - "scale", - ]; - // Used to define the ordering and inclusion of fields. fn field_ordering_position(field: &str) -> Option { - Self::RESERVED_FIELDS + RESERVED_FIELDS .iter() .position(|&f| f == field) .map(|pos| pos + 1) @@ -1365,7 +1366,7 @@ mod schema { use crate::AvroResult; use crate::error::{Details, Error}; use crate::schema::{ - Alias, Aliases, ArraySchema, DecimalMetadata, DecimalSchema, Documentation, EnumSchema, + Alias, Aliases, DecimalMetadata, DecimalSchema, Documentation, EnumSchema, FixedSchema, MapSchema, Name, Names, Namespace, Precision, RecordField, RecordFieldOrder, RecordSchema, ResolvedSchema, Scale, Schema, SchemaFingerprint, SchemaKind, UnionSchema, }; @@ -1373,13 +1374,13 @@ mod schema { use crate::{ types::{Value, ValueKind}, validator::{ - validate_enum_symbol_name, validate_namespace, validate_record_field_name, + validate_enum_symbol_name, validate_record_field_name, validate_schema_name, }, }; use log::{debug, error, warn}; use serde::{ - Deserialize, Serialize, Serializer, + Serialize, ser::{SerializeMap, SerializeSeq}, }; #[synca::cfg(sync)] @@ -1472,8 +1473,7 @@ mod schema { let mut resolved = false; for schema in schemas { - if ValueExt::resolve_internal(&avro_value - .to_owned(), schema, names, &schema.namespace(), &None) + if ValueExt::resolve_internal(&avro_value, schema, names, &schema.namespace(), &None) .await .is_ok() { @@ -1532,9 +1532,9 @@ mod schema { } /// Returns true if this `RecordField` is nullable, meaning the schema is a `UnionSchema` where the first variant is `Null`. - pub fn is_nullable(&self) -> bool { - match self.schema { - Schema::Union(ref inner) => inner.is_nullable(), + pub fn is_nullable(schema: &Schema) -> bool { + match schema { + Schema::Union(inner) => inner.is_nullable(), _ => false, } } @@ -1548,16 +1548,16 @@ mod schema { /// /// Extra arguments: /// - `known_schemata` - mapping between `Name` and `Schema` - if passed, additional external schemas would be used to resolve references. - pub async fn find_schema_with_known_schemata( - &self, - value: &Value, - known_schemata: &HashMap, - enclosing_namespace: &Namespace, - ) -> Option<(usize, &Schema)> { + pub async fn find_schema_with_known_schemata<'a>( + union_schema: &'a UnionSchema, + value: &'a Value, + known_schemata: &'a HashMap, + enclosing_namespace: &'a Namespace, + ) -> Option<(usize, &'a Schema)> { let schema_kind = SchemaKind::from(value); - if let Some(&i) = self.variant_index.get(&schema_kind) { + if let Some(&i) = &union_schema.variant_index.get(&schema_kind) { // fast path - Some((i, &self.schemas[i])) + Some((i, &union_schema.schemas[i])) } else { // slow path (required for matching logical or named types) @@ -1568,7 +1568,7 @@ mod schema { .collect(); let mut i: usize = 0; - for schema in self.schemas.iter() { + for schema in union_schema.schemas.iter() { let resolved_schema = ResolvedSchema::new_with_known_schemata( vec![schema], enclosing_namespace, @@ -1581,7 +1581,7 @@ mod schema { collected_names.extend(resolved_names.clone()); let namespace = &schema.namespace().or_else(|| enclosing_namespace.clone()); - if ValueExt::resolve_internal(&value, schema, &collected_names, namespace, &None) + if ValueExt::resolve_internal(value, schema, &collected_names, namespace, &None) .await .is_ok() { @@ -2275,7 +2275,7 @@ mod schema { let mut position = 0; for field in fields.iter() { if let Some(field) = field.as_object() { - let record_field = Box::pin(RecordField::parse( + let record_field = Box::pin(RecordFieldExt::parse( field, position, self, @@ -2378,13 +2378,13 @@ mod schema { } } - if let Some(ref value) = default { - let value = Value::from(value.clone()); - let resolved = ValueExt::resolve_enum(&value, &symbols, &Some(value.to_string()), &None) + if let Some(ref json_value) = default { + let value = Value::from(json_value.clone()); + let resolved = ValueExt::resolve_enum(&value, &symbols, &Some(json_value.to_string()), &None) .is_ok(); if !resolved { return Err(Details::GetEnumDefault { - symbol: value.to_string(), + symbol: json_value.to_string(), symbols, } .into()); @@ -2867,7 +2867,7 @@ mod schema { .position(1) .build(); - assert!(nullable_record_field.is_nullable()); + assert!(RecordFieldExt::is_nullable(nullable_record_field)); let non_nullable_record_field = RecordField::builder() .name("next".to_string()) @@ -2877,7 +2877,7 @@ mod schema { .position(1) .build(); - assert!(!non_nullable_record_field.is_nullable()); + assert!(!RecordFieldExt::is_nullable(non_nullable_record_field)); Ok(()) } @@ -6924,9 +6924,10 @@ mod schema { Ok(()) } - #[tokio::test] + #[test] + #[synca::cfg(sync)] #[serial(serde_is_human_readable)] - async fn avro_rs_53_uuid_with_fixed() -> TestResult { + fn avro_rs_53_uuid_with_fixed() -> TestResult { #[derive(Debug, Serialize, Deserialize)] struct Comment { id: Uuid, @@ -6934,7 +6935,7 @@ mod schema { #[cfg_attr(feature = "tokio", async_trait::async_trait)] impl AvroSchema for Comment { - async fn get_schema() -> Schema { + fn get_schema() -> Schema { SchemaExt::parse_str( r#"{ "type" : "record", @@ -6963,14 +6964,12 @@ mod schema { // serialize the Uuid as String crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let bytes = SpecificSingleObjectWriter::::with_capacity(64) - .await? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 47); // serialize the Uuid as Bytes crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); let bytes = SpecificSingleObjectWriter::::with_capacity(64) - .await? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 27); diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 416132f8..652a6cf9 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -19,8 +19,8 @@ //! which writes directly to a `Write` stream use crate::{ - bigdecimal::tokio::big_decimal_as_bytes, - encode::tokio::{encode_int, encode_long}, + bigdecimal::sync::big_decimal_as_bytes, + encode::sync::{encode_int, encode_long}, error::{Details, Error}, schema::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, }; @@ -1824,7 +1824,7 @@ #[cfg(test)] mod tests { use super::*; - use crate::schema::tokio::SchemaExt; + use crate::schema::sync::SchemaExt; use crate::{ Days, Duration, Millis, Months, decimal::Decimal, error::Details, schema::ResolvedSchema, @@ -2009,7 +2009,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_record() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2062,7 +2062,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_empty_record() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2115,7 +2115,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_enum() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2164,7 +2164,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_array() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2203,7 +2203,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_map() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2250,7 +2250,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_nullable_union() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2299,7 +2299,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_union() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2351,7 +2351,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_fixed() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2428,7 +2428,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_decimal_bytes() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2466,7 +2466,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_decimal_fixed() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2506,7 +2506,7 @@ Ok(()) } - #[tokio::test] + #[test] #[serial(serde_is_human_readable)] fn test_serialize_bigdecimal() -> TestResult { let schema = SchemaExt::parse_str( @@ -2531,7 +2531,7 @@ Ok(()) } - #[tokio::test] + #[test] #[serial(serde_is_human_readable)] fn test_serialize_uuid() -> TestResult { let schema = SchemaExt::parse_str( @@ -2577,7 +2577,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_date() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2622,7 +2622,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_time_millis() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2667,7 +2667,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_time_micros() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2715,7 +2715,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_timestamp() -> TestResult { for precision in ["millis", "micros", "nanos"] { let schema = SchemaExt::parse_str(&format!( @@ -2774,7 +2774,7 @@ Ok(()) } - #[tokio::test] + #[test] fn test_serialize_duration() -> TestResult { let schema = SchemaExt::parse_str( r#"{ @@ -2820,7 +2820,7 @@ Ok(()) } - #[tokio::test] + #[test] #[serial(serde_is_human_readable)] // for BigDecimal and Uuid fn test_serialize_recursive_record() -> TestResult { let schema = SchemaExt::parse_str( diff --git a/avro/src/types.rs b/avro/src/types.rs index ca250d59..ec6cdbbd 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -17,11 +17,10 @@ //! Logic handling the intermediate representation of Avro values. -use crate::Decimal; -use crate::Duration; -use crate::Uuid; +use crate::{AvroResult, Decimal, Duration, Uuid}; use crate::bigdecimal::BigDecimal; use crate::schema::{RecordSchema, Schema}; +use crate::error::{Details, Error}; use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, @@ -258,6 +257,92 @@ impl From for Value { } } +macro_rules! to_value( + ($type:ty, $variant_constructor:expr) => ( + impl From<$type> for Value { + fn from(value: $type) -> Self { + $variant_constructor(value) + } + } + ); +); + +to_value!(bool, Value::Boolean); +to_value!(i32, Value::Int); +to_value!(i64, Value::Long); +to_value!(f32, Value::Float); +to_value!(f64, Value::Double); +to_value!(String, Value::String); +to_value!(Vec, Value::Bytes); +to_value!(uuid::Uuid, Value::Uuid); +to_value!(Decimal, Value::Decimal); +to_value!(BigDecimal, Value::BigDecimal); +to_value!(Duration, Value::Duration); + + +/// Convert Avro values to Json values +impl TryFrom for serde_json::Value { + type Error = Error; + fn try_from(value: Value) -> AvroResult { + match value { + Value::Null => Ok(serde_json::Value::Null), + Value::Boolean(b) => Ok(serde_json::Value::Bool(b)), + Value::Int(i) => Ok(serde_json::Value::Number(i.into())), + Value::Long(l) => Ok(serde_json::Value::Number(l.into())), + Value::Float(f) => serde_json::Number::from_f64(f.into()) + .map(Self::Number) + .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), + Value::Double(d) => serde_json::Number::from_f64(d) + .map(Self::Number) + .ok_or_else(|| Details::ConvertF64ToJson(d).into()), + Value::Bytes(bytes) => { + Ok(serde_json::Value::Array(bytes.into_iter().map(|b| b.into()).collect())) + } + Value::String(s) => Ok(serde_json::Value::String(s)), + Value::Fixed(_size, items) => { + Ok(serde_json::Value::Array(items.into_iter().map(|v| v.into()).collect())) + } + Value::Enum(_i, s) => Ok(serde_json::Value::String(s)), + Value::Union(_i, b) => Self::try_from(*b), + Value::Array(items) => items + .into_iter() + .map(Self::try_from) + .collect::, _>>() + .map(Self::Array), + Value::Map(items) => items + .into_iter() + .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) + .collect::, _>>() + .map(|v| Self::Object(v.into_iter().collect())), + Value::Record(items) => items + .into_iter() + .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) + .collect::, _>>() + .map(|v| Self::Object(v.into_iter().collect())), + Value::Date(d) => Ok(serde_json::Value::Number(d.into())), + Value::Decimal(ref d) => >::try_from(d) + .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), + Value::BigDecimal(ref bg) => { + let vec1: Vec = crate::bigdecimal::sync::serialize_big_decimal(bg)?; + Ok(serde_json::Value::Array(vec1.into_iter().map(|b| b.into()).collect())) + } + Value::TimeMillis(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimeMicros(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimestampMillis(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimestampMicros(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimestampNanos(t) => Ok(serde_json::Value::Number(t.into())), + Value::LocalTimestampMillis(t) => Ok(serde_json::Value::Number(t.into())), + Value::LocalTimestampMicros(t) => Ok(serde_json::Value::Number(t.into())), + Value::LocalTimestampNanos(t) => Ok(serde_json::Value::Number(t.into())), + Value::Duration(d) => Ok(serde_json::Value::Array( + <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), + )), + Value::Uuid(uuid) => Ok(serde_json::Value::String(uuid.as_hyphenated().to_string())), + } + } +} + + #[synca::synca( #[cfg(feature = "tokio")] pub mod tokio { }, @@ -295,101 +380,19 @@ mod types { DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, }, + schema::tokio::{UnionSchemaExt, RecordFieldExt}, types::Value, }; use log::{debug, error}; use std::collections::HashMap; use std::str::FromStr; - /// Convert Avro values to Json values - impl TryFrom for serde_json::Value { - type Error = Error; - fn try_from(value: Value) -> AvroResult { - match value { - Value::Null => Ok(Self::Null), - Value::Boolean(b) => Ok(Self::Bool(b)), - Value::Int(i) => Ok(Self::Number(i.into())), - Value::Long(l) => Ok(Self::Number(l.into())), - Value::Float(f) => serde_json::Number::from_f64(f.into()) - .map(Self::Number) - .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), - Value::Double(d) => serde_json::Number::from_f64(d) - .map(Self::Number) - .ok_or_else(|| Details::ConvertF64ToJson(d).into()), - Value::Bytes(bytes) => { - Ok(Self::Array(bytes.into_iter().map(|b| b.into()).collect())) - } - Value::String(s) => Ok(Self::String(s)), - Value::Fixed(_size, items) => { - Ok(Self::Array(items.into_iter().map(|v| v.into()).collect())) - } - Value::Enum(_i, s) => Ok(Self::String(s)), - Value::Union(_i, b) => Self::try_from(*b), - Value::Array(items) => items - .into_iter() - .map(Self::try_from) - .collect::, _>>() - .map(Self::Array), - Value::Map(items) => items - .into_iter() - .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) - .collect::, _>>() - .map(|v| Self::Object(v.into_iter().collect())), - Value::Record(items) => items - .into_iter() - .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) - .collect::, _>>() - .map(|v| Self::Object(v.into_iter().collect())), - Value::Date(d) => Ok(Self::Number(d.into())), - Value::Decimal(ref d) => >::try_from(d) - .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), - Value::BigDecimal(ref bg) => { - let vec1: Vec = serialize_big_decimal(bg)?; - Ok(Self::Array(vec1.into_iter().map(|b| b.into()).collect())) - } - Value::TimeMillis(t) => Ok(Self::Number(t.into())), - Value::TimeMicros(t) => Ok(Self::Number(t.into())), - Value::TimestampMillis(t) => Ok(Self::Number(t.into())), - Value::TimestampMicros(t) => Ok(Self::Number(t.into())), - Value::TimestampNanos(t) => Ok(Self::Number(t.into())), - Value::LocalTimestampMillis(t) => Ok(Self::Number(t.into())), - Value::LocalTimestampMicros(t) => Ok(Self::Number(t.into())), - Value::LocalTimestampNanos(t) => Ok(Self::Number(t.into())), - Value::Duration(d) => Ok(Self::Array( - <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), - )), - Value::Uuid(uuid) => Ok(Self::String(uuid.as_hyphenated().to_string())), - } - } - } - /// Compute the maximum decimal value precision of a byte array of length `len` could hold. fn max_prec_for_len(len: usize) -> Result { let len = i32::try_from(len).map_err(|e| Details::ConvertLengthToI32(e, len))?; Ok((2.0_f64.powi(8 * len - 1) - 1.0).log10().floor() as usize) } - macro_rules! to_value( - ($type:ty, $variant_constructor:expr) => ( - impl From<$type> for Value { - fn from(value: $type) -> Self { - $variant_constructor(value) - } - } - ); -); - - to_value!(bool, Value::Boolean); - to_value!(i32, Value::Int); - to_value!(i64, Value::Long); - to_value!(f32, Value::Float); - to_value!(f64, Value::Double); - to_value!(String, Value::String); - to_value!(Vec, Value::Bytes); - to_value!(uuid::Uuid, Value::Uuid); - to_value!(Decimal, Value::Decimal); - to_value!(BigDecimal, Value::BigDecimal); - to_value!(Duration, Value::Duration); pub struct ValueExt; @@ -554,8 +557,7 @@ mod types { } } (v, Schema::Union(inner)) => { - match inner - .find_schema_with_known_schemata(v, names, enclosing_namespace) + match UnionSchemaExt::find_schema_with_known_schemata(inner, v, names, enclosing_namespace) .await { Some(_) => None, @@ -565,7 +567,7 @@ mod types { (Value::Array(items), Schema::Array(inner)) => { let mut acc = None; for item in items.iter() { - acc = Value::accumulate( + acc = ValueExt::accumulate( acc, Box::pin(ValueExt::validate_internal(&item, &inner.items, names, @@ -576,7 +578,7 @@ mod types { } acc // items.iter().fold(None, |acc, item| { - // Value::accumulate( + // ValueExt::accumulate( // acc, // item.validate_internal(&inner.items, names, enclosing_namespace).await, // ) @@ -585,7 +587,7 @@ mod types { (Value::Map(items), Schema::Map(inner)) => { let mut acc = None; for (_, value) in items.iter() { - acc = Value::accumulate( + acc = ValueExt::accumulate( acc, Box::pin(ValueExt::validate_internal(&value, &inner.types, @@ -597,7 +599,7 @@ mod types { } acc // items.iter().fold(None, |acc, (_, value)| { - // Value::accumulate( + // ValueExt::accumulate( // acc, // value.validate_internal(&inner.types, names, enclosing_namespace), // ) @@ -613,7 +615,7 @@ mod types { }), ) => { let non_nullable_fields_count = - fields.iter().filter(|&rf| !rf.is_nullable()).count(); + fields.iter().filter(|&rf| !RecordFieldExt::is_nullable(&rf.schema)).count(); // If the record contains fewer fields as required fields by the schema, it is invalid. if record_fields.len() < non_nullable_fields_count { @@ -640,7 +642,7 @@ mod types { acc = match lookup.get(field_name) { Some(idx) => { let field = &fields[*idx]; - Value::accumulate( + ValueExt::accumulate( acc, Box::pin(ValueExt::validate_internal(&record_field, &field.schema, @@ -650,7 +652,7 @@ mod types { .await, ) } - None => Value::accumulate( + None => ValueExt::accumulate( acc, Some(format!("There is no schema field for field '{field_name}'")), ), @@ -669,9 +671,9 @@ mod types { enclosing_namespace, )) .await; - acc = Value::accumulate(acc, res); - } else if !field.is_nullable() { - acc = Value::accumulate( + acc = ValueExt::accumulate(acc, res); + } else if !RecordFieldExt::is_nullable(&field.schema) { + acc = ValueExt::accumulate( acc, Some(format!( "Field with name '{:?}' is not a member of the map items", @@ -715,16 +717,17 @@ mod types { } else { ResolvedSchema::try_from(schemata)? }; - ValueExt::resolve_internal(value.clone(), schema, rs.get_names(), &enclosing_namespace, &None).await + ValueExt::resolve_internal(&value, schema, rs.get_names(), &enclosing_namespace, &None).await } pub(crate) async fn resolve_internal( - value: &mut Value, + value: &Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, ) -> AvroResult { + let mut value = value; // Check if this schema is a union, and if the reader schema is not. if SchemaKind::from(value) == SchemaKind::Union && SchemaKind::from(schema) != SchemaKind::Union @@ -734,7 +737,7 @@ mod types { Value::Union(_i, b) => *b, _ => unreachable!(), }; - value = v; + value = &v; } match *schema { Schema::Ref { ref name } => { @@ -802,8 +805,8 @@ mod types { } } - fn resolve_uuid(value: Value) -> Result { - Ok(match value { + fn resolve_uuid(value: &Value) -> AvroResult { + Ok(match value.clone() { uuid @ Value::Uuid(_) => uuid, Value::String(ref string) => { Value::Uuid(Uuid::from_str(string).map_err(Details::ConvertStrToUuid)?) @@ -812,16 +815,16 @@ mod types { }) } - async fn resolve_bigdecimal(value: Value) -> Result { - Ok(match value { + async fn resolve_bigdecimal(value: &Value) -> AvroResult { + Ok(match value.clone() { bg @ Value::BigDecimal(_) => bg, Value::Bytes(b) => Value::BigDecimal(deserialize_big_decimal(&b).await.unwrap()), other => return Err(Details::GetBigDecimal(other).into()), }) } - fn resolve_duration(value: Value) -> Result { - Ok(match value { + fn resolve_duration(value: &Value) -> AvroResult { + Ok(match value.clone() { duration @ Value::Duration { .. } => duration, Value::Fixed(size, bytes) => { if size != 12 { @@ -841,7 +844,7 @@ mod types { precision: Precision, scale: Scale, inner: &Schema, - ) -> Result { + ) -> AvroResult { if scale > precision { return Err(Details::GetScaleAndPrecision { scale, precision }.into()); } @@ -854,7 +857,7 @@ mod types { Schema::Bytes => (), _ => return Err(Details::ResolveDecimalSchema(inner.into()).into()), }; - match value { + match value.clone() { Value::Decimal(num) => { let num_bytes = num.len(); if max_prec_for_len(num_bytes)? < precision { @@ -884,54 +887,54 @@ mod types { } } - fn resolve_date(value: Value) -> AvroResult { - match value { + fn resolve_date(value: &Value) -> AvroResult { + match value.clone() { Value::Date(d) | Value::Int(d) => Ok(Value::Date(d)), other => Err(Details::GetDate(other).into()), } } - fn resolve_time_millis(value: Value) -> AvroResult { - match value { + fn resolve_time_millis(value: &Value) -> AvroResult { + match value.clone() { Value::TimeMillis(t) | Value::Int(t) => Ok(Value::TimeMillis(t)), other => Err(Details::GetTimeMillis(other).into()), } } - fn resolve_time_micros(value: Value) -> AvroResult { - match value { + fn resolve_time_micros(value: &Value) -> AvroResult { + match value.clone() { Value::TimeMicros(t) | Value::Long(t) => Ok(Value::TimeMicros(t)), Value::Int(t) => Ok(Value::TimeMicros(i64::from(t))), other => Err(Details::GetTimeMicros(other).into()), } } - fn resolve_timestamp_millis(value: Value) -> AvroResult { - match value { + fn resolve_timestamp_millis(value: &Value) -> AvroResult { + match value.clone() { Value::TimestampMillis(ts) | Value::Long(ts) => Ok(Value::TimestampMillis(ts)), Value::Int(ts) => Ok(Value::TimestampMillis(i64::from(ts))), other => Err(Details::GetTimestampMillis(other).into()), } } - fn resolve_timestamp_micros(value: Value) -> AvroResult { - match value { + fn resolve_timestamp_micros(value: &Value) -> AvroResult { + match value.clone() { Value::TimestampMicros(ts) | Value::Long(ts) => Ok(Value::TimestampMicros(ts)), Value::Int(ts) => Ok(Value::TimestampMicros(i64::from(ts))), other => Err(Details::GetTimestampMicros(other).into()), } } - fn resolve_timestamp_nanos(value: Value) -> AvroResult { - match value { + fn resolve_timestamp_nanos(value: &Value) -> AvroResult { + match value.clone() { Value::TimestampNanos(ts) | Value::Long(ts) => Ok(Value::TimestampNanos(ts)), Value::Int(ts) => Ok(Value::TimestampNanos(i64::from(ts))), other => Err(Details::GetTimestampNanos(other).into()), } } - fn resolve_local_timestamp_millis(value: Value) -> AvroResult { - match value { + fn resolve_local_timestamp_millis(value: &Value) -> AvroResult { + match value.clone() { Value::LocalTimestampMillis(ts) | Value::Long(ts) => { Ok(Value::LocalTimestampMillis(ts)) } @@ -940,8 +943,8 @@ mod types { } } - fn resolve_local_timestamp_micros(value: Value) -> AvroResult { - match value { + fn resolve_local_timestamp_micros(value: &Value) -> AvroResult { + match value.clone() { Value::LocalTimestampMicros(ts) | Value::Long(ts) => { Ok(Value::LocalTimestampMicros(ts)) } @@ -950,8 +953,8 @@ mod types { } } - fn resolve_local_timestamp_nanos(value: Value) -> AvroResult { - match value { + fn resolve_local_timestamp_nanos(value: &Value) -> AvroResult { + match value.clone() { Value::LocalTimestampNanos(ts) | Value::Long(ts) => { Ok(Value::LocalTimestampNanos(ts)) } @@ -960,59 +963,59 @@ mod types { } } - fn resolve_null(value: Value) -> AvroResult { - match value { + fn resolve_null(value: &Value) -> AvroResult { + match value.clone() { Value::Null => Ok(Value::Null), other => Err(Details::GetNull(other).into()), } } - fn resolve_boolean(value: Value) -> AvroResult { - match value { + fn resolve_boolean(value: &Value) -> AvroResult { + match value.clone() { Value::Boolean(b) => Ok(Value::Boolean(b)), other => Err(Details::GetBoolean(other).into()), } } - fn resolve_int(value: Value) -> AvroResult { - match value { + fn resolve_int(value: &Value) -> AvroResult { + match value.clone() { Value::Int(n) => Ok(Value::Int(n)), Value::Long(n) => Ok(Value::Int(n as i32)), other => Err(Details::GetInt(other).into()), } } - fn resolve_long(value: Value) -> AvroResult { - match value { + fn resolve_long(value: &Value) -> AvroResult { + match value.clone() { Value::Int(n) => Ok(Value::Long(i64::from(n))), Value::Long(n) => Ok(Value::Long(n)), other => Err(Details::GetLong(other).into()), } } - fn resolve_float(value: Value) -> AvroResult { - match value { + fn resolve_float(value: &Value) -> AvroResult { + match value.clone() { Value::Int(n) => Ok(Value::Float(n as f32)), Value::Long(n) => Ok(Value::Float(n as f32)), Value::Float(x) => Ok(Value::Float(x)), Value::Double(x) => Ok(Value::Float(x as f32)), Value::String(ref x) => match Self::parse_special_float(x) { Some(f) => Ok(Value::Float(f)), - None => Err(Details::GetFloat(value).into()), + None => Err(Details::GetFloat(value.clone()).into()), }, other => Err(Details::GetFloat(other).into()), } } - fn resolve_double(value: Value) -> AvroResult { - match value { + fn resolve_double(value: &Value) -> AvroResult { + match value.clone() { Value::Int(n) => Ok(Value::Double(f64::from(n))), Value::Long(n) => Ok(Value::Double(n as f64)), Value::Float(x) => Ok(Value::Double(f64::from(x))), Value::Double(x) => Ok(Value::Double(x)), Value::String(ref x) => match Self::parse_special_float(x) { Some(f) => Ok(Value::Double(f64::from(f))), - None => Err(Details::GetDouble(value).into()), + None => Err(Details::GetDouble(value.clone()).into()), }, other => Err(Details::GetDouble(other).into()), } @@ -1029,8 +1032,8 @@ mod types { } } - async fn resolve_bytes(value: Value) -> AvroResult { - match value { + async fn resolve_bytes(value: &Value) -> AvroResult { + match value.clone() { Value::Bytes(bytes) => Ok(Value::Bytes(bytes)), Value::String(s) => Ok(Value::Bytes(s.into_bytes())), Value::Array(items) => { @@ -1045,8 +1048,8 @@ mod types { } } - fn resolve_string(value: Value) -> AvroResult { - match value { + fn resolve_string(value: &Value) -> AvroResult { + match value.clone() { Value::String(s) => Ok(Value::String(s)), Value::Bytes(bytes) | Value::Fixed(_, bytes) => Ok(Value::String( String::from_utf8(bytes).map_err(Details::ConvertToUtf8)?, @@ -1055,8 +1058,8 @@ mod types { } } - fn resolve_fixed(value: Value, size: usize) -> AvroResult { - match value { + fn resolve_fixed(value: &Value, size: usize) -> AvroResult { + match value.clone() { Value::Fixed(n, bytes) => { if n == size { Ok(Value::Fixed(n, bytes)) @@ -1077,7 +1080,7 @@ mod types { } pub(crate) fn resolve_enum( - value: Value, + value: &Value, symbols: &[String], enum_default: &Option, _field_default: &Option, @@ -1107,7 +1110,7 @@ mod types { } }; - match value { + match value.clone() { Value::Enum(_raw_index, s) => validate_symbol(s, symbols), Value::String(s) => validate_symbol(s, symbols), other => Err(Details::GetEnum(other).into()), @@ -1115,20 +1118,19 @@ mod types { } async fn resolve_union( - value: Value, + value: &Value, schema: &UnionSchema, names: &HashMap, enclosing_namespace: &Namespace, field_default: &Option, ) -> AvroResult { - let v = match value { + let v = match value.clone() { // Both are unions case. Value::Union(_i, v) => *v, // Reader is a union, but writer is not. v => v, }; - let (i, inner) = schema - .find_schema_with_known_schemata(&v, names, enclosing_namespace) + let (i, inner) = UnionSchemaExt::find_schema_with_known_schemata(schema, &v, names, enclosing_namespace) .await .ok_or_else(|| Details::FindUnionVariant { schema: schema.clone(), @@ -1145,15 +1147,15 @@ mod types { } async fn resolve_array( - value: Value, + value: &Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, ) -> AvroResult { - match value { + match value.clone() { Value::Array(items) => { let mut resolved_values = Vec::with_capacity(items.len()); - for item in items.into_iter() { + for item in items.iter() { let resolved = Box::pin(ValueExt::resolve_internal( item, schema, @@ -1175,15 +1177,15 @@ mod types { } async fn resolve_map( - value: Value, + value: &Value, schema: &Schema, names: &HashMap, enclosing_namespace: &Namespace, ) -> AvroResult { - match value { + match value.clone() { Value::Map(items) => { let mut resolved = HashMap::with_capacity(items.len()); - for (key, value) in items.into_iter() { + for (key, value) in items.iter() { let v = Box::pin(ValueExt::resolve_internal( value, schema, @@ -1216,12 +1218,12 @@ mod types { } async fn resolve_record( - self, + value: &Value, fields: &[RecordField], names: &HashMap, enclosing_namespace: &Namespace, - ) -> Result { - let mut items = match self { + ) -> AvroResult { + let mut items = match value.clone() { Value::Map(items) => Ok(items), Value::Record(fields) => Ok(fields.into_iter().collect::>()), other => Err(Error::new(Details::GetRecord { @@ -1238,13 +1240,13 @@ mod types { let value = match items.remove(&field.name) { Some(value) => value, None => match field.default { - Some(ref value) => match field.schema { + Some(ref json_value) => match field.schema { Schema::Enum(EnumSchema { ref symbols, ref default, .. }) => ValueExt::resolve_enum( - &Value::from(value.clone()), + &Value::from(json_value.clone()), symbols, default, &field.default.clone(), @@ -1259,7 +1261,7 @@ mod types { 0, Box::new( Box::pin(ValueExt::resolve_internal( - Value::from(value.clone()), + &Value::from(json_value.clone()), first, names, enclosing_namespace, @@ -1270,7 +1272,7 @@ mod types { ), } } - _ => Value::from(value.clone()), + _ => Value::from(json_value.clone()), }, None => { return Err(Details::GetField(field.name.clone()).into()); @@ -1279,7 +1281,7 @@ mod types { }; let v = Box::pin(ValueExt::resolve_internal( - value, + &value, &field.schema, names, enclosing_namespace, @@ -1292,7 +1294,7 @@ mod types { Ok(Value::Record(new_fields)) } - async fn try_u8(value: &Value) -> AvroResult { + async fn try_u8(value: Value) -> AvroResult { let int = ValueExt::resolve(value, &Schema::Int).await?; if let Value::Int(n) = int { if n >= 0 && n <= i32::from(u8::MAX) { @@ -1795,19 +1797,19 @@ Field with name '"b"' is not a member of the map items"#, .await ); - assert!( - Value::Union( - 1, - Box::new(Value::Map( - vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ] + let value = Value::Union( + 1, + Box::new(Value::Map( + vec![ + ("a".to_string(), Value::Long(42i64)), + ("b".to_string(), Value::String("foo".to_string())), + ] .into_iter() .collect() - )) - ) - .validate(&union_schema) + )) + ); + assert!( + ValueExt::validate(value, &union_schema) .await ); @@ -1981,9 +1983,8 @@ Field with name '"b"' is not a member of the map items"#, async fn test_avro_3853_resolve_timestamp_millis() { let value = Value::LocalTimestampMillis(10); assert!( - value - .clone() - .resolve(&Schema::LocalTimestampMillis) + ValueExt::resolve(value + .clone(), &Schema::LocalTimestampMillis) .await .is_ok() ); diff --git a/avro/src/writer.rs b/avro/src/writer.rs index d506e8c6..8724275b 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -58,10 +58,14 @@ mod writer { error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, schema::tokio::AvroSchema, - schema::{Name, ResolvedOwnedSchema, ResolvedSchema, Schema}, - ser_schema::SchemaAwareWriteSerializer, + schema::{ResolvedOwnedSchema, ResolvedSchema, Schema}, types::Value, }; + #[synca::cfg(sync)] + use crate::schema::Name; + #[synca::cfg(sync)] + use crate::ser_schema::SchemaAwareWriteSerializer; + #[synca::cfg(sync)] use serde::Serialize; use std::{ collections::HashMap, marker::PhantomData, mem::ManuallyDrop, @@ -201,7 +205,7 @@ mod writer { /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). pub async fn append>(&mut self, value: T) -> AvroResult { - let n = self.maybe_write_header()?; + let n = self.maybe_write_header().await?; let avro = value.into(); self.append_value_ref(&avro).await.map(|m| m + n) @@ -246,8 +250,9 @@ mod writer { /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). - pub async fn append_ser(&mut self, value: S) -> AvroResult { - let n = self.maybe_write_header().await?; + #[synca::cfg(sync)] + pub fn append_ser(&mut self, value: S) -> AvroResult { + let n = self.maybe_write_header()?; match self.resolved_schema { Some(ref rs) => { @@ -261,7 +266,7 @@ mod writer { self.num_values += 1; if self.buffer.len() >= self.block_size { - return self.flush().await.map(|b| b + n); + return self.flush().map(|b| b + n); } Ok(n) @@ -316,7 +321,8 @@ mod writer { /// /// **NOTE**: This function forces the written data to be flushed (an implicit /// call to [`flush`](Writer::flush) is performed). - pub async fn extend_ser(&mut self, values: I) -> AvroResult + #[synca::cfg(sync)] + pub fn extend_ser(&mut self, values: I) -> AvroResult where I: IntoIterator, { @@ -338,7 +344,7 @@ mod writer { for value in values { num_bytes += self.append_ser(value)?; } - num_bytes += self.flush().await?; + num_bytes += self.flush()?; Ok(num_bytes) } @@ -682,13 +688,14 @@ mod writer { } } + #[synca::cfg(sync)] impl SpecificSingleObjectWriter where T: AvroSchema + Serialize, { /// Write the referenced `Serialize` object to the provided `Write` object. Returns a result with /// the number of bytes written including the header - pub async fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { + pub async fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { let mut bytes_written: usize = 0; if !self.header_written { @@ -788,7 +795,8 @@ mod writer { /// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync /// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible /// if you don't know what you are doing, instead. - pub fn write_avro_datum_ref( + #[synca::cfg(sync)] + pub fn write_avro_datum_ref( schema: &Schema, data: &T, writer: &mut W, @@ -836,6 +844,7 @@ mod writer { #[cfg(test)] mod tests { + #[synca::cfg(sync)] use std::{cell::RefCell, rc::Rc}; use super::*; @@ -898,8 +907,9 @@ mod writer { Ok(()) } - #[tokio::test] - async fn avro_rs_193_write_avro_datum_ref() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn avro_rs_193_write_avro_datum_ref() -> TestResult { #[derive(Serialize)] struct TestStruct { a: i64, @@ -1201,9 +1211,10 @@ mod writer { b: String, } - #[tokio::test] - async fn test_writer_append_ser() -> TestResult { - let schema = SchemaExt::parse_str(SCHEMA).await?; + #[test] + #[synca::cfg(sync)] + fn test_writer_append_ser() -> TestResult { + let schema = SchemaExt::parse_str(SCHEMA)?; let mut writer = Writer::new(&schema, Vec::new()); let record = TestSerdeSerialize { @@ -1212,14 +1223,14 @@ mod writer { }; let n1 = writer.append_ser(record)?; - let n2 = writer.flush().await?; - let result = writer.into_inner().await?; + let n2 = writer.flush()?; + let result = writer.into_inner()?; assert_eq!(n1 + n2, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data).await?; - zig_i64(3, &mut data).await?; + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; data.extend(b"foo"); // starts with magic @@ -1234,8 +1245,9 @@ mod writer { Ok(()) } - #[tokio::test] - async fn test_writer_extend_ser() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn test_writer_extend_ser() -> TestResult { let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); @@ -1246,15 +1258,15 @@ mod writer { let record_copy = record.clone(); let records = vec![record, record_copy]; - let n1 = writer.extend_ser(records).await?; - let n2 = writer.flush().await?; - let result = writer.into_inner().await?; + let n1 = writer.extend_ser(records)?; + let n2 = writer.flush()?; + let result = writer.into_inner()?; assert_eq!(n1 + n2, result.len()); let mut data = Vec::new(); - zig_i64(27, &mut data).await?; - zig_i64(3, &mut data).await?; + zig_i64(27, &mut data)?; + zig_i64(3, &mut data)?; data.extend(b"foo"); data.extend(data.clone()); @@ -1637,8 +1649,9 @@ mod writer { Ok(()) } - #[tokio::test] - async fn test_writer_parity() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn test_writer_parity() -> TestResult { let obj1 = TestSingleObjectWriter { a: 300, b: 34.555, @@ -1650,7 +1663,7 @@ mod writer { let mut buf3: Vec = Vec::new(); let mut generic_writer = GenericSingleObjectWriter::new_with_capacity( - &TestSingleObjectWriter::get_schema().await, + &TestSingleObjectWriter::get_schema(), 1024, ) .expect("Should resolve schema"); @@ -1659,15 +1672,13 @@ mod writer { .await .expect("Resolved should pass"); specific_writer - .write(obj1.clone(), &mut buf1).await + .write(obj1.clone(), &mut buf1) .expect("Serialization expected"); specific_writer .write_value(obj1.clone(), &mut buf2) - .await .expect("Serialization expected"); generic_writer .write_value(obj1.into(), &mut buf3) - .await .expect("Serialization expected"); assert_eq!(buf1, buf2); assert_eq!(buf1, buf3); @@ -1675,8 +1686,9 @@ mod writer { Ok(()) } - #[tokio::test] - async fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { const SCHEMA: &str = r#" { "type": "record", @@ -1698,7 +1710,7 @@ mod writer { time: Some(1234567890), }; - let schema = SchemaExt::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA)?; let mut writer = Writer::new(&schema, Vec::new()); let bytes = writer.append_ser(conf)?; @@ -1708,8 +1720,9 @@ mod writer { Ok(()) } - #[tokio::test] - async fn avro_4014_validation_returns_a_detailed_error() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn avro_4014_validation_returns_a_detailed_error() -> TestResult { const SCHEMA: &str = r#" { "type": "record", @@ -1731,7 +1744,7 @@ mod writer { time: Some(12345678.90), }; - let schema = SchemaExt::parse_str(SCHEMA).await?; + let schema = SchemaExt::parse_str(SCHEMA)?; let mut writer = Writer::new(&schema, Vec::new()); match writer.append_ser(conf) { @@ -1770,7 +1783,7 @@ mod writer { impl std::io::Write for TestBuffer { fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.borrow_mut().write(buf).await + std::io::Write::write(&mut self.0.borrow_mut(), buf) } fn flush(&mut self) -> std::io::Result<()> { From 5f2967e6808584010d8809eff3e5b391c74de97e Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 09:19:54 +0300 Subject: [PATCH 22/47] Optimize imports Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bigdecimal.rs | 2 -- avro/src/de.rs | 2 +- avro/src/decode.rs | 2 +- avro/src/schema.rs | 12 +++--------- avro/src/types.rs | 5 +---- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 5bc93d9d..118acdbc 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -80,8 +80,6 @@ mod bigdecimal { #[synca::cfg(sync)] use std::io::Read; #[synca::cfg(tokio)] - use tokio::io::AsyncRead; - #[synca::cfg(tokio)] use tokio::io::AsyncReadExt; bytes diff --git a/avro/src/de.rs b/avro/src/de.rs index ec5f3347..d524fead 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -40,7 +40,6 @@ )] mod de { - use crate::schema::tokio::SchemaExt; use crate::{ bytes::DE_BYTES_BORROWED, error::Details, error::Error, schema::SchemaKind, types::Value, }; @@ -798,6 +797,7 @@ mod de { #[cfg(test)] mod tests { + use crate::schema::tokio::SchemaExt; use crate::reader::tokio::from_avro_datum; use crate::ser::tokio::to_value; use crate::writer::tokio::to_avro_datum; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 0198401c..06f1d25d 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -56,7 +56,7 @@ mod decode { }, types::Value, }; - use std::{borrow::Borrow, collections::HashMap, io::ErrorKind, str::FromStr}; + use std::{borrow::Borrow, collections::HashMap, io::ErrorKind}; #[inline] pub(crate) async fn decode_long(reader: &mut R) -> AvroResult { diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 6f65a0e1..3ac8d78b 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1366,29 +1366,23 @@ mod schema { use crate::AvroResult; use crate::error::{Details, Error}; use crate::schema::{ - Alias, Aliases, DecimalMetadata, DecimalSchema, Documentation, EnumSchema, - FixedSchema, MapSchema, Name, Names, Namespace, Precision, RecordField, RecordFieldOrder, - RecordSchema, ResolvedSchema, Scale, Schema, SchemaFingerprint, SchemaKind, UnionSchema, + Alias, Aliases, DecimalMetadata, DecimalSchema, EnumSchema, + FixedSchema, Name, Names, Namespace, Precision, RecordField, RecordFieldOrder, + RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, }; use crate::util::MapHelper; use crate::{ types::{Value, ValueKind}, validator::{ validate_enum_symbol_name, validate_record_field_name, - validate_schema_name, }, }; use log::{debug, error, warn}; - use serde::{ - Serialize, - ser::{SerializeMap, SerializeSeq}, - }; #[synca::cfg(sync)] use std::io::Read as AvroRead; use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, - hash::Hash, str::FromStr, }; #[synca::cfg(tokio)] diff --git a/avro/src/types.rs b/avro/src/types.rs index ec6cdbbd..a7423e97 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -368,10 +368,7 @@ mod types { use crate::AvroResult; use crate::{ Uuid, - bigdecimal::{ - BigDecimal, - tokio::{deserialize_big_decimal, serialize_big_decimal}, - }, + bigdecimal::tokio::deserialize_big_decimal, decimal::Decimal, duration::Duration, error::Details, From a717e13480d4eff3f50c002ffe6ac9f72666a4e8 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 10:47:05 +0300 Subject: [PATCH 23/47] WIP More compilation error fixes Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bytes.rs | 9 +- avro/src/de.rs | 1 - avro/src/headers.rs | 3 +- avro/src/reader.rs | 6 +- avro/src/schema.rs | 32 +- avro/src/schema_compatibility.rs | 4 +- avro/src/ser_schema.rs | 4918 +++++++++++++++--------------- avro/src/types.rs | 8 +- avro/src/writer.rs | 136 +- 9 files changed, 2562 insertions(+), 2555 deletions(-) diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index d10e36db..dadf8dff 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -304,10 +304,9 @@ pub mod serde_avro_slice_opt { mod tests { use super::*; use crate::de::tokio::from_value; - use crate::schema::Schema; use crate::schema::tokio::SchemaExt; use crate::ser::tokio::to_value; - use crate::types::Value; + use crate::types::{{tokio::ValueExt}, Value}; use apache_avro_test_helper::TestResult; use serde::{Deserialize, Serialize}; @@ -339,7 +338,7 @@ mod tests { slice_field: &[1, 2, 3], slice_field_opt: Some(&[1, 2, 3]), }; - let value: Value = to_value(test).unwrap(); + let value: Value = to_value(test)?; let schema = SchemaExt::parse_str( r#" { @@ -502,7 +501,7 @@ mod tests { Value::Union(1, Box::new(Value::Null)), ), ]); - assert_eq!(expected, from_value(&value).unwrap()); + assert_eq!(expected, from_value(&value)?); Ok(()) } @@ -701,7 +700,7 @@ mod tests { Value::Union(0, Box::new(Value::Null)), ), ]); - assert_eq!(expected, to_value(test).unwrap()); + assert_eq!(expected, to_value(test)?); Ok(()) } } diff --git a/avro/src/de.rs b/avro/src/de.rs index d524fead..e3e80aff 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -811,7 +811,6 @@ mod de { use apache_avro_test_helper::TestResult; use crate::decimal::Decimal; - use crate::schema::Schema; use super::*; diff --git a/avro/src/headers.rs b/avro/src/headers.rs index ae78718b..4800c169 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -132,10 +132,11 @@ mod headers { } #[cfg(test)] - mod test { + mod tests { use super::*; use crate::error::{Details, Error}; use apache_avro_test_helper::TestResult; + use crate::schema::tokio::SchemaExt; #[tokio::test] async fn test_rabin_fingerprint_header() -> TestResult { diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 51613ea2..54328fa3 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -1091,7 +1091,11 @@ mod reader { &mut to_read_3, ).await .expect("Encode should succeed"); - let mut to_read = (&to_read_1[..]).chain(&to_read_2[..]).chain(&to_read_3[..]); + + #[synca::cfg(sync)] + let mut to_read = std::io::Read::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); + #[synca::cfg(tokio)] + let mut to_read = tokio::io::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) .expect("Schema should resolve"); diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 3ac8d78b..931bf5db 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1549,7 +1549,7 @@ mod schema { enclosing_namespace: &'a Namespace, ) -> Option<(usize, &'a Schema)> { let schema_kind = SchemaKind::from(value); - if let Some(&i) = &union_schema.variant_index.get(&schema_kind) { + if let Some(&i) = union_schema.variant_index.get(&schema_kind) { // fast path Some((i, &union_schema.schemas[i])) } else { @@ -2753,20 +2753,24 @@ mod schema { mod tests { use super::*; use crate::{ - Uuid, de::tokio::from_value, error::Details, rabin::Rabin, reader::tokio::from_avro_datum, ser::tokio::to_value, - writer::tokio::{SpecificSingleObjectWriter, Writer, to_avro_datum}, + writer::tokio::{Writer, to_avro_datum}, }; + #[synca::cfg(sync)] + use crate::writer::sync::SpecificSingleObjectWriter; use apache_avro_test_helper::{ TestResult, logger::{assert_logged, assert_not_logged}, }; use serde_json::json; + use serde::{Deserialize, Serialize}; + #[synca::cfg(sync)] use serial_test::serial; + #[synca::cfg(sync)] use std::sync::atomic::Ordering; #[tokio::test] @@ -2861,7 +2865,7 @@ mod schema { .position(1) .build(); - assert!(RecordFieldExt::is_nullable(nullable_record_field)); + assert!(RecordFieldExt::is_nullable(&nullable_record_field.schema)); let non_nullable_record_field = RecordField::builder() .name("next".to_string()) @@ -2871,7 +2875,7 @@ mod schema { .position(1) .build(); - assert!(!RecordFieldExt::is_nullable(non_nullable_record_field)); + assert!(!RecordFieldExt::is_nullable(&non_nullable_record_field.schema)); Ok(()) } @@ -5402,8 +5406,8 @@ mod schema { PartialOrd, Ord, Clone, - serde::Deserialize, - serde::Serialize, + Deserialize, + Serialize, )] pub enum Bar { #[serde(rename = "bar0")] @@ -5412,7 +5416,7 @@ mod schema { Bar1, } - #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] pub struct Foo { #[serde(rename = "barInit")] pub bar_init: Bar, @@ -5448,8 +5452,8 @@ mod schema { PartialOrd, Ord, Clone, - serde::Deserialize, - serde::Serialize, + Deserialize, + Serialize, )] pub enum Bar { #[serde(rename = "bar0")] @@ -5460,7 +5464,7 @@ mod schema { Bar2, } - #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] pub struct Foo { #[serde(rename = "barInit")] pub bar_init: Bar, @@ -6924,7 +6928,7 @@ mod schema { fn avro_rs_53_uuid_with_fixed() -> TestResult { #[derive(Debug, Serialize, Deserialize)] struct Comment { - id: Uuid, + id: crate::Uuid, } #[cfg_attr(feature = "tokio", async_trait::async_trait)] @@ -6957,13 +6961,13 @@ mod schema { // serialize the Uuid as String crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let bytes = SpecificSingleObjectWriter::::with_capacity(64) + let bytes = SpecificSingleObjectWriter::::with_capacity(64)? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 47); // serialize the Uuid as Bytes crate::util::SERDE_HUMAN_READABLE.store(false, Ordering::Release); - let bytes = SpecificSingleObjectWriter::::with_capacity(64) + let bytes = SpecificSingleObjectWriter::::with_capacity(64)? .write_ref(&payload, &mut buffer)?; assert_eq!(bytes, 27); diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index 0211333d..eda1aab5 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -1599,7 +1599,7 @@ mod schema_compatibility { record.put("b", "foo"); record.put("c", "clubs"); writer.append(record).await.unwrap(); - let input = writer.into_inner()?; + let input = writer.into_inner().await?; let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; assert_eq!( reader.next().await.unwrap().unwrap(), @@ -1663,7 +1663,7 @@ mod schema_compatibility { record.put("b", "foo"); record.put("c", "hearts"); writer.append(record).await.unwrap(); - let input = writer.into_inner()?; + let input = writer.into_inner().await?; let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; assert_eq!( reader.next().await.unwrap().unwrap(), diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 652a6cf9..81f7da0b 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -18,1424 +18,1415 @@ //! Logic for serde-compatible schema-aware serialization //! which writes directly to a `Write` stream - use crate::{ - bigdecimal::sync::big_decimal_as_bytes, - encode::sync::{encode_int, encode_long}, - error::{Details, Error}, - schema::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, - }; - use bigdecimal::BigDecimal; - use serde::{Serialize, ser}; - use std::{borrow::Cow, io::Write, str::FromStr}; - - const COLLECTION_SERIALIZER_ITEM_LIMIT: usize = 1024; - const COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY: usize = 32; - const SINGLE_VALUE_INIT_BUFFER_SIZE: usize = 128; - - /// The sequence serializer for [`SchemaAwareWriteSerializer`]. - /// [`SchemaAwareWriteSerializeSeq`] may break large arrays up into multiple blocks to avoid having - /// to obtain the length of the entire array before being able to write any data to the underlying - /// [`std::fmt::Write`] stream. (See the - /// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) - /// section of the Avro spec for more info.) - pub struct SchemaAwareWriteSerializeSeq<'a, 's, W: Write> { +use crate::{ + bigdecimal::sync::big_decimal_as_bytes, + encode::sync::{encode_int, encode_long}, + error::{Details, Error}, + schema::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, +}; +use bigdecimal::BigDecimal; +use serde::{Serialize, ser}; +use std::{borrow::Cow, io::Write, str::FromStr}; + +const COLLECTION_SERIALIZER_ITEM_LIMIT: usize = 1024; +const COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY: usize = 32; +const SINGLE_VALUE_INIT_BUFFER_SIZE: usize = 128; + +/// The sequence serializer for [`SchemaAwareWriteSerializer`]. +/// [`SchemaAwareWriteSerializeSeq`] may break large arrays up into multiple blocks to avoid having +/// to obtain the length of the entire array before being able to write any data to the underlying +/// [`std::fmt::Write`] stream. (See the +/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) +/// section of the Avro spec for more info.) +pub struct SchemaAwareWriteSerializeSeq<'a, 's, W: Write> { + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + item_schema: &'s Schema, + item_buffer_size: usize, + item_buffers: Vec>, + bytes_written: usize, +} + +impl<'a, 's, W: Write> SchemaAwareWriteSerializeSeq<'a, 's, W> { + fn new( ser: &'a mut SchemaAwareWriteSerializer<'s, W>, item_schema: &'s Schema, - item_buffer_size: usize, - item_buffers: Vec>, - bytes_written: usize, - } - - impl<'a, 's, W: Write> SchemaAwareWriteSerializeSeq<'a, 's, W> { - fn new( - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - item_schema: &'s Schema, - len: Option, - ) -> SchemaAwareWriteSerializeSeq<'a, 's, W> { - SchemaAwareWriteSerializeSeq { - ser, - item_schema, - item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, - item_buffers: Vec::with_capacity( - len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), - ), - bytes_written: 0, - } + len: Option, + ) -> SchemaAwareWriteSerializeSeq<'a, 's, W> { + SchemaAwareWriteSerializeSeq { + ser, + item_schema, + item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, + item_buffers: Vec::with_capacity( + len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), + ), + bytes_written: 0, } + } - fn write_buffered_items(&mut self) -> Result<(), Error> { - if !self.item_buffers.is_empty() { - self.bytes_written += - encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; - for item in self.item_buffers.drain(..) { - self.bytes_written += self - .ser - .writer - .write(item.as_slice()) - .map_err(Details::WriteBytes)?; - } + fn write_buffered_items(&mut self) -> Result<(), Error> { + if !self.item_buffers.is_empty() { + self.bytes_written += + encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; + for item in self.item_buffers.drain(..) { + self.bytes_written += self + .ser + .writer + .write(item.as_slice()) + .map_err(Details::WriteBytes)?; } - - Ok(()) } - fn serialize_element(&mut self, value: &T) -> Result<(), Error> { - let mut item_buffer: Vec = Vec::with_capacity(self.item_buffer_size); - let mut item_ser = SchemaAwareWriteSerializer::new( - &mut item_buffer, - self.item_schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - value.serialize(&mut item_ser)?; - - self.item_buffer_size = std::cmp::max(self.item_buffer_size, item_buffer.len() + 16); + Ok(()) + } - self.item_buffers.push(item_buffer); + fn serialize_element(&mut self, value: &T) -> Result<(), Error> { + let mut item_buffer: Vec = Vec::with_capacity(self.item_buffer_size); + let mut item_ser = SchemaAwareWriteSerializer::new( + &mut item_buffer, + self.item_schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + value.serialize(&mut item_ser)?; - if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { - self.write_buffered_items()?; - } + self.item_buffer_size = std::cmp::max(self.item_buffer_size, item_buffer.len() + 16); - Ok(()) - } + self.item_buffers.push(item_buffer); - fn end(mut self) -> Result { + if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { self.write_buffered_items()?; - self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; - - Ok(self.bytes_written) } + + Ok(()) } - impl ser::SerializeSeq for SchemaAwareWriteSerializeSeq<'_, '_, W> { - type Ok = usize; - type Error = Error; + fn end(mut self) -> Result { + self.write_buffered_items()?; + self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - self.serialize_element(&value) - } + Ok(self.bytes_written) + } +} - fn end(self) -> Result { - self.end() - } +impl ser::SerializeSeq for SchemaAwareWriteSerializeSeq<'_, '_, W> { + type Ok = usize; + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + self.serialize_element(&value) } - impl ser::SerializeTuple for SchemaAwareWriteSerializeSeq<'_, '_, W> { - type Ok = usize; - type Error = Error; + fn end(self) -> Result { + self.end() + } +} - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - ser::SerializeSeq::serialize_element(self, value) - } +impl ser::SerializeTuple for SchemaAwareWriteSerializeSeq<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn end(self) -> Result { - ser::SerializeSeq::end(self) - } + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeSeq::serialize_element(self, value) } - /// The map serializer for [`SchemaAwareWriteSerializer`]. - /// [`SchemaAwareWriteSerializeMap`] may break large maps up into multiple blocks to avoid having to - /// obtain the size of the entire map before being able to write any data to the underlying - /// [`std::fmt::Write`] stream. (See the - /// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) - /// section of the Avro spec for more info.) - pub struct SchemaAwareWriteSerializeMap<'a, 's, W: Write> { + fn end(self) -> Result { + ser::SerializeSeq::end(self) + } +} + +/// The map serializer for [`SchemaAwareWriteSerializer`]. +/// [`SchemaAwareWriteSerializeMap`] may break large maps up into multiple blocks to avoid having to +/// obtain the size of the entire map before being able to write any data to the underlying +/// [`std::fmt::Write`] stream. (See the +/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization) +/// section of the Avro spec for more info.) +pub struct SchemaAwareWriteSerializeMap<'a, 's, W: Write> { + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + item_schema: &'s Schema, + item_buffer_size: usize, + item_buffers: Vec>, + bytes_written: usize, +} + +impl<'a, 's, W: Write> SchemaAwareWriteSerializeMap<'a, 's, W> { + fn new( ser: &'a mut SchemaAwareWriteSerializer<'s, W>, item_schema: &'s Schema, - item_buffer_size: usize, - item_buffers: Vec>, - bytes_written: usize, - } - - impl<'a, 's, W: Write> SchemaAwareWriteSerializeMap<'a, 's, W> { - fn new( - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - item_schema: &'s Schema, - len: Option, - ) -> SchemaAwareWriteSerializeMap<'a, 's, W> { - SchemaAwareWriteSerializeMap { - ser, - item_schema, - item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, - item_buffers: Vec::with_capacity( - len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), - ), - bytes_written: 0, - } - } - - fn write_buffered_items(&mut self) -> Result<(), Error> { - if !self.item_buffers.is_empty() { - self.bytes_written += - encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; - for item in self.item_buffers.drain(..) { - self.bytes_written += self - .ser - .writer - .write(item.as_slice()) - .map_err(Details::WriteBytes)?; - } - } - - Ok(()) + len: Option, + ) -> SchemaAwareWriteSerializeMap<'a, 's, W> { + SchemaAwareWriteSerializeMap { + ser, + item_schema, + item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE, + item_buffers: Vec::with_capacity( + len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY), + ), + bytes_written: 0, } } - impl ser::SerializeMap for SchemaAwareWriteSerializeMap<'_, '_, W> { - type Ok = usize; - type Error = Error; - - fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - let mut element_buffer: Vec = Vec::with_capacity(self.item_buffer_size); - let string_schema = Schema::String; - let mut key_ser = SchemaAwareWriteSerializer::new( - &mut element_buffer, - &string_schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - key.serialize(&mut key_ser)?; - - self.item_buffers.push(element_buffer); - - Ok(()) - } - - fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - let last_index = self.item_buffers.len() - 1; - let element_buffer = &mut self.item_buffers[last_index]; - let mut val_ser = SchemaAwareWriteSerializer::new( - element_buffer, - self.item_schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - value.serialize(&mut val_ser)?; - - self.item_buffer_size = std::cmp::max(self.item_buffer_size, element_buffer.len() + 16); - - if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { - self.write_buffered_items()?; + fn write_buffered_items(&mut self) -> Result<(), Error> { + if !self.item_buffers.is_empty() { + self.bytes_written += + encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?; + for item in self.item_buffers.drain(..) { + self.bytes_written += self + .ser + .writer + .write(item.as_slice()) + .map_err(Details::WriteBytes)?; } - - Ok(()) } - fn end(mut self) -> Result { - self.write_buffered_items()?; - self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; - - Ok(self.bytes_written) - } + Ok(()) } - - /// The struct serializer for [`SchemaAwareWriteSerializer`], which can serialize Avro records. - /// [`SchemaAwareWriteSerializeStruct`] can accept fields out of order, but doing so incurs a - /// performance penalty, since it requires [`SchemaAwareWriteSerializeStruct`] to buffer serialized - /// values in order to write them to the stream in order. - pub struct SchemaAwareWriteSerializeStruct<'a, 's, W: Write> { - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - record_schema: &'s RecordSchema, - bytes_written: usize, +} + +impl ser::SerializeMap for SchemaAwareWriteSerializeMap<'_, '_, W> { + type Ok = usize; + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + let mut element_buffer: Vec = Vec::with_capacity(self.item_buffer_size); + let string_schema = Schema::String; + let mut key_ser = SchemaAwareWriteSerializer::new( + &mut element_buffer, + &string_schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + key.serialize(&mut key_ser)?; + + self.item_buffers.push(element_buffer); + + Ok(()) } - impl<'a, 's, W: Write> SchemaAwareWriteSerializeStruct<'a, 's, W> { - fn new( - ser: &'a mut SchemaAwareWriteSerializer<'s, W>, - record_schema: &'s RecordSchema, - ) -> SchemaAwareWriteSerializeStruct<'a, 's, W> { - SchemaAwareWriteSerializeStruct { - ser, - record_schema, - bytes_written: 0, - } - } - - fn serialize_next_field(&mut self, field: &RecordField, value: &T) -> Result<(), Error> - where - T: ?Sized + ser::Serialize, - { - // If we receive fields in order, write them directly to the main writer - let mut value_ser = SchemaAwareWriteSerializer::new( - &mut *self.ser.writer, - &field.schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ); - self.bytes_written += value.serialize(&mut value_ser)?; - - Ok(()) + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + let last_index = self.item_buffers.len() - 1; + let element_buffer = &mut self.item_buffers[last_index]; + let mut val_ser = SchemaAwareWriteSerializer::new( + element_buffer, + self.item_schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + value.serialize(&mut val_ser)?; + + self.item_buffer_size = std::cmp::max(self.item_buffer_size, element_buffer.len() + 16); + + if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT { + self.write_buffered_items()?; } - fn end(self) -> Result { - Ok(self.bytes_written) - } + Ok(()) } - impl ser::SerializeStruct for SchemaAwareWriteSerializeStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; + fn end(mut self) -> Result { + self.write_buffered_items()?; + self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?; - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - let record_field = self - .record_schema - .lookup - .get(key) - .and_then(|idx| self.record_schema.fields.get(*idx)); - - match record_field { - Some(field) => { - // self.item_count += 1; - self.serialize_next_field(field, value).map_err(|e| { - Details::SerializeRecordFieldWithSchema { - field_name: key, - record_schema: Schema::Record(self.record_schema.clone()), - error: Box::new(e), - } - .into() - }) - } - None => Err(Details::FieldName(String::from(key)).into()), - } + Ok(self.bytes_written) + } +} + +/// The struct serializer for [`SchemaAwareWriteSerializer`], which can serialize Avro records. +/// [`SchemaAwareWriteSerializeStruct`] can accept fields out of order, but doing so incurs a +/// performance penalty, since it requires [`SchemaAwareWriteSerializeStruct`] to buffer serialized +/// values in order to write them to the stream in order. +pub struct SchemaAwareWriteSerializeStruct<'a, 's, W: Write> { + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + record_schema: &'s RecordSchema, + bytes_written: usize, +} + +impl<'a, 's, W: Write> SchemaAwareWriteSerializeStruct<'a, 's, W> { + fn new( + ser: &'a mut SchemaAwareWriteSerializer<'s, W>, + record_schema: &'s RecordSchema, + ) -> SchemaAwareWriteSerializeStruct<'a, 's, W> { + SchemaAwareWriteSerializeStruct { + ser, + record_schema, + bytes_written: 0, } + } - fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> { - let skipped_field = self - .record_schema - .lookup - .get(key) - .and_then(|idx| self.record_schema.fields.get(*idx)); + fn serialize_next_field(&mut self, field: &RecordField, value: &T) -> Result<(), Error> + where + T: ?Sized + ser::Serialize, + { + // If we receive fields in order, write them directly to the main writer + let mut value_ser = SchemaAwareWriteSerializer::new( + &mut *self.ser.writer, + &field.schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ); + self.bytes_written += value.serialize(&mut value_ser)?; + + Ok(()) + } - if let Some(skipped_field) = skipped_field { + fn end(self) -> Result { + Ok(self.bytes_written) + } +} + +impl ser::SerializeStruct for SchemaAwareWriteSerializeStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + let record_field = self + .record_schema + .lookup + .get(key) + .and_then(|idx| self.record_schema.fields.get(*idx)); + + match record_field { + Some(field) => { // self.item_count += 1; - skipped_field - .default - .serialize(&mut SchemaAwareWriteSerializer::new( - self.ser.writer, - &skipped_field.schema, - self.ser.names, - self.ser.enclosing_namespace.clone(), - ))?; - } else { - return Err(Details::GetField(key.to_string()).into()); - } - - Ok(()) + self.serialize_next_field(field, value).map_err(|e| { + Details::SerializeRecordFieldWithSchema { + field_name: key, + record_schema: Schema::Record(self.record_schema.clone()), + error: Box::new(e), + } + .into() + }) + } + None => Err(Details::FieldName(String::from(key)).into()), } + } - fn end(self) -> Result { - self.end() - } + fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> { + let skipped_field = self + .record_schema + .lookup + .get(key) + .and_then(|idx| self.record_schema.fields.get(*idx)); + + if let Some(skipped_field) = skipped_field { + // self.item_count += 1; + skipped_field + .default + .serialize(&mut SchemaAwareWriteSerializer::new( + self.ser.writer, + &skipped_field.schema, + self.ser.names, + self.ser.enclosing_namespace.clone(), + ))?; + } else { + return Err(Details::GetField(key.to_string()).into()); + } + + Ok(()) } - impl ser::SerializeStructVariant for SchemaAwareWriteSerializeStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; + fn end(self) -> Result { + self.end() + } +} - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - ser::SerializeStruct::serialize_field(self, key, value) - } +impl ser::SerializeStructVariant for SchemaAwareWriteSerializeStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn end(self) -> Result { - ser::SerializeStruct::end(self) - } + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + ser::SerializeStruct::serialize_field(self, key, value) } - /// The tuple struct serializer for [`SchemaAwareWriteSerializer`]. - /// [`SchemaAwareWriteSerializeTupleStruct`] can serialize to an Avro array, record, or big-decimal. - /// When serializing to a record, fields must be provided in the correct order, since no names are provided. - pub enum SchemaAwareWriteSerializeTupleStruct<'a, 's, W: Write> { - Record(SchemaAwareWriteSerializeStruct<'a, 's, W>), - Array(SchemaAwareWriteSerializeSeq<'a, 's, W>), + fn end(self) -> Result { + ser::SerializeStruct::end(self) } - - impl SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { - fn serialize_field(&mut self, value: &T) -> Result<(), Error> - where - T: ?Sized + ser::Serialize, - { - use SchemaAwareWriteSerializeTupleStruct::*; - match self { - Record(_record_ser) => { - unimplemented!("Tuple struct serialization to record is not supported!"); - } - Array(array_ser) => array_ser.serialize_element(&value), +} + +/// The tuple struct serializer for [`SchemaAwareWriteSerializer`]. +/// [`SchemaAwareWriteSerializeTupleStruct`] can serialize to an Avro array, record, or big-decimal. +/// When serializing to a record, fields must be provided in the correct order, since no names are provided. +pub enum SchemaAwareWriteSerializeTupleStruct<'a, 's, W: Write> { + Record(SchemaAwareWriteSerializeStruct<'a, 's, W>), + Array(SchemaAwareWriteSerializeSeq<'a, 's, W>), +} + +impl SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { + fn serialize_field(&mut self, value: &T) -> Result<(), Error> + where + T: ?Sized + ser::Serialize, + { + use SchemaAwareWriteSerializeTupleStruct::*; + match self { + Record(_record_ser) => { + unimplemented!("Tuple struct serialization to record is not supported!"); } + Array(array_ser) => array_ser.serialize_element(&value), } + } - fn end(self) -> Result { - use SchemaAwareWriteSerializeTupleStruct::*; - match self { - Record(record_ser) => record_ser.end(), - Array(array_ser) => array_ser.end(), - } + fn end(self) -> Result { + use SchemaAwareWriteSerializeTupleStruct::*; + match self { + Record(record_ser) => record_ser.end(), + Array(array_ser) => array_ser.end(), } } +} - impl ser::SerializeTupleStruct for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; +impl ser::SerializeTupleStruct for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - self.serialize_field(&value) - } - - fn end(self) -> Result { - self.end() - } + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + self.serialize_field(&value) } - impl ser::SerializeTupleVariant for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { - type Ok = usize; - type Error = Error; + fn end(self) -> Result { + self.end() + } +} - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> - where - T: ?Sized + ser::Serialize, - { - self.serialize_field(&value) - } +impl ser::SerializeTupleVariant for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> { + type Ok = usize; + type Error = Error; - fn end(self) -> Result { - self.end() - } + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: ?Sized + ser::Serialize, + { + self.serialize_field(&value) } - /// A [`serde::ser::Serializer`] implementation that serializes directly to a [`std::fmt::Write`] - /// using the provided schema. If [`SchemaAwareWriteSerializer`] isn't able to match the incoming - /// data with its schema, it will return an error. - /// A [`SchemaAwareWriteSerializer`] instance can be re-used to serialize multiple values matching - /// the schema to its [`std::fmt::Write`] stream. - pub struct SchemaAwareWriteSerializer<'s, W: Write> { + fn end(self) -> Result { + self.end() + } +} + +/// A [`serde::ser::Serializer`] implementation that serializes directly to a [`std::fmt::Write`] +/// using the provided schema. If [`SchemaAwareWriteSerializer`] isn't able to match the incoming +/// data with its schema, it will return an error. +/// A [`SchemaAwareWriteSerializer`] instance can be re-used to serialize multiple values matching +/// the schema to its [`std::fmt::Write`] stream. +pub struct SchemaAwareWriteSerializer<'s, W: Write> { + writer: &'s mut W, + root_schema: &'s Schema, + names: &'s Names, + enclosing_namespace: Namespace, +} + +impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { + /// Create a new [`SchemaAwareWriteSerializer`]. + /// + /// `writer` is the [`std::fmt::Write`] stream to be written to. + /// + /// `schema` is the schema of the value to be written. + /// + /// `names` is the mapping of schema names to schemas, to be used for type reference lookups + /// + /// `enclosing_namespace` is the enclosing namespace to be used for type reference lookups + pub fn new( writer: &'s mut W, - root_schema: &'s Schema, + schema: &'s Schema, names: &'s Names, enclosing_namespace: Namespace, - } - - impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { - /// Create a new [`SchemaAwareWriteSerializer`]. - /// - /// `writer` is the [`std::fmt::Write`] stream to be written to. - /// - /// `schema` is the schema of the value to be written. - /// - /// `names` is the mapping of schema names to schemas, to be used for type reference lookups - /// - /// `enclosing_namespace` is the enclosing namespace to be used for type reference lookups - pub fn new( - writer: &'s mut W, - schema: &'s Schema, - names: &'s Names, - enclosing_namespace: Namespace, - ) -> SchemaAwareWriteSerializer<'s, W> { - SchemaAwareWriteSerializer { - writer, - root_schema: schema, - names, - enclosing_namespace, - } + ) -> SchemaAwareWriteSerializer<'s, W> { + SchemaAwareWriteSerializer { + writer, + root_schema: schema, + names, + enclosing_namespace, } + } - fn get_ref_schema(&self, name: &'s Name) -> Result<&'s Schema, Error> { - let full_name = match name.namespace { - Some(_) => Cow::Borrowed(name), - None => Cow::Owned(Name { - name: name.name.clone(), - namespace: self.enclosing_namespace.clone(), - }), - }; + fn get_ref_schema(&self, name: &'s Name) -> Result<&'s Schema, Error> { + let full_name = match name.namespace { + Some(_) => Cow::Borrowed(name), + None => Cow::Owned(Name { + name: name.name.clone(), + namespace: self.enclosing_namespace.clone(), + }), + }; - let ref_schema = self.names.get(full_name.as_ref()).clone(); + let ref_schema = self.names.get(full_name.as_ref()).clone(); - ref_schema - .ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) - } + ref_schema + .ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) + } - fn write_bytes(&mut self, bytes: &[u8]) -> Result { - let mut bytes_written: usize = 0; + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let mut bytes_written: usize = 0; - bytes_written += encode_long(bytes.len() as i64, &mut self.writer)?; - bytes_written += self.writer.write(bytes).map_err(Details::WriteBytes)?; + bytes_written += encode_long(bytes.len() as i64, &mut self.writer)?; + bytes_written += self.writer.write(bytes).map_err(Details::WriteBytes)?; - Ok(bytes_written) - } + Ok(bytes_written) + } - fn serialize_bool_with_schema( - &mut self, - value: bool, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "bool", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_bool_with_schema( + &mut self, + value: bool, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "bool", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Boolean => self - .writer - .write(&[u8::from(value)]) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Boolean => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_bool_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Boolean => self + .writer + .write(&[u8::from(value)]) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Boolean => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_bool_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "No matching Schema::Bool found in {:?}", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected {expected}. Got: Bool"))), + Err(create_error(format!( + "No matching Schema::Bool found in {:?}", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected {expected}. Got: Bool"))), } + } - fn serialize_i32_with_schema( - &mut self, - value: i32, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "int (i8 | i16 | i32)", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_i32_with_schema( + &mut self, + value: i32, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "int (i8 | i16 | i32)", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - encode_int(value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_i32_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + encode_int(value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_i32_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching int-like schema in {union_schema:?}" - ))) } - expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), + Err(create_error(format!( + "Cannot find a matching int-like schema in {union_schema:?}" + ))) } + expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), } + } - fn serialize_i64_with_schema( - &mut self, - value: i64, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "i64", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_i64_with_schema( + &mut self, + value: i64, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "i64", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - let int_value = - i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_int(int_value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value, &mut self.writer), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_i64_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + let int_value = + i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_int(int_value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value, &mut self.writer), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_i64_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching int/long-like schema in {:?}", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), + Err(create_error(format!( + "Cannot find a matching int/long-like schema in {:?}", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), } + } - fn serialize_u8_with_schema(&mut self, value: u8, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "u8", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_u8_with_schema(&mut self, value: u8, schema: &Schema) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "u8", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - encode_int(value as i32, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), - Schema::Bytes => self.write_bytes(&[value]), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos - | Schema::Bytes => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_u8_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + encode_int(value as i32, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), + Schema::Bytes => self.write_bytes(&[value]), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos + | Schema::Bytes => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_u8_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching Int-like, Long-like or Bytes schema in {union_schema:?}" - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: Int"))), + Err(create_error(format!( + "Cannot find a matching Int-like, Long-like or Bytes schema in {union_schema:?}" + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: Int"))), } + } - fn serialize_u32_with_schema( - &mut self, - value: u32, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "unsigned int (u16 | u32)", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_u32_with_schema( + &mut self, + value: u32, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "unsigned int (u16 | u32)", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - let int_value = - i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_int(int_value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_u32_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + let int_value = + i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_int(int_value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_u32_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching Int-like or Long-like schema in {union_schema:?}" - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), + Err(create_error(format!( + "Cannot find a matching Int-like or Long-like schema in {union_schema:?}" + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))), } + } - fn serialize_u64_with_schema( - &mut self, - value: u64, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "u64", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_u64_with_schema( + &mut self, + value: u64, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "u64", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - let int_value = - i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_int(int_value, &mut self.writer) - } - Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - let long_value = - i64::try_from(value).map_err(|cause| create_error(cause.to_string()))?; - encode_long(long_value, &mut self.writer) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Int - | Schema::TimeMillis - | Schema::Date - | Schema::Long - | Schema::TimeMicros - | Schema::TimestampMillis - | Schema::TimestampMicros - | Schema::TimestampNanos - | Schema::LocalTimestampMillis - | Schema::LocalTimestampMicros - | Schema::LocalTimestampNanos => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_u64_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Int | Schema::TimeMillis | Schema::Date => { + let int_value = + i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_int(int_value, &mut self.writer) + } + Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + let long_value = + i64::try_from(value).map_err(|cause| create_error(cause.to_string()))?; + encode_long(long_value, &mut self.writer) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Int + | Schema::TimeMillis + | Schema::Date + | Schema::Long + | Schema::TimeMicros + | Schema::TimestampMillis + | Schema::TimestampMicros + | Schema::TimestampNanos + | Schema::LocalTimestampMillis + | Schema::LocalTimestampMicros + | Schema::LocalTimestampNanos => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_u64_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching Int-like or Long-like schema in {:?}", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), + Err(create_error(format!( + "Cannot find a matching Int-like or Long-like schema in {:?}", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))), } + } - fn serialize_f32_with_schema( - &mut self, - value: f32, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "f32", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_f32_with_schema( + &mut self, + value: f32, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "f32", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Float => self - .writer - .write(&value.to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Double => self - .writer - .write(&(value as f64).to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Float | Schema::Double => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_f32_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Float => self + .writer + .write(&value.to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Double => self + .writer + .write(&(value as f64).to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Float | Schema::Double => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_f32_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a Float schema in {:?}", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: Float"))), + Err(create_error(format!( + "Cannot find a Float schema in {:?}", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: Float"))), } + } - fn serialize_f64_with_schema( - &mut self, - value: f64, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "f64", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_f64_with_schema( + &mut self, + value: f64, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "f64", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Float => self - .writer - .write(&(value as f32).to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Double => self - .writer - .write(&value.to_le_bytes()) - .map_err(|e| Details::WriteBytes(e).into()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Float | Schema::Double => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_f64_with_schema(value, variant_schema); - } - _ => { /* skip */ } + match schema { + Schema::Float => self + .writer + .write(&(value as f32).to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Double => self + .writer + .write(&value.to_le_bytes()) + .map_err(|e| Details::WriteBytes(e).into()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Float | Schema::Double => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_f64_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a Double schema in {:?}", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: Double"))), + Err(create_error(format!( + "Cannot find a Double schema in {:?}", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: Double"))), } + } - fn serialize_char_with_schema( - &mut self, - value: char, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "char", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::String | Schema::Bytes => self.write_bytes(String::from(value).as_bytes()), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::String | Schema::Bytes => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_char_with_schema(value, variant_schema); - } - _ => { /* skip */ } + fn serialize_char_with_schema( + &mut self, + value: char, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "char", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::String | Schema::Bytes => self.write_bytes(String::from(value).as_bytes()), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::String | Schema::Bytes => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_char_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching String or Bytes schema in {union_schema:?}" - ))) } - expected => Err(create_error(format!("Expected {expected}. Got: char"))), + Err(create_error(format!( + "Cannot find a matching String or Bytes schema in {union_schema:?}" + ))) } + expected => Err(create_error(format!("Expected {expected}. Got: char"))), } + } - fn serialize_str_with_schema( - &mut self, - value: &str, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "string", - value: format!("{value}. Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::String | Schema::Bytes | Schema::Uuid => self.write_bytes(value.as_bytes()), - Schema::BigDecimal => { - // If we get a string for a `BigDecimal` type, expect a display string representation, such as "12.75" - let decimal_val = - BigDecimal::from_str(value).map_err(|e| create_error(e.to_string()))?; - let decimal_bytes = big_decimal_as_bytes(&decimal_val)?; - self.write_bytes(decimal_bytes.as_slice()) - } - Schema::Fixed(fixed_schema) => { - if value.len() == fixed_schema.size { - self.writer - .write(value.as_bytes()) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - Err(create_error(format!( - "Fixed schema size ({}) does not match the value length ({})", - fixed_schema.size, - value.len() - ))) - } - } - Schema::Ref { name } => { - let ref_schema = self.get_ref_schema(name)?; - self.serialize_str_with_schema(value, ref_schema) + fn serialize_str_with_schema( + &mut self, + value: &str, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "string", + value: format!("{value}. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::String | Schema::Bytes | Schema::Uuid => self.write_bytes(value.as_bytes()), + Schema::BigDecimal => { + // If we get a string for a `BigDecimal` type, expect a display string representation, such as "12.75" + let decimal_val = + BigDecimal::from_str(value).map_err(|e| create_error(e.to_string()))?; + let decimal_bytes = big_decimal_as_bytes(&decimal_val)?; + self.write_bytes(decimal_bytes.as_slice()) + } + Schema::Fixed(fixed_schema) => { + if value.len() == fixed_schema.size { + self.writer + .write(value.as_bytes()) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + Err(create_error(format!( + "Fixed schema size ({}) does not match the value length ({})", + fixed_schema.size, + value.len() + ))) } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::String - | Schema::Bytes - | Schema::Uuid - | Schema::Fixed(_) - | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_str_with_schema(value, variant_schema); - } - _ => { /* skip */ } + } + Schema::Ref { name } => { + let ref_schema = self.get_ref_schema(name)?; + self.serialize_str_with_schema(value, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::String + | Schema::Bytes + | Schema::Uuid + | Schema::Fixed(_) + | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_str_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Expected one of the union variants {:?}. Got: String", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: String"))), + Err(create_error(format!( + "Expected one of the union variants {:?}. Got: String", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: String"))), } + } - fn serialize_bytes_with_schema( - &mut self, - value: &[u8], - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - use std::fmt::Write; - let mut v_str = String::with_capacity(value.len()); - for b in value { - if write!(&mut v_str, "{b:x}").is_err() { - v_str.push_str("??"); - } + fn serialize_bytes_with_schema( + &mut self, + value: &[u8], + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + use std::fmt::Write; + let mut v_str = String::with_capacity(value.len()); + for b in value { + if write!(&mut v_str, "{b:x}").is_err() { + v_str.push_str("??"); } - Error::new(Details::SerializeValueWithSchema { - value_type: "bytes", - value: format!("{v_str}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + } + Error::new(Details::SerializeValueWithSchema { + value_type: "bytes", + value: format!("{v_str}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::String | Schema::Bytes | Schema::Uuid | Schema::BigDecimal => { - self.write_bytes(value) + match schema { + Schema::String | Schema::Bytes | Schema::Uuid | Schema::BigDecimal => { + self.write_bytes(value) + } + Schema::Fixed(fixed_schema) => { + if value.len() == fixed_schema.size { + self.writer + .write(value) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + Err(create_error(format!( + "Fixed schema size ({}) does not match the value length ({})", + fixed_schema.size, + value.len() + ))) } - Schema::Fixed(fixed_schema) => { - if value.len() == fixed_schema.size { - self.writer - .write(value) - .map_err(|e| Details::WriteBytes(e).into()) - } else { - Err(create_error(format!( - "Fixed schema size ({}) does not match the value length ({})", - fixed_schema.size, - value.len() - ))) - } + } + Schema::Duration => { + if value.len() == 12 { + self.writer + .write(value) + .map_err(|e| Details::WriteBytes(e).into()) + } else { + Err(create_error(format!( + "Duration length must be 12! Got ({})", + value.len() + ))) } - Schema::Duration => { - if value.len() == 12 { + } + Schema::Decimal(decimal_schema) => match decimal_schema.inner.as_ref() { + Schema::Bytes => self.write_bytes(value), + Schema::Fixed(fixed_schema) => match fixed_schema.size.checked_sub(value.len()) + { + Some(pad) => { + let pad_val = match value.len() { + 0 => 0, + _ => value[0], + }; + let padding = vec![pad_val; pad]; + self.writer + .write(padding.as_slice()) + .map_err(Details::WriteBytes)?; self.writer .write(value) .map_err(|e| Details::WriteBytes(e).into()) - } else { - Err(create_error(format!( - "Duration length must be 12! Got ({})", - value.len() - ))) } - } - Schema::Decimal(decimal_schema) => match decimal_schema.inner.as_ref() { - Schema::Bytes => self.write_bytes(value), - Schema::Fixed(fixed_schema) => match fixed_schema.size.checked_sub(value.len()) - { - Some(pad) => { - let pad_val = match value.len() { - 0 => 0, - _ => value[0], - }; - let padding = vec![pad_val; pad]; - self.writer - .write(padding.as_slice()) - .map_err(Details::WriteBytes)?; - self.writer - .write(value) - .map_err(|e| Details::WriteBytes(e).into()) - } - None => Err(Details::CompareFixedSizes { - size: fixed_schema.size, - n: value.len(), - } - .into()), - }, - unsupported => Err(create_error(format!( - "Decimal schema's inner should be Bytes or Fixed schema. Got: {unsupported}" - ))), + None => Err(Details::CompareFixedSizes { + size: fixed_schema.size, + n: value.len(), + } + .into()), }, - Schema::Ref { name } => { - let ref_schema = self.get_ref_schema(name)?; - self.serialize_bytes_with_schema(value, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::String - | Schema::Bytes - | Schema::Uuid - | Schema::BigDecimal - | Schema::Fixed(_) - | Schema::Duration - | Schema::Decimal(_) - | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_bytes_with_schema(value, variant_schema); - } - _ => { /* skip */ } + unsupported => Err(create_error(format!( + "Decimal schema's inner should be Bytes or Fixed schema. Got: {unsupported}" + ))), + }, + Schema::Ref { name } => { + let ref_schema = self.get_ref_schema(name)?; + self.serialize_bytes_with_schema(value, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::String + | Schema::Bytes + | Schema::Uuid + | Schema::BigDecimal + | Schema::Fixed(_) + | Schema::Duration + | Schema::Decimal(_) + | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_bytes_with_schema(value, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal or Ref schema in {union_schema:?}" - ))) } - unsupported => Err(create_error(format!( - "Expected String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal, Ref or Union schema. Got: {unsupported}" - ))), + Err(create_error(format!( + "Cannot find a matching String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal or Ref schema in {union_schema:?}" + ))) } + unsupported => Err(create_error(format!( + "Expected String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal, Ref or Union schema. Got: {unsupported}" + ))), } + } - fn serialize_none_with_schema(&mut self, schema: &Schema) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "none", - value: format!("None. Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Null => Ok(0), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Null => { - return encode_int(i as i32, &mut *self.writer); - } - _ => { /* skip */ } + fn serialize_none_with_schema(&mut self, schema: &Schema) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "none", + value: format!("None. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Null => Ok(0), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Null => { + return encode_int(i as i32, &mut *self.writer); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching Null schema in {:?}", - union_schema.schemas - ))) } - expected => Err(create_error(format!("Expected: {expected}. Got: Null"))), + Err(create_error(format!( + "Cannot find a matching Null schema in {:?}", + union_schema.schemas + ))) } + expected => Err(create_error(format!("Expected: {expected}. Got: Null"))), } + } - fn serialize_some_with_schema( - &mut self, - value: &T, - schema: &Schema, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "some", - value: format!("Some(?). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Null => { /* skip */ } - _ => { - encode_long(i as i64, &mut *self.writer)?; - let mut variant_ser = SchemaAwareWriteSerializer::new( - &mut *self.writer, - variant_schema, - self.names, - self.enclosing_namespace.clone(), - ); - return value.serialize(&mut variant_ser); - } + fn serialize_some_with_schema( + &mut self, + value: &T, + schema: &Schema, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "some", + value: format!("Some(?). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Null => { /* skip */ } + _ => { + encode_long(i as i64, &mut *self.writer)?; + let mut variant_ser = SchemaAwareWriteSerializer::new( + &mut *self.writer, + variant_schema, + self.names, + self.enclosing_namespace.clone(), + ); + return value.serialize(&mut variant_ser); } } - Err(create_error(format!( - "Cannot find a matching Null schema in {:?}", - union_schema.schemas - ))) } - _ => value.serialize(self), + Err(create_error(format!( + "Cannot find a matching Null schema in {:?}", + union_schema.schemas + ))) } + _ => value.serialize(self), } + } - fn serialize_unit_struct_with_schema( - &mut self, - name: &'static str, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "unit struct", - value: format!("{name}. Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Record(sch) => match sch.fields.len() { - 0 => Ok(0), - too_many => Err(create_error(format!( - "Too many fields: {too_many}. Expected: 0" - ))), - }, - Schema::Null => Ok(0), - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_unit_struct_with_schema(name, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Record(record_schema) if record_schema.fields.is_empty() => { - encode_int(i as i32, &mut *self.writer)?; - return self - .serialize_unit_struct_with_schema(name, variant_schema); - } - Schema::Null | Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self - .serialize_unit_struct_with_schema(name, variant_schema); - } - _ => { /* skip */ } + fn serialize_unit_struct_with_schema( + &mut self, + name: &'static str, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "unit struct", + value: format!("{name}. Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Record(sch) => match sch.fields.len() { + 0 => Ok(0), + too_many => Err(create_error(format!( + "Too many fields: {too_many}. Expected: 0" + ))), + }, + Schema::Null => Ok(0), + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_unit_struct_with_schema(name, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Record(record_schema) if record_schema.fields.is_empty() => { + encode_int(i as i32, &mut *self.writer)?; + return self + .serialize_unit_struct_with_schema(name, variant_schema); + } + Schema::Null | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self + .serialize_unit_struct_with_schema(name, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Cannot find a matching Null schema in {union_schema:?}" - ))) } - unsupported => Err(create_error(format!( - "Expected Null or Union schema. Got: {unsupported}" - ))), + Err(create_error(format!( + "Cannot find a matching Null schema in {union_schema:?}" + ))) } + unsupported => Err(create_error(format!( + "Expected Null or Union schema. Got: {unsupported}" + ))), } + } - fn serialize_unit_variant_with_schema( - &mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - schema: &Schema, - ) -> Result { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "unit variant", - value: format!("{name}::{variant} (index={variant_index}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Enum(enum_schema) => { - if variant_index as usize >= enum_schema.symbols.len() { - return Err(create_error(format!( - "Variant index out of bounds: {}. The Enum schema has '{}' symbols", - variant_index, - enum_schema.symbols.len() - ))); - } + fn serialize_unit_variant_with_schema( + &mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + schema: &Schema, + ) -> Result { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "unit variant", + value: format!("{name}::{variant} (index={variant_index}). Cause: {cause}"), + schema: schema.clone(), + }) + }; - encode_int(variant_index as i32, &mut self.writer) + match schema { + Schema::Enum(enum_schema) => { + if variant_index as usize >= enum_schema.symbols.len() { + return Err(create_error(format!( + "Variant index out of bounds: {}. The Enum schema has '{}' symbols", + variant_index, + enum_schema.symbols.len() + ))); } - Schema::Union(union_schema) => { - if variant_index as usize >= union_schema.schemas.len() { - return Err(create_error(format!( - "Variant index out of bounds: {}. The union schema has '{}' schemas", - variant_index, - union_schema.schemas.len() - ))); - } - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_unit_struct_with_schema( - name, - &union_schema.schemas[variant_index as usize], - ) - } - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_unit_variant_with_schema( - name, + encode_int(variant_index as i32, &mut self.writer) + } + Schema::Union(union_schema) => { + if variant_index as usize >= union_schema.schemas.len() { + return Err(create_error(format!( + "Variant index out of bounds: {}. The union schema has '{}' schemas", variant_index, - variant, - ref_schema, - ) + union_schema.schemas.len() + ))); } - unsupported => Err(create_error(format!( - "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref" - ))), + + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_unit_struct_with_schema( + name, + &union_schema.schemas[variant_index as usize], + ) + } + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_unit_variant_with_schema( + name, + variant_index, + variant, + ref_schema, + ) } + unsupported => Err(create_error(format!( + "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref" + ))), } + } - fn serialize_newtype_struct_with_schema( - &mut self, - _name: &'static str, - value: &T, - schema: &Schema, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - let mut inner_ser = SchemaAwareWriteSerializer::new( - &mut *self.writer, - schema, - self.names, - self.enclosing_namespace.clone(), - ); - // Treat any newtype struct as a transparent wrapper around the contained type - value.serialize(&mut inner_ser) - } + fn serialize_newtype_struct_with_schema( + &mut self, + _name: &'static str, + value: &T, + schema: &Schema, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let mut inner_ser = SchemaAwareWriteSerializer::new( + &mut *self.writer, + schema, + self.names, + self.enclosing_namespace.clone(), + ); + // Treat any newtype struct as a transparent wrapper around the contained type + value.serialize(&mut inner_ser) + } - fn serialize_newtype_variant_with_schema( - &mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - value: &T, - schema: &Schema, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "newtype variant", - value: format!("{name}::{variant}(?) (index={variant_index}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Union(union_schema) => { - let variant_schema = union_schema - .schemas - .get(variant_index as usize) - .ok_or_else(|| { - create_error(format!( - "No variant schema at position {variant_index} for {union_schema:?}" - )) - })?; - - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_newtype_struct_with_schema(variant, value, variant_schema) - } - _ => Err(create_error(format!( - "Expected Union schema. Got: {schema}" - ))), + fn serialize_newtype_variant_with_schema( + &mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + schema: &Schema, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "newtype variant", + value: format!("{name}::{variant}(?) (index={variant_index}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Union(union_schema) => { + let variant_schema = union_schema + .schemas + .get(variant_index as usize) + .ok_or_else(|| { + create_error(format!( + "No variant schema at position {variant_index} for {union_schema:?}" + )) + })?; + + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_newtype_struct_with_schema(variant, value, variant_schema) } + _ => Err(create_error(format!( + "Expected Union schema. Got: {schema}" + ))), } + } - fn serialize_seq_with_schema<'a>( - &'a mut self, - len: Option, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - let len_str = len - .map(|l| format!("{l}")) - .unwrap_or_else(|| String::from("?")); - - Error::new(Details::SerializeValueWithSchema { - value_type: "sequence", - value: format!("sequence (len={len_str}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( - self, - array_schema.items.as_ref(), - len, - )), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Array(_) => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_seq_with_schema(len, variant_schema); - } - _ => { /* skip */ } + fn serialize_seq_with_schema<'a>( + &'a mut self, + len: Option, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + let len_str = len + .map(|l| format!("{l}")) + .unwrap_or_else(|| String::from("?")); + + Error::new(Details::SerializeValueWithSchema { + value_type: "sequence", + value: format!("sequence (len={len_str}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( + self, + array_schema.items.as_ref(), + len, + )), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Array(_) => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_seq_with_schema(len, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Expected Array schema in {union_schema:?}" - ))) } - _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), + Err(create_error(format!( + "Expected Array schema in {union_schema:?}" + ))) } + _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), } + } - fn serialize_tuple_with_schema<'a>( - &'a mut self, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "tuple", - value: format!("tuple (len={len}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( - self, - array_schema.items.as_ref(), - Some(len), - )), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Array(_) => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_tuple_with_schema(len, variant_schema); - } - _ => { /* skip */ } + fn serialize_tuple_with_schema<'a>( + &'a mut self, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "tuple", + value: format!("tuple (len={len}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new( + self, + array_schema.items.as_ref(), + Some(len), + )), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Array(_) => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_tuple_with_schema(len, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Expected Array schema in {union_schema:?}" - ))) } - _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), + Err(create_error(format!( + "Expected Array schema in {union_schema:?}" + ))) } + _ => Err(create_error(format!("Expected: {schema}. Got: Array"))), } + } - fn serialize_tuple_struct_with_schema<'a>( - &'a mut self, - name: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "tuple struct", - value: format!( - "{name}({}). Cause: {cause}", - vec!["?"; len].as_slice().join(",") - ), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Array(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Array( - SchemaAwareWriteSerializeSeq::new(self, &sch.items, Some(len)), - )), - Schema::Record(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Record( - SchemaAwareWriteSerializeStruct::new(self, sch), - )), - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_tuple_struct_with_schema(name, len, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Record(inner) => { - if inner.fields.len() == len { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_tuple_struct_with_schema( - name, - len, - variant_schema, - ); - } - } - Schema::Array(_) | Schema::Ref { name: _ } => { + fn serialize_tuple_struct_with_schema<'a>( + &'a mut self, + name: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "tuple struct", + value: format!( + "{name}({}). Cause: {cause}", + vec!["?"; len].as_slice().join(",") + ), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Array(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Array( + SchemaAwareWriteSerializeSeq::new(self, &sch.items, Some(len)), + )), + Schema::Record(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Record( + SchemaAwareWriteSerializeStruct::new(self, sch), + )), + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_tuple_struct_with_schema(name, len, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Record(inner) => { + if inner.fields.len() == len { encode_int(i as i32, &mut *self.writer)?; return self.serialize_tuple_struct_with_schema( name, @@ -1443,1238 +1434,1296 @@ variant_schema, ); } - _ => { /* skip */ } } + Schema::Array(_) | Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_tuple_struct_with_schema( + name, + len, + variant_schema, + ); + } + _ => { /* skip */ } } - Err(create_error(format!( - "Expected Record, Array or Ref schema in {union_schema:?}" - ))) } - _ => Err(create_error(format!( - "Expected Record, Array, Ref or Union schema. Got: {schema}" - ))), + Err(create_error(format!( + "Expected Record, Array or Ref schema in {union_schema:?}" + ))) } + _ => Err(create_error(format!( + "Expected Record, Array, Ref or Union schema. Got: {schema}" + ))), } + } - fn serialize_tuple_variant_with_schema<'a>( - &'a mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "tuple variant", - value: format!( - "{name}::{variant}({}) (index={variant_index}). Cause: {cause}", - vec!["?"; len].as_slice().join(",") - ), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Union(union_schema) => { - let variant_schema = union_schema - .schemas - .get(variant_index as usize) - .ok_or_else(|| { - create_error(format!( - "Cannot find a variant at position {variant_index} in {union_schema:?}" - )) - })?; + fn serialize_tuple_variant_with_schema<'a>( + &'a mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "tuple variant", + value: format!( + "{name}::{variant}({}) (index={variant_index}). Cause: {cause}", + vec!["?"; len].as_slice().join(",") + ), + schema: schema.clone(), + }) + }; - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_tuple_struct_with_schema(variant, len, variant_schema) - } - _ => Err(create_error(format!( - "Expected Union schema. Got: {schema}" - ))), + match schema { + Schema::Union(union_schema) => { + let variant_schema = union_schema + .schemas + .get(variant_index as usize) + .ok_or_else(|| { + create_error(format!( + "Cannot find a variant at position {variant_index} in {union_schema:?}" + )) + })?; + + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_tuple_struct_with_schema(variant, len, variant_schema) } + _ => Err(create_error(format!( + "Expected Union schema. Got: {schema}" + ))), } + } - fn serialize_map_with_schema<'a>( - &'a mut self, - len: Option, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - let len_str = len - .map(|l| format!("{l}")) - .unwrap_or_else(|| String::from("?")); - - Error::new(Details::SerializeValueWithSchema { - value_type: "map", - value: format!("map (size={len_str}). Cause: {cause}"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Map(map_schema) => Ok(SchemaAwareWriteSerializeMap::new( - self, - map_schema.types.as_ref(), - len, - )), - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Map(_) => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_map_with_schema(len, variant_schema); - } - _ => { /* skip */ } + fn serialize_map_with_schema<'a>( + &'a mut self, + len: Option, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + let len_str = len + .map(|l| format!("{l}")) + .unwrap_or_else(|| String::from("?")); + + Error::new(Details::SerializeValueWithSchema { + value_type: "map", + value: format!("map (size={len_str}). Cause: {cause}"), + schema: schema.clone(), + }) + }; + + match schema { + Schema::Map(map_schema) => Ok(SchemaAwareWriteSerializeMap::new( + self, + map_schema.types.as_ref(), + len, + )), + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Map(_) => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_map_with_schema(len, variant_schema); } + _ => { /* skip */ } } - Err(create_error(format!( - "Expected a Map schema in {union_schema:?}" - ))) } - _ => Err(create_error(format!( - "Expected Map or Union schema. Got: {schema}" - ))), + Err(create_error(format!( + "Expected a Map schema in {union_schema:?}" + ))) } + _ => Err(create_error(format!( + "Expected Map or Union schema. Got: {schema}" + ))), } + } - fn serialize_struct_with_schema<'a>( - &'a mut self, - name: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "struct", - value: format!("{name}{{ ... }}. Cause: {cause}"), - schema: schema.clone(), - }) - }; + fn serialize_struct_with_schema<'a>( + &'a mut self, + name: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "struct", + value: format!("{name}{{ ... }}. Cause: {cause}"), + schema: schema.clone(), + }) + }; - match schema { - Schema::Record(record_schema) => { - Ok(SchemaAwareWriteSerializeStruct::new(self, record_schema)) - } - Schema::Ref { name: ref_name } => { - let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_struct_with_schema(name, len, ref_schema) - } - Schema::Union(union_schema) => { - for (i, variant_schema) in union_schema.schemas.iter().enumerate() { - match variant_schema { - Schema::Record(inner) - if inner.fields.len() == len && inner.name.name == name => - { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_struct_with_schema( - name, - len, - variant_schema, - ); - } - Schema::Ref { name: _ } => { - encode_int(i as i32, &mut *self.writer)?; - return self.serialize_struct_with_schema( - name, - len, - variant_schema, - ); - } - _ => { /* skip */ } + match schema { + Schema::Record(record_schema) => { + Ok(SchemaAwareWriteSerializeStruct::new(self, record_schema)) + } + Schema::Ref { name: ref_name } => { + let ref_schema = self.get_ref_schema(ref_name)?; + self.serialize_struct_with_schema(name, len, ref_schema) + } + Schema::Union(union_schema) => { + for (i, variant_schema) in union_schema.schemas.iter().enumerate() { + match variant_schema { + Schema::Record(inner) + if inner.fields.len() == len && inner.name.name == name => + { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_struct_with_schema( + name, + len, + variant_schema, + ); } + Schema::Ref { name: _ } => { + encode_int(i as i32, &mut *self.writer)?; + return self.serialize_struct_with_schema( + name, + len, + variant_schema, + ); + } + _ => { /* skip */ } } - Err(create_error(format!( - "Expected Record or Ref schema in {union_schema:?}" - ))) } - _ => Err(create_error(format!( - "Expected Record, Ref or Union schema. Got: {schema}" - ))), + Err(create_error(format!( + "Expected Record or Ref schema in {union_schema:?}" + ))) } + _ => Err(create_error(format!( + "Expected Record, Ref or Union schema. Got: {schema}" + ))), } + } - fn serialize_struct_variant_with_schema<'a>( - &'a mut self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - schema: &'s Schema, - ) -> Result, Error> { - let create_error = |cause: String| { - Error::new(Details::SerializeValueWithSchema { - value_type: "struct variant", - value: format!("{name}::{variant}{{ ... }} (size={len}. Cause: {cause})"), - schema: schema.clone(), - }) - }; - - match schema { - Schema::Union(union_schema) => { - let variant_schema = union_schema - .schemas - .get(variant_index as usize) - .ok_or_else(|| { - create_error(format!( - "Cannot find variant at position {variant_index} in {union_schema:?}" - )) - })?; + fn serialize_struct_variant_with_schema<'a>( + &'a mut self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + schema: &'s Schema, + ) -> Result, Error> { + let create_error = |cause: String| { + Error::new(Details::SerializeValueWithSchema { + value_type: "struct variant", + value: format!("{name}::{variant}{{ ... }} (size={len}. Cause: {cause})"), + schema: schema.clone(), + }) + }; - encode_int(variant_index as i32, &mut self.writer)?; - self.serialize_struct_with_schema(variant, len, variant_schema) - } - _ => Err(create_error(format!( - "Expected Union schema. Got: {schema}" - ))), + match schema { + Schema::Union(union_schema) => { + let variant_schema = union_schema + .schemas + .get(variant_index as usize) + .ok_or_else(|| { + create_error(format!( + "Cannot find variant at position {variant_index} in {union_schema:?}" + )) + })?; + + encode_int(variant_index as i32, &mut self.writer)?; + self.serialize_struct_with_schema(variant, len, variant_schema) } + _ => Err(create_error(format!( + "Expected Union schema. Got: {schema}" + ))), } } +} + +impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s, W> { + type Ok = usize; + type Error = Error; + type SerializeSeq = SchemaAwareWriteSerializeSeq<'a, 's, W>; + type SerializeTuple = SchemaAwareWriteSerializeSeq<'a, 's, W>; + type SerializeTupleStruct = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; + type SerializeTupleVariant = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; + type SerializeMap = SchemaAwareWriteSerializeMap<'a, 's, W>; + type SerializeStruct = SchemaAwareWriteSerializeStruct<'a, 's, W>; + type SerializeStructVariant = SchemaAwareWriteSerializeStruct<'a, 's, W>; + + fn serialize_bool(self, v: bool) -> Result { + self.serialize_bool_with_schema(v, self.root_schema) + } - impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s, W> { - type Ok = usize; - type Error = Error; - type SerializeSeq = SchemaAwareWriteSerializeSeq<'a, 's, W>; - type SerializeTuple = SchemaAwareWriteSerializeSeq<'a, 's, W>; - type SerializeTupleStruct = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; - type SerializeTupleVariant = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>; - type SerializeMap = SchemaAwareWriteSerializeMap<'a, 's, W>; - type SerializeStruct = SchemaAwareWriteSerializeStruct<'a, 's, W>; - type SerializeStructVariant = SchemaAwareWriteSerializeStruct<'a, 's, W>; - - fn serialize_bool(self, v: bool) -> Result { - self.serialize_bool_with_schema(v, self.root_schema) - } + fn serialize_i8(self, v: i8) -> Result { + self.serialize_i32(v as i32) + } - fn serialize_i8(self, v: i8) -> Result { - self.serialize_i32(v as i32) - } + fn serialize_i16(self, v: i16) -> Result { + self.serialize_i32(v as i32) + } - fn serialize_i16(self, v: i16) -> Result { - self.serialize_i32(v as i32) - } + fn serialize_i32(self, v: i32) -> Result { + self.serialize_i32_with_schema(v, self.root_schema) + } - fn serialize_i32(self, v: i32) -> Result { - self.serialize_i32_with_schema(v, self.root_schema) - } + fn serialize_i64(self, v: i64) -> Result { + self.serialize_i64_with_schema(v, self.root_schema) + } - fn serialize_i64(self, v: i64) -> Result { - self.serialize_i64_with_schema(v, self.root_schema) - } + fn serialize_u8(self, v: u8) -> Result { + self.serialize_u8_with_schema(v, self.root_schema) + } - fn serialize_u8(self, v: u8) -> Result { - self.serialize_u8_with_schema(v, self.root_schema) - } + fn serialize_u16(self, v: u16) -> Result { + self.serialize_u32(v as u32) + } - fn serialize_u16(self, v: u16) -> Result { - self.serialize_u32(v as u32) - } + fn serialize_u32(self, v: u32) -> Result { + self.serialize_u32_with_schema(v, self.root_schema) + } - fn serialize_u32(self, v: u32) -> Result { - self.serialize_u32_with_schema(v, self.root_schema) - } + fn serialize_u64(self, v: u64) -> Result { + self.serialize_u64_with_schema(v, self.root_schema) + } - fn serialize_u64(self, v: u64) -> Result { - self.serialize_u64_with_schema(v, self.root_schema) - } + fn serialize_f32(self, v: f32) -> Result { + self.serialize_f32_with_schema(v, self.root_schema) + } - fn serialize_f32(self, v: f32) -> Result { - self.serialize_f32_with_schema(v, self.root_schema) - } + fn serialize_f64(self, v: f64) -> Result { + self.serialize_f64_with_schema(v, self.root_schema) + } - fn serialize_f64(self, v: f64) -> Result { - self.serialize_f64_with_schema(v, self.root_schema) - } + fn serialize_char(self, v: char) -> Result { + self.serialize_char_with_schema(v, self.root_schema) + } - fn serialize_char(self, v: char) -> Result { - self.serialize_char_with_schema(v, self.root_schema) - } + fn serialize_str(self, v: &str) -> Result { + self.serialize_str_with_schema(v, self.root_schema) + } - fn serialize_str(self, v: &str) -> Result { - self.serialize_str_with_schema(v, self.root_schema) - } + fn serialize_bytes(self, v: &[u8]) -> Result { + self.serialize_bytes_with_schema(v, self.root_schema) + } - fn serialize_bytes(self, v: &[u8]) -> Result { - self.serialize_bytes_with_schema(v, self.root_schema) - } + fn serialize_none(self) -> Result { + self.serialize_none_with_schema(self.root_schema) + } - fn serialize_none(self) -> Result { - self.serialize_none_with_schema(self.root_schema) - } + fn serialize_some(self, value: &T) -> Result + where + T: ?Sized + ser::Serialize, + { + self.serialize_some_with_schema(value, self.root_schema) + } - fn serialize_some(self, value: &T) -> Result - where - T: ?Sized + ser::Serialize, - { - self.serialize_some_with_schema(value, self.root_schema) - } + fn serialize_unit(self) -> Result { + self.serialize_none() + } - fn serialize_unit(self) -> Result { - self.serialize_none() - } + fn serialize_unit_struct(self, name: &'static str) -> Result { + self.serialize_unit_struct_with_schema(name, self.root_schema) + } - fn serialize_unit_struct(self, name: &'static str) -> Result { - self.serialize_unit_struct_with_schema(name, self.root_schema) - } + fn serialize_unit_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + ) -> Result { + self.serialize_unit_variant_with_schema(name, variant_index, variant, self.root_schema) + } - fn serialize_unit_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - ) -> Result { - self.serialize_unit_variant_with_schema(name, variant_index, variant, self.root_schema) - } + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + self.serialize_newtype_struct_with_schema(name, value, self.root_schema) + } - fn serialize_newtype_struct( - self, - name: &'static str, - value: &T, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - self.serialize_newtype_struct_with_schema(name, value, self.root_schema) - } + fn serialize_newtype_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: ?Sized + ser::Serialize, + { + self.serialize_newtype_variant_with_schema( + name, + variant_index, + variant, + value, + self.root_schema, + ) + } - fn serialize_newtype_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - value: &T, - ) -> Result - where - T: ?Sized + ser::Serialize, - { - self.serialize_newtype_variant_with_schema( - name, - variant_index, - variant, - value, - self.root_schema, - ) - } + fn serialize_seq(self, len: Option) -> Result { + self.serialize_seq_with_schema(len, self.root_schema) + } - fn serialize_seq(self, len: Option) -> Result { - self.serialize_seq_with_schema(len, self.root_schema) - } + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_tuple_with_schema(len, self.root_schema) + } - fn serialize_tuple(self, len: usize) -> Result { - self.serialize_tuple_with_schema(len, self.root_schema) - } + fn serialize_tuple_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + self.serialize_tuple_struct_with_schema(name, len, self.root_schema) + } - fn serialize_tuple_struct( - self, - name: &'static str, - len: usize, - ) -> Result { - self.serialize_tuple_struct_with_schema(name, len, self.root_schema) - } + fn serialize_tuple_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.serialize_tuple_variant_with_schema( + name, + variant_index, + variant, + len, + self.root_schema, + ) + } - fn serialize_tuple_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - ) -> Result { - self.serialize_tuple_variant_with_schema( - name, - variant_index, - variant, - len, - self.root_schema, - ) - } + fn serialize_map(self, len: Option) -> Result { + self.serialize_map_with_schema(len, self.root_schema) + } - fn serialize_map(self, len: Option) -> Result { - self.serialize_map_with_schema(len, self.root_schema) - } + fn serialize_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + self.serialize_struct_with_schema(name, len, self.root_schema) + } - fn serialize_struct( - self, - name: &'static str, - len: usize, - ) -> Result { - self.serialize_struct_with_schema(name, len, self.root_schema) - } + fn serialize_struct_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.serialize_struct_variant_with_schema( + name, + variant_index, + variant, + len, + self.root_schema, + ) + } - fn serialize_struct_variant( - self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, - ) -> Result { - self.serialize_struct_variant_with_schema( - name, - variant_index, - variant, - len, - self.root_schema, - ) - } + fn is_human_readable(&self) -> bool { + crate::util::is_human_readable() + } +} - fn is_human_readable(&self) -> bool { - crate::util::is_human_readable() - } +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::sync::SchemaExt; + use crate::{ + Days, Duration, Millis, Months, decimal::Decimal, error::Details, + schema::ResolvedSchema, + }; + use apache_avro_test_helper::TestResult; + use bigdecimal::BigDecimal; + use num_bigint::{BigInt, Sign}; + use serde::Serialize; + use serde_bytes::{ByteArray, Bytes}; + use serial_test::serial; + use std::{ + collections::{BTreeMap, HashMap}, + marker::PhantomData, + sync::atomic::Ordering, + }; + use uuid::Uuid; + + #[test] + fn test_serialize_null() -> TestResult { + let schema = Schema::Null; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + ().serialize(&mut serializer)?; + None::<()>.serialize(&mut serializer)?; + None::.serialize(&mut serializer)?; + None::.serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); + + assert_eq!(buffer.as_slice(), Vec::::new().as_slice()); + + Ok(()) } - #[cfg(test)] - mod tests { - use super::*; - use crate::schema::sync::SchemaExt; - use crate::{ - Days, Duration, Millis, Months, decimal::Decimal, error::Details, - schema::ResolvedSchema, - }; - use apache_avro_test_helper::TestResult; - use bigdecimal::BigDecimal; - use num_bigint::{BigInt, Sign}; - use serde::Serialize; - use serde_bytes::{ByteArray, Bytes}; - use serial_test::serial; - use std::{ - collections::{BTreeMap, HashMap}, - marker::PhantomData, - sync::atomic::Ordering, - }; - use uuid::Uuid; + #[test] + fn test_serialize_bool() -> TestResult { + let schema = Schema::Boolean; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - #[test] - fn test_serialize_null() -> TestResult { - let schema = Schema::Null; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + true.serialize(&mut serializer)?; + false.serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - ().serialize(&mut serializer)?; - None::<()>.serialize(&mut serializer)?; - None::.serialize(&mut serializer)?; - None::.serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + assert_eq!(buffer.as_slice(), &[1, 0]); - assert_eq!(buffer.as_slice(), Vec::::new().as_slice()); + Ok(()) + } - Ok(()) - } + #[test] + fn test_serialize_int() -> TestResult { + let schema = Schema::Int; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 4u8.serialize(&mut serializer)?; + 31u16.serialize(&mut serializer)?; + 13u32.serialize(&mut serializer)?; + 7i8.serialize(&mut serializer)?; + (-57i16).serialize(&mut serializer)?; + 129i32.serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); + + assert_eq!(buffer.as_slice(), &[8, 62, 26, 14, 113, 130, 2]); + + Ok(()) + } - #[test] - fn test_serialize_bool() -> TestResult { - let schema = Schema::Boolean; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + #[test] + fn test_serialize_long() -> TestResult { + let schema = Schema::Long; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 4u8.serialize(&mut serializer)?; + 31u16.serialize(&mut serializer)?; + 13u32.serialize(&mut serializer)?; + 291u64.serialize(&mut serializer)?; + 7i8.serialize(&mut serializer)?; + (-57i16).serialize(&mut serializer)?; + 129i32.serialize(&mut serializer)?; + (-432i64).serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); + + assert_eq!( + buffer.as_slice(), + &[8, 62, 26, 198, 4, 14, 113, 130, 2, 223, 6] + ); + + Ok(()) + } - true.serialize(&mut serializer)?; - false.serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + #[test] + fn test_serialize_float() -> TestResult { + let schema = Schema::Float; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - assert_eq!(buffer.as_slice(), &[1, 0]); + 4.7f32.serialize(&mut serializer)?; + (-14.1f64).serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - Ok(()) - } + assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); - #[test] - fn test_serialize_int() -> TestResult { - let schema = Schema::Int; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + Ok(()) + } - 4u8.serialize(&mut serializer)?; - 31u16.serialize(&mut serializer)?; - 13u32.serialize(&mut serializer)?; - 7i8.serialize(&mut serializer)?; - (-57i16).serialize(&mut serializer)?; - 129i32.serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + #[test] + fn test_serialize_double() -> TestResult { + let schema = Schema::Float; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - assert_eq!(buffer.as_slice(), &[8, 62, 26, 14, 113, 130, 2]); + 4.7f32.serialize(&mut serializer)?; + (-14.1f64).serialize(&mut serializer)?; + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - Ok(()) - } + assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); - #[test] - fn test_serialize_long() -> TestResult { - let schema = Schema::Long; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + Ok(()) + } + + #[test] + fn test_serialize_bytes() -> TestResult { + let schema = Schema::Bytes; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 'a'.serialize(&mut serializer)?; + "test".serialize(&mut serializer)?; + Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; + assert!(().serialize(&mut serializer).is_err()); + assert!(PhantomData::.serialize(&mut serializer).is_err()); + + assert_eq!( + buffer.as_slice(), + &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] + ); + + Ok(()) + } - 4u8.serialize(&mut serializer)?; - 31u16.serialize(&mut serializer)?; - 13u32.serialize(&mut serializer)?; - 291u64.serialize(&mut serializer)?; - 7i8.serialize(&mut serializer)?; - (-57i16).serialize(&mut serializer)?; - 129i32.serialize(&mut serializer)?; - (-432i64).serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + #[test] + fn test_serialize_string() -> TestResult { + let schema = Schema::String; + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 'a'.serialize(&mut serializer)?; + "test".serialize(&mut serializer)?; + Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; + assert!(().serialize(&mut serializer).is_err()); + assert!(PhantomData::.serialize(&mut serializer).is_err()); + + assert_eq!( + buffer.as_slice(), + &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] + ); + + Ok(()) + } - assert_eq!( - buffer.as_slice(), - &[8, 62, 26, 198, 4, 14, 113, 130, 2, 223, 6] - ); + #[test] + fn test_serialize_record() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "record", + "name": "TestRecord", + "fields": [ + {"name": "stringField", "type": "string"}, + {"name": "intField", "type": "int"} + ] + }"#, + ) + ?; + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct GoodTestRecord { + string_field: String, + int_field: i32, + } + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct BadTestRecord { + foo_string_field: String, + bar_int_field: i32, + } + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let good_record = GoodTestRecord { + string_field: String::from("test"), + int_field: 10, + }; + good_record.serialize(&mut serializer)?; - Ok(()) - } + let bad_record = BadTestRecord { + foo_string_field: String::from("test"), + bar_int_field: 10, + }; + assert!(bad_record.serialize(&mut serializer).is_err()); - #[test] - fn test_serialize_float() -> TestResult { - let schema = Schema::Float; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert!("".serialize(&mut serializer).is_err()); + assert!(Some("").serialize(&mut serializer).is_err()); - 4.7f32.serialize(&mut serializer)?; - (-14.1f64).serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + assert_eq!(buffer.as_slice(), &[8, b't', b'e', b's', b't', 20]); - assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); + Ok(()) + } - Ok(()) + #[test] + fn test_serialize_empty_record() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "record", + "name": "EmptyRecord", + "fields": [] + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + #[derive(Serialize)] + struct EmptyRecord; + EmptyRecord.serialize(&mut serializer)?; + + #[derive(Serialize)] + struct NonEmptyRecord { + foo: String, + } + let record = NonEmptyRecord { + foo: "bar".to_string(), + }; + match record + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::FieldName(field_name)) if field_name == "foo" => (), + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_double() -> TestResult { - let schema = Schema::Float; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + match ().serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); // serialize_unit() delegates to serialize_none() + assert_eq!(value, "None. Cause: Expected: Record. Got: Null"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), + } - 4.7f32.serialize(&mut serializer)?; - (-14.1f64).serialize(&mut serializer)?; - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); + assert_eq!(buffer.len(), 0); - assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]); + Ok(()) + } - Ok(()) + #[test] + fn test_serialize_enum() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "enum", + "name": "Suit", + "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] + }"#, + ) + ?; + + #[derive(Serialize)] + enum Suit { + Spades, + Hearts, + Diamonds, + Clubs, + } + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + Suit::Spades.serialize(&mut serializer)?; + Suit::Hearts.serialize(&mut serializer)?; + Suit::Diamonds.serialize(&mut serializer)?; + Suit::Clubs.serialize(&mut serializer)?; + match None::<()> + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); + assert_eq!(value, "None. Cause: Expected: Enum. Got: Null"); + assert_eq!(schema, schema); + } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_bytes() -> TestResult { - let schema = Schema::Bytes; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!(buffer.as_slice(), &[0, 2, 4, 6]); - 'a'.serialize(&mut serializer)?; - "test".serialize(&mut serializer)?; - Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; - assert!(().serialize(&mut serializer).is_err()); - assert!(PhantomData::.serialize(&mut serializer).is_err()); - - assert_eq!( - buffer.as_slice(), - &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] - ); - - Ok(()) - } - - #[test] - fn test_serialize_string() -> TestResult { - let schema = Schema::String; - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 'a'.serialize(&mut serializer)?; - "test".serialize(&mut serializer)?; - Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?; - assert!(().serialize(&mut serializer).is_err()); - assert!(PhantomData::.serialize(&mut serializer).is_err()); - - assert_eq!( - buffer.as_slice(), - &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4] - ); - - Ok(()) - } - - #[test] - fn test_serialize_record() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "record", - "name": "TestRecord", - "fields": [ - {"name": "stringField", "type": "string"}, - {"name": "intField", "type": "int"} - ] - }"#, - ) - ?; - - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct GoodTestRecord { - string_field: String, - int_field: i32, - } + Ok(()) + } - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct BadTestRecord { - foo_string_field: String, - bar_int_field: i32, + #[test] + fn test_serialize_array() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "array", + "items": "long" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let arr: Vec = vec![10, 5, 400]; + arr.serialize(&mut serializer)?; + + match vec![1_f32] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "1. Cause: Expected: Long. Got: Float"); + assert_eq!(schema, schema); } - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - let good_record = GoodTestRecord { - string_field: String::from("test"), - int_field: 10, - }; - good_record.serialize(&mut serializer)?; - - let bad_record = BadTestRecord { - foo_string_field: String::from("test"), - bar_int_field: 10, - }; - assert!(bad_record.serialize(&mut serializer).is_err()); - - assert!("".serialize(&mut serializer).is_err()); - assert!(Some("").serialize(&mut serializer).is_err()); - - assert_eq!(buffer.as_slice(), &[8, b't', b'e', b's', b't', 20]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_empty_record() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "record", - "name": "EmptyRecord", - "fields": [] - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - #[derive(Serialize)] - struct EmptyRecord; - EmptyRecord.serialize(&mut serializer)?; + assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0]); - #[derive(Serialize)] - struct NonEmptyRecord { - foo: String, - } - let record = NonEmptyRecord { - foo: "bar".to_string(), - }; - match record - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::FieldName(field_name)) if field_name == "foo" => (), - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } + Ok(()) + } - match ().serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); // serialize_unit() delegates to serialize_none() - assert_eq!(value, "None. Cause: Expected: Record. Got: Null"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + fn test_serialize_map() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "map", + "values": "long" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let mut map: BTreeMap = BTreeMap::new(); + map.insert(String::from("item1"), 10); + map.insert(String::from("item2"), 400); + + map.serialize(&mut serializer)?; + + let mut map: BTreeMap = BTreeMap::new(); + map.insert(String::from("item1"), "value1"); + match map.serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "string"); + assert_eq!(value, "value1. Cause: Expected: Long. Got: String"); + assert_eq!(schema, schema); } - - assert_eq!(buffer.len(), 0); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_enum() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "enum", - "name": "Suit", - "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] - }"#, - ) - ?; - - #[derive(Serialize)] - enum Suit { - Spades, - Hearts, - Diamonds, - Clubs, - } + assert_eq!( + buffer.as_slice(), + &[ + 4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, + 6, 0 + ] + ); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + Ok(()) + } - Suit::Spades.serialize(&mut serializer)?; - Suit::Hearts.serialize(&mut serializer)?; - Suit::Diamonds.serialize(&mut serializer)?; - Suit::Clubs.serialize(&mut serializer)?; - match None::<()> - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, + #[test] + fn test_serialize_nullable_union() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": ["null", "long"] + }"#, + ) + ?; + + #[derive(Serialize)] + enum NullableLong { + Null, + Long(i64), + } + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + Some(10i64).serialize(&mut serializer)?; + None::.serialize(&mut serializer)?; + NullableLong::Long(400).serialize(&mut serializer)?; + NullableLong::Null.serialize(&mut serializer)?; + + match "invalid" + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "string"); + assert_eq!( value, - schema, - }) => { - assert_eq!(value_type, "none"); - assert_eq!(value, "None. Cause: Expected: Enum. Got: Null"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + "invalid. Cause: Expected one of the union variants [Null, Long]. Got: String" + ); + assert_eq!(schema, schema); } - - assert_eq!(buffer.as_slice(), &[0, 2, 4, 6]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_array() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "array", - "items": "long" - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!(buffer.as_slice(), &[2, 20, 0, 2, 160, 6, 0]); - let arr: Vec = vec![10, 5, 400]; - arr.serialize(&mut serializer)?; + Ok(()) + } - match vec![1_f32] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, + #[test] + fn test_serialize_union() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": ["null", "long", "string"] + }"#, + ) + ?; + + #[derive(Serialize)] + enum LongOrString { + Null, + Long(i64), + Str(String), + } + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + LongOrString::Null.serialize(&mut serializer)?; + LongOrString::Long(400).serialize(&mut serializer)?; + LongOrString::Str(String::from("test")).serialize(&mut serializer)?; + + match 1_f64 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f64"); + assert_eq!( value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "1. Cause: Expected: Long. Got: Float"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + "1. Cause: Cannot find a Double schema in [Null, Long, String]" + ); + assert_eq!(schema, schema); } - - assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_map() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "map", - "values": "long" - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - let mut map: BTreeMap = BTreeMap::new(); - map.insert(String::from("item1"), 10); - map.insert(String::from("item2"), 400); + assert_eq!( + buffer.as_slice(), + &[0, 2, 160, 6, 4, 8, b't', b'e', b's', b't'] + ); - map.serialize(&mut serializer)?; + Ok(()) + } - let mut map: BTreeMap = BTreeMap::new(); - map.insert(String::from("item1"), "value1"); - match map.serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, + #[test] + fn test_serialize_fixed() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "fixed", + "size": 8, + "name": "LongVal" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?; + + // non-8 size + match Bytes::new(&[123]) + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "bytes"); + assert_eq!( value, - schema, - }) => { - assert_eq!(value_type, "string"); - assert_eq!(value, "value1. Cause: Expected: Long. Got: String"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + "7b. Cause: Fixed schema size (8) does not match the value length (1)" + ); // Bytes represents its values as hexadecimals: '7b' is 123 + assert_eq!(schema, schema); } - - assert_eq!( - buffer.as_slice(), - &[ - 4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, - 6, 0 - ] - ); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_nullable_union() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": ["null", "long"] - }"#, - ) - ?; - - #[derive(Serialize)] - enum NullableLong { - Null, - Long(i64), - } - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - Some(10i64).serialize(&mut serializer)?; - None::.serialize(&mut serializer)?; - NullableLong::Long(400).serialize(&mut serializer)?; - NullableLong::Null.serialize(&mut serializer)?; - - match "invalid" - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "string"); - assert_eq!( - value, - "invalid. Cause: Expected one of the union variants [Null, Long]. Got: String" - ); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + // array + match [1; 8] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! + assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); + assert_eq!(schema, schema); } - - assert_eq!(buffer.as_slice(), &[2, 20, 0, 2, 160, 6, 0]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_union() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": ["null", "long", "string"] - }"#, - ) - ?; - - #[derive(Serialize)] - enum LongOrString { - Null, - Long(i64), - Str(String), - } - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - LongOrString::Null.serialize(&mut serializer)?; - LongOrString::Long(400).serialize(&mut serializer)?; - LongOrString::Str(String::from("test")).serialize(&mut serializer)?; - - match 1_f64 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f64"); - assert_eq!( - value, - "1. Cause: Cannot find a Double schema in [Null, Long, String]" - ); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + // slice + match &[1, 2, 3, 4, 5, 6, 7, 8] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(*value_type, "tuple"); // TODO: why is this 'tuple' ?! + assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); + assert_eq!(schema, schema); } - - assert_eq!( - buffer.as_slice(), - &[0, 2, 160, 6, 4, 8, b't', b'e', b's', b't'] - ); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_fixed() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "fixed", - "size": 8, - "name": "LongVal" - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?; - - // non-8 size - match Bytes::new(&[123]) - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "bytes"); - assert_eq!( - value, - "7b. Cause: Fixed schema size (8) does not match the value length (1)" - ); // Bytes represents its values as hexadecimals: '7b' is 123 - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } + assert_eq!(buffer.as_slice(), &[10, 124, 31, 97, 14, 201, 3, 88]); - // array - match [1; 8] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! - assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } + Ok(()) + } - // slice - match &[1, 2, 3, 4, 5, 6, 7, 8] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(*value_type, "tuple"); // TODO: why is this 'tuple' ?! - assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + fn test_serialize_decimal_bytes() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "bytes", + "logicalType": "decimal", + "precision": 16, + "scale": 2 + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let val = Decimal::from(&[251, 155]); + val.serialize(&mut serializer)?; + + match ().serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); + assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); + assert_eq!(schema, schema); } - - assert_eq!(buffer.as_slice(), &[10, 124, 31, 97, 14, 201, 3, 88]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_decimal_bytes() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "bytes", - "logicalType": "decimal", - "precision": 16, - "scale": 2 - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!(buffer.as_slice(), &[4, 251, 155]); - let val = Decimal::from(&[251, 155]); - val.serialize(&mut serializer)?; + Ok(()) + } - match ().serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); - assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + fn test_serialize_decimal_fixed() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "fixed", + "name": "FixedDecimal", + "size": 8, + "logicalType": "decimal", + "precision": 16, + "scale": 2 + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]); + val.serialize(&mut serializer)?; + + match ().serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "none"); + assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); + assert_eq!(schema, schema); } - - assert_eq!(buffer.as_slice(), &[4, 251, 155]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_decimal_fixed() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "fixed", - "name": "FixedDecimal", - "size": 8, - "logicalType": "decimal", - "precision": 16, - "scale": 2 - }"#, - ) - ?; + assert_eq!(buffer.as_slice(), &[0, 0, 0, 0, 0, 0, 251, 155]); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + Ok(()) + } - let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]); - val.serialize(&mut serializer)?; + #[test] + #[serial(serde_is_human_readable)] + fn test_serialize_bigdecimal() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "bytes", + "logicalType": "big-decimal" + }"#, + ) + ?; + + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2); + val.serialize(&mut serializer)?; + + assert_eq!(buffer.as_slice(), &[10, 6, 0, 195, 104, 4]); + + Ok(()) + } - match ().serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "none"); - assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + #[serial(serde_is_human_readable)] + fn test_serialize_uuid() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "string", + "logicalType": "uuid" + }"#, + ) + ?; + + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + "8c28da81-238c-4326-bddd-4e3d00cc5099" + .parse::()? + .serialize(&mut serializer)?; + + match 1_u8.serialize(&mut serializer).map_err(Error::into_details) { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "u8"); + assert_eq!(value, "1. Cause: Expected: Uuid. Got: Int"); + assert_eq!(schema, schema); } - - assert_eq!(buffer.as_slice(), &[0, 0, 0, 0, 0, 0, 251, 155]); - - Ok(()) - } - - #[test] - #[serial(serde_is_human_readable)] - fn test_serialize_bigdecimal() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "bytes", - "logicalType": "big-decimal" - }"#, - ) - ?; - - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2); - val.serialize(&mut serializer)?; - - assert_eq!(buffer.as_slice(), &[10, 6, 0, 195, 104, 4]); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - #[serial(serde_is_human_readable)] - fn test_serialize_uuid() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "string", - "logicalType": "uuid" - }"#, - ) - ?; - - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!( + buffer.as_slice(), + &[ + 72, b'8', b'c', b'2', b'8', b'd', b'a', b'8', b'1', b'-', b'2', b'3', b'8', + b'c', b'-', b'4', b'3', b'2', b'6', b'-', b'b', b'd', b'd', b'd', b'-', b'4', + b'e', b'3', b'd', b'0', b'0', b'c', b'c', b'5', b'0', b'9', b'9' + ] + ); - "8c28da81-238c-4326-bddd-4e3d00cc5099" - .parse::()? - .serialize(&mut serializer)?; + Ok(()) + } - match 1_u8.serialize(&mut serializer).map_err(Error::into_details) { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "u8"); - assert_eq!(value, "1. Cause: Expected: Uuid. Got: Int"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + fn test_serialize_date() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "int", + "logicalType": "date" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 100_u8.serialize(&mut serializer)?; + 1000_u16.serialize(&mut serializer)?; + 10000_u32.serialize(&mut serializer)?; + 1000_i16.serialize(&mut serializer)?; + 10000_i32.serialize(&mut serializer)?; + + match 10000_f32 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "10000. Cause: Expected: Date. Got: Float"); + assert_eq!(schema, schema); } - - assert_eq!( - buffer.as_slice(), - &[ - 72, b'8', b'c', b'2', b'8', b'd', b'a', b'8', b'1', b'-', b'2', b'3', b'8', - b'c', b'-', b'4', b'3', b'2', b'6', b'-', b'b', b'd', b'd', b'd', b'-', b'4', - b'e', b'3', b'd', b'0', b'0', b'c', b'c', b'5', b'0', b'9', b'9' - ] - ); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_date() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "int", - "logicalType": "date" - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + assert_eq!( + buffer.as_slice(), + &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] + ); - 100_u8.serialize(&mut serializer)?; - 1000_u16.serialize(&mut serializer)?; - 10000_u32.serialize(&mut serializer)?; - 1000_i16.serialize(&mut serializer)?; - 10000_i32.serialize(&mut serializer)?; + Ok(()) + } - match 10000_f32 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "10000. Cause: Expected: Date. Got: Float"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + fn test_serialize_time_millis() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "int", + "logicalType": "time-millis" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 100_u8.serialize(&mut serializer)?; + 1000_u16.serialize(&mut serializer)?; + 10000_u32.serialize(&mut serializer)?; + 1000_i16.serialize(&mut serializer)?; + 10000_i32.serialize(&mut serializer)?; + + match 10000_f32 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "10000. Cause: Expected: TimeMillis. Got: Float"); + assert_eq!(schema, schema); } - - assert_eq!( - buffer.as_slice(), - &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] - ); - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_time_millis() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "int", - "logicalType": "time-millis" - }"#, - ) - ?; + assert_eq!( + buffer.as_slice(), + &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] + ); - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 100_u8.serialize(&mut serializer)?; - 1000_u16.serialize(&mut serializer)?; - 10000_u32.serialize(&mut serializer)?; - 1000_i16.serialize(&mut serializer)?; - 10000_i32.serialize(&mut serializer)?; + Ok(()) + } - match 10000_f32 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "10000. Cause: Expected: TimeMillis. Got: Float"); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), + #[test] + fn test_serialize_time_micros() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "long", + "logicalType": "time-micros" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + 100_u8.serialize(&mut serializer)?; + 1000_u16.serialize(&mut serializer)?; + 10000_u32.serialize(&mut serializer)?; + 1000_i16.serialize(&mut serializer)?; + 10000_i32.serialize(&mut serializer)?; + 10000_i64.serialize(&mut serializer)?; + + match 10000_f32 + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "f32"); + assert_eq!(value, "10000. Cause: Expected: TimeMicros. Got: Float"); + assert_eq!(schema, schema); } + unexpected => panic!("Expected an error. Got: {unexpected:?}"), + } - assert_eq!( - buffer.as_slice(), - &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1] - ); + assert_eq!( + buffer.as_slice(), + &[ + 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1 + ] + ); - Ok(()) - } + Ok(()) + } - #[test] - fn test_serialize_time_micros() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ + #[test] + fn test_serialize_timestamp() -> TestResult { + for precision in ["millis", "micros", "nanos"] { + let schema = SchemaExt::parse_str(&format!( + r#"{{ "type": "long", - "logicalType": "time-micros" - }"#, - ) + "logicalType": "timestamp-{precision}" + }}"# + )) ?; let mut buffer: Vec = Vec::new(); @@ -2689,7 +2738,7 @@ 10000_i32.serialize(&mut serializer)?; 10000_i64.serialize(&mut serializer)?; - match 10000_f32 + match 10000_f64 .serialize(&mut serializer) .map_err(Error::into_details) { @@ -2698,8 +2747,17 @@ value, schema, }) => { - assert_eq!(value_type, "f32"); - assert_eq!(value, "10000. Cause: Expected: TimeMicros. Got: Float"); + let mut capital_precision = precision.to_string(); + if let Some(c) = capital_precision.chars().next() { + capital_precision.replace_range(..1, &c.to_uppercase().to_string()); + } + assert_eq!(value_type, "f64"); + assert_eq!( + value, + format!( + "10000. Cause: Expected: Timestamp{capital_precision}. Got: Double" + ) + ); assert_eq!(schema, schema); } unexpected => panic!("Expected an error. Got: {unexpected:?}"), @@ -2711,176 +2769,118 @@ 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1 ] ); - - Ok(()) } - #[test] - fn test_serialize_timestamp() -> TestResult { - for precision in ["millis", "micros", "nanos"] { - let schema = SchemaExt::parse_str(&format!( - r#"{{ - "type": "long", - "logicalType": "timestamp-{precision}" - }}"# - )) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - 100_u8.serialize(&mut serializer)?; - 1000_u16.serialize(&mut serializer)?; - 10000_u32.serialize(&mut serializer)?; - 1000_i16.serialize(&mut serializer)?; - 10000_i32.serialize(&mut serializer)?; - 10000_i64.serialize(&mut serializer)?; - - match 10000_f64 - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - let mut capital_precision = precision.to_string(); - if let Some(c) = capital_precision.chars().next() { - capital_precision.replace_range(..1, &c.to_uppercase().to_string()); - } - assert_eq!(value_type, "f64"); - assert_eq!( - value, - format!( - "10000. Cause: Expected: Timestamp{capital_precision}. Got: Double" - ) - ); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } + Ok(()) + } + #[test] + fn test_serialize_duration() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "fixed", + "size": 12, + "name": "duration", + "logicalType": "duration" + }"#, + ) + ?; + + let mut buffer: Vec = Vec::new(); + let names = HashMap::new(); + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + + let duration_bytes = ByteArray::new( + Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into(), + ); + duration_bytes.serialize(&mut serializer)?; + + match [1; 12] + .serialize(&mut serializer) + .map_err(Error::into_details) + { + Err(Details::SerializeValueWithSchema { + value_type, + value, + schema, + }) => { + assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! assert_eq!( - buffer.as_slice(), - &[ - 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1 - ] + value, + "tuple (len=12). Cause: Expected: Duration. Got: Array" ); + assert_eq!(schema, schema); } - - Ok(()) + unexpected => panic!("Expected an error. Got: {unexpected:?}"), } - #[test] - fn test_serialize_duration() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "fixed", - "size": 12, - "name": "duration", - "logicalType": "duration" - }"#, - ) - ?; - - let mut buffer: Vec = Vec::new(); - let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - - let duration_bytes = ByteArray::new( - Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into(), - ); - duration_bytes.serialize(&mut serializer)?; + assert_eq!(buffer.as_slice(), &[3, 0, 0, 0, 2, 0, 0, 0, 176, 4, 0, 0]); - match [1; 12] - .serialize(&mut serializer) - .map_err(Error::into_details) - { - Err(Details::SerializeValueWithSchema { - value_type, - value, - schema, - }) => { - assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?! - assert_eq!( - value, - "tuple (len=12). Cause: Expected: Duration. Got: Array" - ); - assert_eq!(schema, schema); - } - unexpected => panic!("Expected an error. Got: {unexpected:?}"), - } - - assert_eq!(buffer.as_slice(), &[3, 0, 0, 0, 2, 0, 0, 0, 176, 4, 0, 0]); - - Ok(()) - } + Ok(()) + } - #[test] - #[serial(serde_is_human_readable)] // for BigDecimal and Uuid - fn test_serialize_recursive_record() -> TestResult { - let schema = SchemaExt::parse_str( - r#"{ - "type": "record", - "name": "TestRecord", - "fields": [ - {"name": "stringField", "type": "string"}, - {"name": "intField", "type": "int"}, - {"name": "bigDecimalField", "type": {"type": "bytes", "logicalType": "big-decimal"}}, - {"name": "uuidField", "type": "fixed", "size": 16, "logicalType": "uuid"}, - {"name": "innerRecord", "type": ["null", "TestRecord"]} + #[test] + #[serial(serde_is_human_readable)] // for BigDecimal and Uuid + fn test_serialize_recursive_record() -> TestResult { + let schema = SchemaExt::parse_str( + r#"{ + "type": "record", + "name": "TestRecord", + "fields": [ + {"name": "stringField", "type": "string"}, + {"name": "intField", "type": "int"}, + {"name": "bigDecimalField", "type": {"type": "bytes", "logicalType": "big-decimal"}}, + {"name": "uuidField", "type": "fixed", "size": 16, "logicalType": "uuid"}, + {"name": "innerRecord", "type": ["null", "TestRecord"]} + ] + }"#, + )?; + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct TestRecord { + string_field: String, + int_field: i32, + big_decimal_field: BigDecimal, + uuid_field: Uuid, + // #[serde(skip_serializing_if = "Option::is_none")] => Never ignore None! + inner_record: Option>, + } + + crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); + let mut buffer: Vec = Vec::new(); + let rs = ResolvedSchema::try_from(&schema)?; + let mut serializer = + SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None); + + let good_record = TestRecord { + string_field: String::from("test"), + int_field: 10, + big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2), + uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5098".parse::()?, + inner_record: Some(Box::new(TestRecord { + string_field: String::from("inner_test"), + int_field: 100, + big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![20038]), 2), + uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5099".parse::()?, + inner_record: None, + })), + }; + good_record.serialize(&mut serializer)?; + + assert_eq!( + buffer.as_slice(), + &[ + 8, 116, 101, 115, 116, 20, 10, 6, 0, 195, 104, 4, 72, 56, 99, 50, 56, 100, 97, + 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, + 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 56, 2, 20, 105, 110, 110, 101, 114, + 95, 116, 101, 115, 116, 200, 1, 8, 4, 78, 70, 4, 72, 56, 99, 50, 56, 100, 97, + 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, + 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 57, 0 ] - }"#, - )?; - - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct TestRecord { - string_field: String, - int_field: i32, - big_decimal_field: BigDecimal, - uuid_field: Uuid, - // #[serde(skip_serializing_if = "Option::is_none")] => Never ignore None! - inner_record: Option>, - } + ); - crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); - let mut buffer: Vec = Vec::new(); - let rs = ResolvedSchema::try_from(&schema)?; - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None); - - let good_record = TestRecord { - string_field: String::from("test"), - int_field: 10, - big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2), - uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5098".parse::()?, - inner_record: Some(Box::new(TestRecord { - string_field: String::from("inner_test"), - int_field: 100, - big_decimal_field: BigDecimal::new(BigInt::new(Sign::Plus, vec![20038]), 2), - uuid_field: "8c28da81-238c-4326-bddd-4e3d00cc5099".parse::()?, - inner_record: None, - })), - }; - good_record.serialize(&mut serializer)?; - - assert_eq!( - buffer.as_slice(), - &[ - 8, 116, 101, 115, 116, 20, 10, 6, 0, 195, 104, 4, 72, 56, 99, 50, 56, 100, 97, - 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, - 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 56, 2, 20, 105, 110, 110, 101, 114, - 95, 116, 101, 115, 116, 200, 1, 8, 4, 78, 70, 4, 72, 56, 99, 50, 56, 100, 97, - 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, - 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 57, 0 - ] - ); - - Ok(()) - } + Ok(()) } +} diff --git a/avro/src/types.rs b/avro/src/types.rs index a7423e97..6e131f8e 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -730,8 +730,8 @@ mod types { && SchemaKind::from(schema) != SchemaKind::Union { // Pull out the Union, and attempt to resolve against it. - let v = match *value { - Value::Union(_i, b) => *b, + let v = match value { + Value::Union(_i, b) => b, _ => unreachable!(), }; value = &v; @@ -1806,7 +1806,7 @@ Field with name '"b"' is not a member of the map items"#, )) ); assert!( - ValueExt::validate(value, &union_schema) + ValueExt::validate(&value, &union_schema) .await ); @@ -3253,7 +3253,7 @@ Field with name '"b"' is not a member of the map items"#, let schema = r#"{"name": "bigDecimalSchema", "logicalType": "big-decimal", "type": "bytes" }"#; - let avro_value = Value::BigDecimal(BigDecimal::from(12345678u32)); + let avro_value = Value::BigDecimal(bigdecimal::BigDecimal::from(12345678u32)); let schema = SchemaExt::parse_str(schema).await?; let resolve_result: AvroResult = ValueExt::resolve(avro_value, &schema).await; assert!( diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 8724275b..07f85eda 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -844,9 +844,6 @@ mod writer { #[cfg(test)] mod tests { - #[synca::cfg(sync)] - use std::{cell::RefCell, rc::Rc}; - use super::*; use crate::{ decimal::Decimal, @@ -1018,7 +1015,7 @@ mod writer { &Schema::Int, 1_i32, ) - .await + .await } #[tokio::test] @@ -1030,7 +1027,7 @@ mod writer { &Schema::Int, 1_i32, ) - .await + .await } #[tokio::test] @@ -1042,7 +1039,7 @@ mod writer { &Schema::Long, 1_i64, ) - .await + .await } #[tokio::test] @@ -1054,7 +1051,7 @@ mod writer { &Schema::Long, 1_i64, ) - .await + .await } #[tokio::test] @@ -1066,7 +1063,7 @@ mod writer { &Schema::Long, 1_i64, ) - .await + .await } #[tokio::test] @@ -1109,7 +1106,7 @@ mod writer { &inner, value, ) - .await + .await } #[tokio::test] @@ -1591,7 +1588,7 @@ mod writer { &TestSingleObjectWriter::get_schema().await, 1024, ) - .expect("Should resolve schema"); + .expect("Should resolve schema"); let value = obj.into(); let written_bytes = writer .write_value_ref(&value, &mut buf) @@ -1615,7 +1612,7 @@ mod writer { &TestSingleObjectWriter::get_schema().await, &mut msg_binary, ).await - .expect("encode should have failed by here as a dependency of any writing"); + .expect("encode should have failed by here as a dependency of any writing"); assert_eq!(&buf[10..], &msg_binary[..]); Ok(()) @@ -1636,7 +1633,7 @@ mod writer { 1024, header_builder, ) - .expect("Should resolve schema"); + .expect("Should resolve schema"); let value = obj.into(); writer .write_value_ref(&value, &mut buf) @@ -1666,7 +1663,7 @@ mod writer { &TestSingleObjectWriter::get_schema(), 1024, ) - .expect("Should resolve schema"); + .expect("Should resolve schema"); let mut specific_writer = SpecificSingleObjectWriter::::with_capacity(1024) .await @@ -1758,60 +1755,63 @@ mod writer { } Ok(()) } - - #[synca::cfg(sync)] - #[test] - fn avro_4063_flush_applies_to_inner_writer() -> TestResult { - const SCHEMA: &str = r#" - { - "type": "record", - "name": "ExampleSchema", - "fields": [ - {"name": "exampleField", "type": "string"} - ] - } - "#; - - #[derive(Clone, Default)] - struct TestBuffer(Rc>>); - - impl TestBuffer { - fn len(&self) -> usize { - self.0.borrow().len() - } - } - - impl std::io::Write for TestBuffer { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - std::io::Write::write(&mut self.0.borrow_mut(), buf) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - let shared_buffer = TestBuffer::default(); - - let buffered_writer = std::io::BufWriter::new(shared_buffer.clone()); - - let schema = SchemaExt::parse_str(SCHEMA).await?; - - let mut writer = Writer::new(&schema, buffered_writer); - - let mut record = Record::new(writer.schema()).unwrap(); - record.put("exampleField", "value"); - - writer.append(record).await?; - writer.flush().await?; - - assert_eq!( - shared_buffer.len(), - 167, - "the test buffer was not fully written to after Writer::flush was called" - ); - - Ok(()) - } } + + // #[cfg(feature = "sync")] + // #[test] + // fn avro_4063_flush_applies_to_inner_writer() -> TestResult { + // const SCHEMA: &str = r#" + // { + // "type": "record", + // "name": "ExampleSchema", + // "fields": [ + // {"name": "exampleField", "type": "string"} + // ] + // } + // "#; + // use std::{cell::RefCell, rc::Rc}; + // use crate::writer::sync::Writer; + // use crate::schema::sync::SchemaExt; + // + // #[derive(Clone, Default)] + // struct TestBuffer(Rc>>); + // + // impl TestBuffer { + // fn len(&self) -> usize { + // self.0.borrow().len() + // } + // } + // + // impl std::io::Write for TestBuffer { + // fn write(&mut self, buf: &[u8]) -> std::io::Result { + // self.0.borrow_mut().write(buf) + // } + // + // fn flush(&mut self) -> std::io::Result<()> { + // Ok(()) + // } + // } + // + // let shared_buffer = TestBuffer::default(); + // + // let buffered_writer = std::io::BufWriter::new(shared_buffer.clone()); + // + // let schema = SchemaExt::parse_str(SCHEMA)?; + // + // let mut writer = Writer::new(&schema, buffered_writer); + // + // let mut record = Record::new(writer.schema()).unwrap(); + // record.put("exampleField", "value"); + // + // writer.append(record)?; + // writer.flush()?; + // + // assert_eq!( + // shared_buffer.len(), + // 167, + // "the test buffer was not fully written to after Writer::flush was called" + // ); + // + // Ok(()) + // } } From 2cb22665bdb6a041deb15d8e664c16ac12666929 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 10:54:59 +0300 Subject: [PATCH 24/47] Import (and thus compile) ser_schema.rs only when `sync` feature is enabled Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/bytes.rs | 2 +- avro/src/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index dadf8dff..a30d2b82 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -306,7 +306,7 @@ mod tests { use crate::de::tokio::from_value; use crate::schema::tokio::SchemaExt; use crate::ser::tokio::to_value; - use crate::types::{{tokio::ValueExt}, Value}; + use crate::types::{tokio::ValueExt, Value}; use apache_avro_test_helper::TestResult; use serde::{Deserialize, Serialize}; diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 975f92ca..752b17f8 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -881,6 +881,7 @@ mod duration; mod encode; mod reader; mod ser; +#[cfg(feature = "sync")] mod ser_schema; mod util; mod writer; From 90f938ef2a566373b34cb29c842c01ba229a8162 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 10:58:15 +0300 Subject: [PATCH 25/47] The `rstest` tests will be run only for `sync` for now The compilation fails with cfg_attr Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/schema_compatibility.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index eda1aab5..b304752c 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -565,6 +565,7 @@ mod schema_compatibility { use apache_avro_test_helper::TestResult; #[synca::cfg(tokio)] use futures::StreamExt; + #[synca::cfg(sync)] use rstest::*; async fn int_array_schema() -> Schema { @@ -817,7 +818,7 @@ mod schema_compatibility { ); } - #[cfg_attr(feature = "tokio", tokio::test)] + #[synca::cfg(sync)] #[rstest] // Record type test #[case( @@ -886,7 +887,7 @@ mod schema_compatibility { r#"{"type": "array", "items": "int"}"#, r#"{"type": "array", "items": "long"}"# )] - async fn test_avro_3950_match_schemas_ok( + fn test_avro_3950_match_schemas_ok( #[case] writer_schema_str: &str, #[case] reader_schema_str: &str, ) { @@ -896,7 +897,7 @@ mod schema_compatibility { assert!(SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).is_ok()); } - #[cfg_attr(feature = "tokio", tokio::test)] + #[synca::cfg(sync)] #[rstest] // Record type test #[case( @@ -1067,7 +1068,7 @@ mod schema_compatibility { r#"{"type": "fixed", "name": "EmployeeId", "size": 16}"#, CompatibilityError::Inconclusive(String::from("writers_schema")) )] - async fn test_avro_3950_match_schemas_error( + fn test_avro_3950_match_schemas_error( #[case] writer_schema_str: &str, #[case] reader_schema_str: &str, #[case] expected_error: CompatibilityError, From 1d88f57427782e862a7946c6fa77cb63d084db04 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 11:32:13 +0300 Subject: [PATCH 26/47] WIP: Fix the signature of SchemaExt::parse_reader() Signed-off-by: Martin Tzvetanov Grigorov --- avro/Cargo.toml | 8 ++++---- avro/examples/benchmark.rs | 5 +---- avro/examples/generate_interop_data.rs | 7 ++----- avro/src/schema.rs | 4 ++-- avro/src/types.rs | 2 +- avro/tests/append_to_existing.rs | 5 +---- avro/tests/avro-3786.rs | 16 +++++++-------- avro/tests/avro-3787.rs | 7 ++++--- avro/tests/codecs.rs | 5 +---- avro/tests/io.rs | 5 +++-- avro/tests/schema.rs | 24 +++++++++-------------- avro/tests/shared.rs | 2 +- avro/tests/to_from_avro_datum_schemata.rs | 5 +---- avro/tests/union_schema.rs | 2 +- avro_derive/Cargo.toml | 5 +++++ avro_derive/tests/derive.rs | 6 +++--- 16 files changed, 47 insertions(+), 61 deletions(-) diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 4f224eb6..409632d7 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -118,7 +118,7 @@ required-features = ["sync"] [[example]] name = "specific_single_object" path = "examples/specific_single_object.rs" -required-features = ["sync"] +required-features = ["sync", "derive"] [[example]] name = "test_interop_data" @@ -138,7 +138,7 @@ required-features = ["sync"] [[test]] name = "append_to_existing" path = "tests/append_to_existing.rs" -required-features = ["sync"] +required-features = ["sync", "derive"] [[test]] name = "avro-3786" @@ -158,7 +158,7 @@ required-features = ["sync"] [[test]] name = "avro-rs-226" path = "tests/avro-rs-226.rs" -required-features = ["sync"] +required-features = ["sync", "derive"] [[test]] name = "big_decimal" @@ -201,6 +201,6 @@ path = "tests/uuids.rs" required-features = ["sync"] [[test]] -name = "validatrs" +name = "validators" path = "tests/validators.rs" required-features = ["sync"] diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index d5d38ff9..cbb30fd8 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{ - Reader, Schema, Writer, - types::{Record, Value}, -}; +use apache_avro::{Reader, Schema, Writer, types::{Record, Value}, SchemaExt}; use apache_avro_test_helper::TestResult; use std::{ io::{BufReader, BufWriter}, diff --git a/avro/examples/generate_interop_data.rs b/avro/examples/generate_interop_data.rs index 57ef327a..bbf46241 100644 --- a/avro/examples/generate_interop_data.rs +++ b/avro/examples/generate_interop_data.rs @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{ - Codec, Schema, Writer, - types::{Record, Value}, -}; +use apache_avro::{Codec, Schema, Writer, types::{Record, Value}, SchemaExt}; use std::{ collections::HashMap, error::Error, @@ -108,7 +105,7 @@ fn main() -> Result<(), Box> { Ok(()) } -fn write_user_metadata(writer: &mut Writer>) -> Result<(), Box> { +fn write_user_metadata(writer: &mut Writer>) -> Result<(), Box> { writer.add_user_metadata("user_metadata".to_string(), b"someByteArray")?; Ok(()) diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 931bf5db..ba717c7f 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1730,8 +1730,8 @@ mod schema { } /// Create a `Schema` from a reader which implements [`Read`]. - pub async fn parse_reader( - reader: &mut (impl AvroRead + ?Sized + Unpin), + pub async fn parse_reader( + reader: &mut R, ) -> AvroResult { let mut buf = String::new(); match reader.read_to_string(&mut buf).await { diff --git a/avro/src/types.rs b/avro/src/types.rs index 6e131f8e..52417278 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -323,7 +323,7 @@ impl TryFrom for serde_json::Value { Value::Decimal(ref d) => >::try_from(d) .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), Value::BigDecimal(ref bg) => { - let vec1: Vec = crate::bigdecimal::sync::serialize_big_decimal(bg)?; + let vec1: Vec = vec![];//crate::bigdecimal::sync::serialize_big_decimal(bg)?; Ok(serde_json::Value::Array(vec1.into_iter().map(|b| b.into()).collect())) } Value::TimeMillis(t) => Ok(serde_json::Value::Number(t.into())), diff --git a/avro/tests/append_to_existing.rs b/avro/tests/append_to_existing.rs index c087cd3d..a1eafd2b 100644 --- a/avro/tests/append_to_existing.rs +++ b/avro/tests/append_to_existing.rs @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{ - AvroResult, Reader, Schema, Writer, read_marker, - types::{Record, Value}, -}; +use apache_avro::{AvroResult, Reader, Schema, Writer, read_marker, types::{Record, Value}, SchemaExt}; use apache_avro_test_helper::TestResult; const SCHEMA: &str = r#"{ diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index f63e1dd3..b3a9e78d 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::schema::Schema; use apache_avro::types::Value; -use apache_avro::{from_avro_datum, to_avro_datum, to_value}; +use apache_avro::{from_avro_datum, to_avro_datum, to_value, SchemaExt}; +use apache_avro::types::sync::ValueExt; use apache_avro_test_helper::TestResult; #[test] @@ -131,7 +131,7 @@ fn avro_3786_deserialize_union_with_different_enum_order() -> TestResult { }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; @@ -255,7 +255,7 @@ fn avro_3786_deserialize_union_with_different_enum_order_defined_in_record() -> }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; @@ -368,7 +368,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; @@ -481,7 +481,7 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_ }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; @@ -594,7 +594,7 @@ fn deserialize_union_with_different_enum_order_defined_in_record() -> TestResult }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; @@ -868,7 +868,7 @@ fn deserialize_union_with_record_with_enum_defined_inline_reader_has_different_i }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index 4d87b71f..bc371ebd 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::Schema; +use apache_avro::SchemaExt; use apache_avro::types::Value; use apache_avro::{from_avro_datum, to_avro_datum, to_value}; +use apache_avro::types::sync::ValueExt; use apache_avro_test_helper::TestResult; #[test] @@ -132,7 +133,7 @@ fn avro_3787_deserialize_union_with_unknown_symbol() -> TestResult { }; let avro_value = to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; @@ -259,7 +260,7 @@ fn avro_3787_deserialize_union_with_unknown_symbol_no_ref() -> TestResult { }; let avro_value = to_value(foo2)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; diff --git a/avro/tests/codecs.rs b/avro/tests/codecs.rs index b4603bc0..f9fca887 100644 --- a/avro/tests/codecs.rs +++ b/avro/tests/codecs.rs @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{ - Codec, DeflateSettings, Reader, Schema, Writer, - types::{Record, Value}, -}; +use apache_avro::{Codec, DeflateSettings, Reader, Writer, types::{Record, Value}, SchemaExt}; use apache_avro_test_helper::TestResult; use miniz_oxide::deflate::CompressionLevel; diff --git a/avro/tests/io.rs b/avro/tests/io.rs index c1166506..a9a94de7 100644 --- a/avro/tests/io.rs +++ b/avro/tests/io.rs @@ -16,10 +16,11 @@ // under the License. //! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py -use apache_avro::{Error, Schema, error::Details, from_avro_datum, to_avro_datum, types::Value}; +use apache_avro::{Error, Schema, error::Details, from_avro_datum, to_avro_datum, types::Value, SchemaExt}; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use std::{io::Cursor, sync::OnceLock}; +use apache_avro::types::sync::ValueExt; fn schemas_to_validate() -> &'static Vec<(&'static str, Value)> { static SCHEMAS_TO_VALIDATE_ONCE: OnceLock> = OnceLock::new(); @@ -219,7 +220,7 @@ fn test_validate() -> TestResult { for (raw_schema, value) in schemas_to_validate().iter() { let schema = SchemaExt::parse_str(raw_schema)?; assert!( - value.validate(&schema), + ValueExt::validate(&value, &schema), "value {value:?} does not validate schema: {raw_schema}" ); } diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index 3e04c7fa..de7797e3 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -17,17 +17,11 @@ use std::{ collections::HashMap, - io::{Cursor, Read}, -}; - -use apache_avro::{ - Codec, Reader, Writer, - error::{Details, Error}, - from_avro_datum, from_value, - schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, - to_avro_datum, to_value, - types::{Record, Value}, + io::Cursor, }; +use std::io::Read; +use apache_avro::{Codec, Reader, Writer, error::{Details, Error}, from_avro_datum, from_value, schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, to_avro_datum, to_value, types::{Record, Value}, SchemaExt}; +use apache_avro::types::sync::ValueExt; use apache_avro_test_helper::{ TestResult, data::{DOC_EXAMPLES, examples, valid_examples}, @@ -109,7 +103,8 @@ fn test_parse() -> TestResult { fn test_3799_parse_reader() -> TestResult { init(); for (raw_schema, valid) in examples().iter() { - let schema = SchemaExt::parse_reader(&mut Cursor::new(raw_schema)); + let mut reader: &mut (dyn Read + Unpin) = &mut Cursor::new(raw_schema); + let schema = SchemaExt::parse_reader(&mut reader); if *valid { assert!( schema.is_ok(), @@ -125,7 +120,7 @@ fn test_3799_parse_reader() -> TestResult { // Ensure it works for trait objects too. for (raw_schema, valid) in examples().iter() { - let reader: &mut dyn Read = &mut Cursor::new(raw_schema); + let reader = &mut Cursor::new(raw_schema); let schema = SchemaExt::parse_reader(reader); if *valid { assert!( @@ -145,8 +140,7 @@ fn test_3799_parse_reader() -> TestResult { #[test] fn test_3799_raise_io_error_from_parse_read() -> Result<(), String> { // 0xDF is invalid for UTF-8. - let mut invalid_data = Cursor::new([0xDF]); - + let mut invalid_data: &mut (dyn Read + Unpin) = &mut Cursor::new([0xDF]); let error = SchemaExt::parse_reader(&mut invalid_data) .unwrap_err() .into_details(); @@ -995,7 +989,7 @@ fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_ }; let avro_value = crate::to_value(foo1)?; assert!( - avro_value.validate(&writer_schema), + ValueExt::validate(&avro_value, &writer_schema), "value is valid for schema", ); let datum = to_avro_datum(&writer_schema, avro_value)?; diff --git a/avro/tests/shared.rs b/avro/tests/shared.rs index 5d598a1d..c40a10e8 100644 --- a/avro/tests/shared.rs +++ b/avro/tests/shared.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, Reader, Schema, Writer, types::Value}; +use apache_avro::{Codec, Reader, Schema, Writer, types::Value, SchemaExt}; use apache_avro_test_helper::TestResult; use std::{ fmt, diff --git a/avro/tests/to_from_avro_datum_schemata.rs b/avro/tests/to_from_avro_datum_schemata.rs index 546ecd0c..b204e79e 100644 --- a/avro/tests/to_from_avro_datum_schemata.rs +++ b/avro/tests/to_from_avro_datum_schemata.rs @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{ - Codec, Reader, Schema, Writer, from_avro_datum_reader_schemata, from_avro_datum_schemata, - to_avro_datum_schemata, types::Value, -}; +use apache_avro::{Codec, Reader, Schema, Writer, from_avro_datum_reader_schemata, from_avro_datum_schemata, to_avro_datum_schemata, types::Value, SchemaExt}; use apache_avro_test_helper::{TestResult, init}; static SCHEMA_A_STR: &str = r#"{ diff --git a/avro/tests/union_schema.rs b/avro/tests/union_schema.rs index e3401f94..3dbaaff4 100644 --- a/avro/tests/union_schema.rs +++ b/avro/tests/union_schema.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{AvroResult, Codec, Reader, Schema, Writer, from_value}; +use apache_avro::{AvroResult, Codec, Reader, Schema, Writer, from_value, SchemaExt}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; static SCHEMA_A_STR: &str = r#"{ diff --git a/avro_derive/Cargo.toml b/avro_derive/Cargo.toml index 92f1dd5b..4fa05d3c 100644 --- a/avro_derive/Cargo.toml +++ b/avro_derive/Cargo.toml @@ -51,3 +51,8 @@ pretty_assertions = { workspace = true } [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docsrs"] + +[[test]] +name = "derive" +path = "tests/derive.rs" +required-features = ["sync"] diff --git a/avro_derive/tests/derive.rs b/avro_derive/tests/derive.rs index f75e5a2a..f01ec10d 100644 --- a/avro_derive/tests/derive.rs +++ b/avro_derive/tests/derive.rs @@ -19,15 +19,15 @@ mod test_derive { use apache_avro::{ - AsyncReader, AsyncSchema, AsyncWriter, AvroSchema, async_from_value, - derive::AvroSchemaComponent, + Reader, Schema, Writer, AvroSchema, from_value, + schema::sync::AvroSchemaComponent, schema::sync::SchemaExt, }; use apache_avro_derive::*; use proptest::prelude::*; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use std::collections::HashMap; - use apache_avro::schema::tokio::{Alias, EnumSchema, RecordSchema}; + use apache_avro::schema::{Alias, EnumSchema, RecordSchema}; use std::{borrow::Cow, sync::Mutex}; use super::*; From d47035f98fd19caa7f05dae98b6bd1d5078325bb Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 11:59:01 +0300 Subject: [PATCH 27/47] Move AvroSchema out of Synca's scope Now the build passes for both sync and async! But some test hangs ... Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 12 -- avro/Cargo.toml | 4 +- avro/src/lib.rs | 5 +- avro/src/reader.rs | 61 ++++--- avro/src/schema.rs | 395 ++++++++++++++++++++++----------------------- avro/src/writer.rs | 31 ++-- 6 files changed, 241 insertions(+), 267 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44993373..ae06cbaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,7 +51,6 @@ dependencies = [ "anyhow", "apache-avro-derive", "apache-avro-test-helper", - "async-trait", "bigdecimal", "bon", "bzip2", @@ -112,17 +111,6 @@ dependencies = [ "log", ] -[[package]] -name = "async-trait" -version = "0.1.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "autocfg" version = "1.4.0" diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 409632d7..2530663f 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -36,7 +36,7 @@ snappy = ["dep:crc32fast", "dep:snap"] xz = ["dep:xz2"] zstandard = ["dep:zstd"] sync = [] -tokio = ["dep:tokio", "dep:futures", "dep:async-trait"] +tokio = ["dep:tokio", "dep:futures"] [lib] # disable benchmarks to allow passing criterion arguments to `cargo bench` @@ -80,8 +80,6 @@ zstd = { default-features = false, version = "0.13.3", optional = true } #synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing" } synca = { path = "/home/martin/git/rust/rs_synca/synca" } futures = { version = "0.3.31", optional = true } -async-trait = { version = "0.1.88", optional = true } - [target.'cfg(target_arch = "wasm32")'.dependencies] quad-rand = { default-features = false, version = "0.2.3" } diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 752b17f8..3a2a7e69 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -931,10 +931,7 @@ pub use reader::tokio::{ from_avro_datum_schemata as async_from_avro_datum_schemata, read_marker as async_read_marker, }; pub use schema::Schema; -#[cfg(feature = "sync")] -pub use schema::sync::AvroSchema; -#[cfg(feature = "tokio")] -pub use schema::tokio::AvroSchema as AsyncAvroSchema; +pub use schema::AvroSchema; #[cfg(feature = "sync")] pub use ser::sync::to_value; #[cfg(feature = "tokio")] diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 54328fa3..93ad73c0 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -58,7 +58,8 @@ mod reader { error::Details, error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, - schema::tokio::{AvroSchema, SchemaExt}, + schema::AvroSchema, + schema::tokio::SchemaExt, schema::{ Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, resolve_names_with_schemata, @@ -662,7 +663,7 @@ mod reader { { pub async fn new() -> AvroResult> { Ok(SpecificSingleObjectReader { - inner: GenericSingleObjectReader::new(T::get_schema().await)?, + inner: GenericSingleObjectReader::new(T::get_schema())?, _model: PhantomData, }) } @@ -959,9 +960,9 @@ mod reader { c: Vec, } - #[cfg_attr(feature = "tokio", async_trait::async_trait)] + #[synca::cfg(sync)] impl AvroSchema for TestSingleObjectReader { - async fn get_schema() -> Schema { + fn get_schema() -> Schema { let schema = r#" { "type":"record", @@ -985,7 +986,7 @@ mod reader { ] } "#; - SchemaExt::parse_str(schema).await.unwrap() + SchemaExt::parse_str(schema).unwrap() } } @@ -1033,8 +1034,9 @@ mod reader { } } - #[tokio::test] - async fn test_avro_3507_single_object_reader() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn test_avro_3507_single_object_reader() -> TestResult { let obj = TestSingleObjectReader { a: 42, b: 3.33, @@ -1044,23 +1046,21 @@ mod reader { to_read.extend_from_slice(&[0xC3, 0x01]); to_read.extend_from_slice( &TestSingleObjectReader::get_schema() - .await .fingerprint::() .bytes[..], ); encode( &obj.clone().into(), - &TestSingleObjectReader::get_schema().await, + &TestSingleObjectReader::get_schema(), &mut to_read, - ).await + ) .expect("Encode should succeed"); let mut to_read = &to_read[..]; let generic_reader = - GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) .expect("Schema should resolve"); let val = generic_reader .read_value(&mut to_read) - .await .expect("Should read"); let expected_value: Value = obj.into(); assert_eq!(expected_value, val); @@ -1068,8 +1068,9 @@ mod reader { Ok(()) } - #[tokio::test] - async fn avro_3642_test_single_object_reader_incomplete_reads() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn avro_3642_test_single_object_reader_incomplete_reads() -> TestResult { let obj = TestSingleObjectReader { a: 42, b: 3.33, @@ -1080,16 +1081,15 @@ mod reader { let mut to_read_2 = Vec::::new(); to_read_2.extend_from_slice( &TestSingleObjectReader::get_schema() - .await .fingerprint::() .bytes[..], ); let mut to_read_3 = Vec::::new(); encode( &obj.clone().into(), - &TestSingleObjectReader::get_schema().await, + &TestSingleObjectReader::get_schema(), &mut to_read_3, - ).await + ) .expect("Encode should succeed"); #[synca::cfg(sync)] @@ -1097,11 +1097,10 @@ mod reader { #[synca::cfg(tokio)] let mut to_read = tokio::io::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); let generic_reader = - GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) .expect("Schema should resolve"); let val = generic_reader .read_value(&mut to_read) - .await .expect("Should read"); let expected_value: Value = obj.into(); assert_eq!(expected_value, val); @@ -1109,8 +1108,9 @@ mod reader { Ok(()) } - #[tokio::test] - async fn test_avro_3507_reader_parity() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn test_avro_3507_reader_parity() -> TestResult { let obj = TestSingleObjectReader { a: 42, b: 3.33, @@ -1121,21 +1121,19 @@ mod reader { to_read.extend_from_slice(&[0xC3, 0x01]); to_read.extend_from_slice( &TestSingleObjectReader::get_schema() - .await .fingerprint::() .bytes[..], ); encode( &obj.clone().into(), - &TestSingleObjectReader::get_schema().await, + &TestSingleObjectReader::get_schema(), &mut to_read, - ).await + ) .expect("Encode should succeed"); let generic_reader = - GenericSingleObjectReader::new(TestSingleObjectReader::get_schema().await) + GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) .expect("Schema should resolve"); let specific_reader = SpecificSingleObjectReader::::new() - .await .expect("schema should resolve"); let mut to_read1 = &to_read[..]; let mut to_read2 = &to_read[..]; @@ -1143,15 +1141,12 @@ mod reader { let val = generic_reader .read_value(&mut to_read1) - .await .expect("Should read"); let read_obj1 = specific_reader .read_from_value(&mut to_read2) - .await .expect("Should read from value"); let read_obj2 = specific_reader .read(&mut to_read3) - .await .expect("Should read from deserilize"); let expected_value: Value = obj.clone().into(); assert_eq!(obj, read_obj1); @@ -1161,12 +1156,13 @@ mod reader { Ok(()) } - #[tokio::test] - async fn avro_rs_164_generic_reader_alternate_header() -> TestResult { + #[test] + #[synca::cfg(sync)] + fn avro_rs_164_generic_reader_alternate_header() -> TestResult { let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); let generic_reader = GenericSingleObjectReader::new_with_header_builder( - TestSingleObjectReader::get_schema().await, + TestSingleObjectReader::get_schema(), header_builder, ) .expect("failed to build reader"); @@ -1176,7 +1172,6 @@ mod reader { let mut to_read = &data_to_read[..]; let read_result = generic_reader .read_value(&mut to_read) - .await .map_err(Error::into_details); matches!(read_result, Err(Details::ReadBytes(_))); Ok(()) diff --git a/avro/src/schema.rs b/avro/src/schema.rs index ba717c7f..39a835ad 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1339,6 +1339,201 @@ impl Serialize for RecordField { } } +/// Trait for types that serve as an Avro data model. Derive implementation available +/// through `derive` feature. Do not implement directly! +/// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait +/// through a blanket implementation. +pub trait AvroSchema { + fn get_schema() -> Schema; +} + + +#[cfg(feature = "derive")] +pub mod derive { + use super::*; + use std::borrow::Cow; + + /// Trait for types that serve as fully defined components inside an Avro data model. Derive + /// implementation available through `derive` feature. This is what is implemented by + /// the `derive(AvroSchema)` macro. + /// + /// # Implementation guide + /// + ///### Simple implementation + /// To construct a non named simple schema, it is possible to ignore the input argument making the + /// general form implementation look like + /// ```ignore + /// impl AvroSchemaComponent for AType { + /// fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { + /// Schema::? + /// } + ///} + /// ``` + /// ### Passthrough implementation + /// To construct a schema for a Type that acts as in "inner" type, such as for smart pointers, simply + /// pass through the arguments to the inner type + /// ```ignore + /// impl AvroSchemaComponent for PassthroughType { + /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { + /// InnerType::get_schema_in_ctxt(names, enclosing_namespace) + /// } + ///} + /// ``` + ///### Complex implementation + /// To implement this for Named schema there is a general form needed to avoid creating invalid + /// schemas or infinite loops. + /// ```ignore + /// impl AvroSchemaComponent for ComplexType { + /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { + /// // Create the fully qualified name for your type given the enclosing namespace + /// let name = apache_avro::schema::Name::new("MyName") + /// .expect("Unable to parse schema name") + /// .fully_qualified_name(enclosing_namespace); + /// let enclosing_namespace = &name.namespace; + /// // Check, if your name is already defined, and if so, return a ref to that name + /// if named_schemas.contains_key(&name) { + /// apache_avro::schema::Schema::Ref{name: name.clone()} + /// } else { + /// named_schemas.insert(name.clone(), apache_avro::schema::Schema::Ref{name: name.clone()}); + /// // YOUR SCHEMA DEFINITION HERE with the name equivalent to "MyName". + /// // For non-simple sub types delegate to their implementation of AvroSchemaComponent + /// } + /// } + ///} + /// ``` + pub trait AvroSchemaComponent { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema; + } + + impl AvroSchema for T + where + T: AvroSchemaComponent, + { + fn get_schema() -> Schema { + T::get_schema_in_ctxt(&mut HashMap::default(), &None) + } + } + + macro_rules! impl_schema ( + ($type:ty, $variant_constructor:expr) => ( + impl AvroSchemaComponent for $type { + fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { + $variant_constructor + } + } + ); + ); + + impl_schema!(bool, Schema::Boolean); + impl_schema!(i8, Schema::Int); + impl_schema!(i16, Schema::Int); + impl_schema!(i32, Schema::Int); + impl_schema!(i64, Schema::Long); + impl_schema!(u8, Schema::Int); + impl_schema!(u16, Schema::Int); + impl_schema!(u32, Schema::Long); + impl_schema!(f32, Schema::Float); + impl_schema!(f64, Schema::Double); + impl_schema!(String, Schema::String); + impl_schema!(uuid::Uuid, Schema::Uuid); + impl_schema!(core::time::Duration, Schema::Duration); + + impl AvroSchemaComponent for Vec + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + Schema::array(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + } + } + + impl AvroSchemaComponent for Option + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + let inner_schema = T::get_schema_in_ctxt(named_schemas, enclosing_namespace); + Schema::Union(UnionSchema { + schemas: vec![Schema::Null, inner_schema.clone()], + variant_index: vec![Schema::Null, inner_schema] + .iter() + .enumerate() + .map(|(idx, s)| (SchemaKind::from(s), idx)) + .collect(), + }) + } + } + + impl AvroSchemaComponent for serde_json::Map + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + } + } + + impl AvroSchemaComponent for HashMap + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) + } + } + + impl AvroSchemaComponent for Box + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + } + } + + impl AvroSchemaComponent for std::sync::Mutex + where + T: AvroSchemaComponent, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + } + } + + impl AvroSchemaComponent for Cow<'_, T> + where + T: AvroSchemaComponent + Clone, + { + fn get_schema_in_ctxt( + named_schemas: &mut Names, + enclosing_namespace: &Namespace, + ) -> Schema { + T::get_schema_in_ctxt(named_schemas, enclosing_namespace) + } + } +} + #[synca::synca( #[cfg(feature = "tokio")] pub mod tokio {}, @@ -2553,202 +2748,6 @@ mod schema { } } - /// Trait for types that serve as an Avro data model. Derive implementation available - /// through `derive` feature. Do not implement directly! - /// Implement `apache_avro::schema::derive::AvroSchemaComponent` to get this trait - /// through a blanket implementation. - #[cfg_attr(feature = "tokio", async_trait::async_trait)] - pub trait AvroSchema { - async fn get_schema() -> Schema; - } - - #[cfg(feature = "derive")] - pub mod derive { - use super::*; - use std::borrow::Cow; - - /// Trait for types that serve as fully defined components inside an Avro data model. Derive - /// implementation available through `derive` feature. This is what is implemented by - /// the `derive(AvroSchema)` macro. - /// - /// # Implementation guide - /// - ///### Simple implementation - /// To construct a non named simple schema, it is possible to ignore the input argument making the - /// general form implementation look like - /// ```ignore - /// impl AvroSchemaComponent for AType { - /// fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { - /// Schema::? - /// } - ///} - /// ``` - /// ### Passthrough implementation - /// To construct a schema for a Type that acts as in "inner" type, such as for smart pointers, simply - /// pass through the arguments to the inner type - /// ```ignore - /// impl AvroSchemaComponent for PassthroughType { - /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { - /// InnerType::get_schema_in_ctxt(names, enclosing_namespace) - /// } - ///} - /// ``` - ///### Complex implementation - /// To implement this for Named schema there is a general form needed to avoid creating invalid - /// schemas or infinite loops. - /// ```ignore - /// impl AvroSchemaComponent for ComplexType { - /// fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) -> Schema { - /// // Create the fully qualified name for your type given the enclosing namespace - /// let name = apache_avro::schema::Name::new("MyName") - /// .expect("Unable to parse schema name") - /// .fully_qualified_name(enclosing_namespace); - /// let enclosing_namespace = &name.namespace; - /// // Check, if your name is already defined, and if so, return a ref to that name - /// if named_schemas.contains_key(&name) { - /// apache_avro::schema::Schema::Ref{name: name.clone()} - /// } else { - /// named_schemas.insert(name.clone(), apache_avro::schema::Schema::Ref{name: name.clone()}); - /// // YOUR SCHEMA DEFINITION HERE with the name equivalent to "MyName". - /// // For non-simple sub types delegate to their implementation of AvroSchemaComponent - /// } - /// } - ///} - /// ``` - pub trait AvroSchemaComponent { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema; - } - - #[cfg_attr(feature = "tokio", async_trait::async_trait)] - impl AvroSchema for T - where - T: AvroSchemaComponent, - { - async fn get_schema() -> Schema { - T::get_schema_in_ctxt(&mut HashMap::default(), &None) - } - } - - macro_rules! impl_schema ( - ($type:ty, $variant_constructor:expr) => ( - impl AvroSchemaComponent for $type { - fn get_schema_in_ctxt(_: &mut Names, _: &Namespace) -> Schema { - $variant_constructor - } - } - ); - ); - - impl_schema!(bool, Schema::Boolean); - impl_schema!(i8, Schema::Int); - impl_schema!(i16, Schema::Int); - impl_schema!(i32, Schema::Int); - impl_schema!(i64, Schema::Long); - impl_schema!(u8, Schema::Int); - impl_schema!(u16, Schema::Int); - impl_schema!(u32, Schema::Long); - impl_schema!(f32, Schema::Float); - impl_schema!(f64, Schema::Double); - impl_schema!(String, Schema::String); - impl_schema!(uuid::Uuid, Schema::Uuid); - impl_schema!(core::time::Duration, Schema::Duration); - - impl AvroSchemaComponent for Vec - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - Schema::array(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) - } - } - - impl AvroSchemaComponent for Option - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - let inner_schema = T::get_schema_in_ctxt(named_schemas, enclosing_namespace); - Schema::Union(UnionSchema { - schemas: vec![Schema::Null, inner_schema.clone()], - variant_index: vec![Schema::Null, inner_schema] - .iter() - .enumerate() - .map(|(idx, s)| (SchemaKind::from(s), idx)) - .collect(), - }) - } - } - - impl AvroSchemaComponent for serde_json::Map - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) - } - } - - impl AvroSchemaComponent for HashMap - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - Schema::map(T::get_schema_in_ctxt(named_schemas, enclosing_namespace)) - } - } - - impl AvroSchemaComponent for Box - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - T::get_schema_in_ctxt(named_schemas, enclosing_namespace) - } - } - - impl AvroSchemaComponent for std::sync::Mutex - where - T: AvroSchemaComponent, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - T::get_schema_in_ctxt(named_schemas, enclosing_namespace) - } - } - - impl AvroSchemaComponent for Cow<'_, T> - where - T: AvroSchemaComponent + Clone, - { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema { - T::get_schema_in_ctxt(named_schemas, enclosing_namespace) - } - } - } - #[cfg(test)] mod tests { use super::*; @@ -6931,8 +6930,7 @@ mod schema { id: crate::Uuid, } - #[cfg_attr(feature = "tokio", async_trait::async_trait)] - impl AvroSchema for Comment { + impl crate::AvroSchema for Comment { fn get_schema() -> Schema { SchemaExt::parse_str( r#"{ @@ -6949,7 +6947,6 @@ mod schema { } ] }"#, ) - .await .expect("Invalid Comment Avro schema") } } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 07f85eda..9a1f43a8 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -57,8 +57,7 @@ mod writer { error::Details, error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, - schema::tokio::AvroSchema, - schema::{ResolvedOwnedSchema, ResolvedSchema, Schema}, + schema::{AvroSchema, ResolvedOwnedSchema, ResolvedSchema, Schema}, types::Value, }; #[synca::cfg(sync)] @@ -662,7 +661,7 @@ mod writer { T: AvroSchema, { pub async fn with_capacity(buffer_cap: usize) -> AvroResult> { - let schema = T::get_schema().await; + let schema = T::get_schema(); Ok(SpecificSingleObjectWriter { inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, schema, @@ -864,6 +863,7 @@ mod writer { use crate::{codec::tokio::DeflateSettings, error::Details}; use apache_avro_test_helper::TestResult; + use crate::schema::AvroSchema; const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); @@ -1533,9 +1533,9 @@ mod writer { c: Vec, } - #[cfg_attr(feature = "tokio", async_trait::async_trait)] + #[synca::cfg(sync)] impl AvroSchema for TestSingleObjectWriter { - async fn get_schema() -> Schema { + fn get_schema() -> Schema { let schema = r#" { "type":"record", @@ -1559,7 +1559,7 @@ mod writer { ] } "#; - SchemaExt::parse_str(schema).await.unwrap() + SchemaExt::parse_str(schema).unwrap() } } @@ -1576,8 +1576,9 @@ mod writer { } } - #[tokio::test] - async fn test_single_object_writer() -> TestResult { + #[synca::cfg(sync)] + #[test] + fn test_single_object_writer() -> TestResult { let mut buf: Vec = Vec::new(); let obj = TestSingleObjectWriter { a: 300, @@ -1585,14 +1586,13 @@ mod writer { c: vec!["cat".into(), "dog".into()], }; let mut writer = GenericSingleObjectWriter::new_with_capacity( - &TestSingleObjectWriter::get_schema().await, + &TestSingleObjectWriter::get_schema(), 1024, ) .expect("Should resolve schema"); let value = obj.into(); let written_bytes = writer .write_value_ref(&value, &mut buf) - .await .expect("Error serializing properly"); assert!(buf.len() > 10, "no bytes written"); @@ -1602,23 +1602,23 @@ mod writer { assert_eq!( &buf[2..10], &TestSingleObjectWriter::get_schema() - .await .fingerprint::() .bytes[..] ); let mut msg_binary = Vec::new(); encode( &value, - &TestSingleObjectWriter::get_schema().await, + &TestSingleObjectWriter::get_schema(), &mut msg_binary, - ).await + ) .expect("encode should have failed by here as a dependency of any writing"); assert_eq!(&buf[10..], &msg_binary[..]); Ok(()) } - #[tokio::test] + #[test] + #[synca::cfg(sync)] async fn test_single_object_writer_with_header_builder() -> TestResult { let mut buf: Vec = Vec::new(); let obj = TestSingleObjectWriter { @@ -1629,7 +1629,7 @@ mod writer { let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder( - &TestSingleObjectWriter::get_schema().await, + &TestSingleObjectWriter::get_schema(), 1024, header_builder, ) @@ -1637,7 +1637,6 @@ mod writer { let value = obj.into(); writer .write_value_ref(&value, &mut buf) - .await .expect("Error serializing properly"); assert_eq!(buf[0], 0x03); From c3d472561e2ec3da4ba4016383467d082096a4a6 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 13:37:46 +0300 Subject: [PATCH 28/47] format the code Signed-off-by: Martin Tzvetanov Grigorov --- avro/examples/benchmark.rs | 5 +- avro/examples/generate_interop_data.rs | 9 +- avro/src/bigdecimal.rs | 7 +- avro/src/bytes.rs | 2 +- avro/src/de.rs | 2 +- avro/src/decode.rs | 64 ++-- avro/src/encode.rs | 41 ++- avro/src/headers.rs | 2 +- avro/src/lib.rs | 10 +- avro/src/reader.rs | 6 +- avro/src/schema.rs | 83 +++-- avro/src/ser_schema.rs | 276 +++++----------- avro/src/types.rs | 378 ++++++++++++++-------- avro/src/util.rs | 3 +- avro/src/writer.rs | 91 +++--- avro/tests/append_to_existing.rs | 5 +- avro/tests/avro-3786.rs | 2 +- avro/tests/avro-3787.rs | 2 +- avro/tests/codecs.rs | 5 +- avro/tests/io.rs | 6 +- avro/tests/schema.rs | 16 +- avro/tests/shared.rs | 2 +- avro/tests/to_from_avro_datum_schemata.rs | 5 +- avro/tests/union_schema.rs | 2 +- avro_derive/tests/derive.rs | 4 +- 25 files changed, 561 insertions(+), 467 deletions(-) diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index cbb30fd8..b79cbccd 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Reader, Schema, Writer, types::{Record, Value}, SchemaExt}; +use apache_avro::{ + Reader, Schema, SchemaExt, Writer, + types::{Record, Value}, +}; use apache_avro_test_helper::TestResult; use std::{ io::{BufReader, BufWriter}, diff --git a/avro/examples/generate_interop_data.rs b/avro/examples/generate_interop_data.rs index bbf46241..2f46db4f 100644 --- a/avro/examples/generate_interop_data.rs +++ b/avro/examples/generate_interop_data.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, Schema, Writer, types::{Record, Value}, SchemaExt}; +use apache_avro::{ + Codec, Schema, SchemaExt, Writer, + types::{Record, Value}, +}; use std::{ collections::HashMap, error::Error, @@ -105,7 +108,9 @@ fn main() -> Result<(), Box> { Ok(()) } -fn write_user_metadata(writer: &mut Writer>) -> Result<(), Box> { +fn write_user_metadata( + writer: &mut Writer>, +) -> Result<(), Box> { writer.add_user_metadata("user_metadata".to_string(), b"someByteArray")?; Ok(()) diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 118acdbc..44c5b584 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -83,7 +83,8 @@ mod bigdecimal { use tokio::io::AsyncReadExt; bytes - .read_exact(&mut big_decimal_buffer[..]).await + .read_exact(&mut big_decimal_buffer[..]) + .await .map_err(Details::ReadDouble)?; match decode_long(&mut bytes).await { @@ -100,8 +101,8 @@ mod bigdecimal { mod tests { use super::*; use crate::{ - codec::tokio::Codec, error::Error, reader::tokio::Reader, - schema::tokio::SchemaExt, types::Record, writer::tokio::Writer, + codec::tokio::Codec, error::Error, reader::tokio::Reader, schema::tokio::SchemaExt, + types::Record, writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; use bigdecimal::{One, Zero}; diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index a30d2b82..5f5f1dbf 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -306,7 +306,7 @@ mod tests { use crate::de::tokio::from_value; use crate::schema::tokio::SchemaExt; use crate::ser::tokio::to_value; - use crate::types::{tokio::ValueExt, Value}; + use crate::types::{Value, tokio::ValueExt}; use apache_avro_test_helper::TestResult; use serde::{Deserialize, Serialize}; diff --git a/avro/src/de.rs b/avro/src/de.rs index e3e80aff..11251453 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -797,8 +797,8 @@ mod de { #[cfg(test)] mod tests { - use crate::schema::tokio::SchemaExt; use crate::reader::tokio::from_avro_datum; + use crate::schema::tokio::SchemaExt; use crate::ser::tokio::to_value; use crate::writer::tokio::to_avro_datum; use num_bigint::BigInt; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 06f1d25d..eed85b90 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -159,8 +159,13 @@ mod decode { } } Schema::Uuid => { - let Value::Bytes(bytes) = - Box::pin(decode_internal(&Schema::Bytes, names, enclosing_namespace, reader)).await? + let Value::Bytes(bytes) = Box::pin(decode_internal( + &Schema::Bytes, + names, + enclosing_namespace, + reader, + )) + .await? else { // Calling decode_internal with Schema::Bytes can only return a Value::Bytes or an error unreachable!(); @@ -169,7 +174,8 @@ mod decode { let uuid = if bytes.len() == 16 { Uuid::from_slice(&bytes).map_err(Details::ConvertSliceToUuid)? } else { - let string = std::str::from_utf8(&bytes).map_err(Details::ConvertToUtf8Error)?; + let string = + std::str::from_utf8(&bytes).map_err(Details::ConvertToUtf8Error)?; Uuid::parse_str(string).map_err(Details::ConvertStrToUuid)? }; Ok(Value::Uuid(uuid)) @@ -390,6 +396,7 @@ mod decode { #[cfg(test)] #[allow(clippy::expect_fun_call)] mod tests { + use crate::schema::tokio::SchemaExt; use crate::{ decimal::Decimal, decode::tokio::decode, @@ -397,7 +404,6 @@ mod decode { schema::{DecimalSchema, FixedSchema, Name, Schema}, types::Value, }; - use crate::schema::tokio::SchemaExt; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use std::collections::HashMap; @@ -467,7 +473,9 @@ mod decode { let value = Value::Decimal(Decimal::from(bigint.to_signed_bytes_be())); let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer) + .await + .expect(&success(&value, &schema)); let mut bytes = &buffer[..]; let result = decode(&schema, &mut bytes).await?; @@ -497,7 +505,9 @@ mod decode { )); let mut buffer = Vec::::new(); - encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer) + .await + .expect(&success(&value, &schema)); let mut bytes: &[u8] = &buffer[..]; let result = decode(&schema, &mut bytes).await?; assert_eq!(result, value); @@ -541,7 +551,9 @@ mod decode { ("b".into(), inner_value2.clone()), ]); let mut buf = Vec::new(); - encode(&outer_value1, &schema, &mut buf).await.expect(&success(&outer_value1, &schema)); + encode(&outer_value1, &schema, &mut buf) + .await + .expect(&success(&outer_value1, &schema)); assert!(!buf.is_empty()); let mut bytes = &buf[..]; assert_eq!( @@ -557,7 +569,9 @@ mod decode { ("a".into(), Value::Union(0, Box::new(Value::Null))), ("b".into(), inner_value2), ]); - encode(&outer_value2, &schema, &mut buf).await.expect(&success(&outer_value2, &schema)); + encode(&outer_value2, &schema, &mut buf) + .await + .expect(&success(&outer_value2, &schema)); let mut bytes = &buf[..]; assert_eq!( outer_value2, @@ -608,7 +622,9 @@ mod decode { ("b".into(), inner_value2), ]); let mut buf = Vec::new(); - encode(&outer_value, &schema, &mut buf).await.expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); let mut bytes = &buf[..]; assert_eq!( outer_value, @@ -662,7 +678,9 @@ mod decode { ("b".into(), inner_value2), ]); let mut buf = Vec::new(); - encode(&outer_value, &schema, &mut buf).await.expect(&success(&outer_value, &schema)); + encode(&outer_value, &schema, &mut buf) + .await + .expect(&success(&outer_value, &schema)); let mut bytes = &buf[..]; assert_eq!( outer_value, @@ -753,7 +771,8 @@ mod decode { ]); let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf).await + encode(&outer_record_variation_1, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_1, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -765,7 +784,8 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_2, &schema, &mut buf).await + encode(&outer_record_variation_2, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_2, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -777,7 +797,8 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_3, &schema, &mut buf).await + encode(&outer_record_variation_3, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_3, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -870,7 +891,8 @@ mod decode { ]); let mut buf = Vec::new(); - encode(&outer_record_variation_1, &schema, &mut buf).await + encode(&outer_record_variation_1, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_1, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -882,7 +904,8 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_2, &schema, &mut buf).await + encode(&outer_record_variation_2, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_2, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -894,7 +917,8 @@ mod decode { ); let mut buf = Vec::new(); - encode(&outer_record_variation_3, &schema, &mut buf).await + encode(&outer_record_variation_3, &schema, &mut buf) + .await .expect(&success(&outer_record_variation_3, &schema)); let mut bytes = &buf[..]; assert_eq!( @@ -914,7 +938,9 @@ mod decode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer) + .await + .expect(&success(&value, &schema)); let result = decode(&Schema::Uuid, &mut &buffer[..]).await?; assert_eq!(result, value); @@ -935,7 +961,9 @@ mod decode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - encode(&value, &schema, &mut buffer).await.expect(&success(&value, &schema)); + encode(&value, &schema, &mut buffer) + .await + .expect(&success(&value, &schema)); let result = decode(&Schema::Uuid, &mut &buffer[..]).await?; assert_eq!(result, value); diff --git a/avro/src/encode.rs b/avro/src/encode.rs index bbd2bd5f..d4b13d5e 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -58,14 +58,20 @@ mod encode { use log::error; use std::{borrow::Borrow, collections::HashMap}; - pub async fn encode( value: &Value, schema: &Schema, writer: &mut W, ) -> AvroResult { let rs = ResolvedSchema::try_from(schema)?; - Box::pin(encode_internal(value, schema, rs.get_names(), &None, writer)).await + Box::pin(encode_internal( + value, + schema, + rs.get_names(), + &None, + writer, + )) + .await } pub(crate) async fn encode_bytes + ?Sized, W: AvroWrite + Unpin>( @@ -126,7 +132,8 @@ mod encode { } } Value::Boolean(b) => writer - .write(&[u8::from(*b)]).await + .write(&[u8::from(*b)]) + .await .map_err(|e| Details::WriteBytes(e).into()), // Pattern | Pattern here to signify that these _must_ have the same encoding. Value::Int(i) | Value::Date(i) | Value::TimeMillis(i) => encode_int(*i, writer).await, @@ -139,10 +146,12 @@ mod encode { | Value::LocalTimestampNanos(i) | Value::TimeMicros(i) => encode_long(*i, writer).await, Value::Float(x) => writer - .write(&x.to_le_bytes()).await + .write(&x.to_le_bytes()) + .await .map_err(|e| Details::WriteBytes(e).into()), Value::Double(x) => writer - .write(&x.to_le_bytes()).await + .write(&x.to_le_bytes()) + .await .map_err(|e| Details::WriteBytes(e).into()), Value::Decimal(decimal) => match schema { Schema::Decimal(DecimalSchema { inner, .. }) => match *inner.clone() { @@ -154,7 +163,9 @@ mod encode { } encode(&Value::Fixed(size, bytes), inner, writer).await } - Schema::Bytes => encode(&Value::Bytes(decimal.try_into()?), inner, writer).await, + Schema::Bytes => { + encode(&Value::Bytes(decimal.try_into()?), inner, writer).await + } _ => { Err(Details::ResolveDecimalSchema(SchemaKind::from(*inner.clone())).into()) } @@ -168,7 +179,8 @@ mod encode { &Value::Duration(duration) => { let slice: [u8; 12] = duration.into(); writer - .write(&slice).await + .write(&slice) + .await .map_err(|e| Details::WriteBytes(e).into()) } Value::Uuid(uuid) => match *schema { @@ -198,13 +210,15 @@ mod encode { Value::BigDecimal(bg) => { let buf: Vec = serialize_big_decimal(bg).await?; writer - .write(buf.as_slice()).await + .write(buf.as_slice()) + .await .map_err(|e| Details::WriteBytes(e).into()) } Value::Bytes(bytes) => match *schema { Schema::Bytes => encode_bytes(bytes, writer).await, Schema::Fixed { .. } => writer - .write(bytes.as_slice()).await + .write(bytes.as_slice()) + .await .map_err(|e| Details::WriteBytes(e).into()), _ => Err(Details::EncodeValueAsSchemaError { value_kind: ValueKind::Bytes, @@ -229,7 +243,8 @@ mod encode { .into()), }, Value::Fixed(_, bytes) => writer - .write(bytes.as_slice()).await + .write(bytes.as_slice()) + .await .map_err(|e| Details::WriteBytes(e).into()), Value::Enum(i, _) => encode_int(*i as i32, writer).await, Value::Union(idx, item) => { @@ -366,7 +381,8 @@ mod encode { names, enclosing_namespace, &mut union_buffer, - )).await; + )) + .await; match encode_res { Ok(_) => { return writer @@ -1044,7 +1060,8 @@ mod encode { let value = Value::Uuid(Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?); let mut buffer = Vec::new(); - match encode(&value, &schema, &mut buffer).await + match encode(&value, &schema, &mut buffer) + .await .map_err(Error::into_details) { Err(Details::ConvertFixedToUuid(actual)) => { diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 4800c169..4cd532e1 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -135,8 +135,8 @@ mod headers { mod tests { use super::*; use crate::error::{Details, Error}; - use apache_avro_test_helper::TestResult; use crate::schema::tokio::SchemaExt; + use apache_avro_test_helper::TestResult; #[tokio::test] async fn test_rabin_fingerprint_header() -> TestResult { diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 3a2a7e69..45781430 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -930,16 +930,16 @@ pub use reader::tokio::{ from_avro_datum_reader_schemata as async_from_avro_datum_reader_schemata, from_avro_datum_schemata as async_from_avro_datum_schemata, read_marker as async_read_marker, }; -pub use schema::Schema; pub use schema::AvroSchema; -#[cfg(feature = "sync")] -pub use ser::sync::to_value; -#[cfg(feature = "tokio")] -pub use ser::tokio::to_value as async_to_value; +pub use schema::Schema; #[cfg(feature = "sync")] pub use schema::sync::SchemaExt; #[cfg(feature = "tokio")] pub use schema::tokio::SchemaExt as AsyncSchemaExt; +#[cfg(feature = "sync")] +pub use ser::sync::to_value; +#[cfg(feature = "tokio")] +pub use ser::tokio::to_value as async_to_value; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; #[cfg(feature = "sync")] diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 93ad73c0..4f3e2702 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -1093,9 +1093,11 @@ mod reader { .expect("Encode should succeed"); #[synca::cfg(sync)] - let mut to_read = std::io::Read::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); + let mut to_read = + std::io::Read::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); #[synca::cfg(tokio)] - let mut to_read = tokio::io::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); + let mut to_read = tokio::io::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]) + .chain(&to_read_3[..]); let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) .expect("Schema should resolve"); diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 39a835ad..9229043a 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -647,7 +647,6 @@ const RESERVED_FIELDS: &[&str] = &[ "scale", ]; - impl std::fmt::Display for SchemaFingerprint { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( @@ -1347,7 +1346,6 @@ pub trait AvroSchema { fn get_schema() -> Schema; } - #[cfg(feature = "derive")] pub mod derive { use super::*; @@ -1402,10 +1400,8 @@ pub mod derive { ///} /// ``` pub trait AvroSchemaComponent { - fn get_schema_in_ctxt( - named_schemas: &mut Names, - enclosing_namespace: &Namespace, - ) -> Schema; + fn get_schema_in_ctxt(named_schemas: &mut Names, enclosing_namespace: &Namespace) + -> Schema; } impl AvroSchema for T @@ -1561,16 +1557,15 @@ mod schema { use crate::AvroResult; use crate::error::{Details, Error}; use crate::schema::{ - Alias, Aliases, DecimalMetadata, DecimalSchema, EnumSchema, - FixedSchema, Name, Names, Namespace, Precision, RecordField, RecordFieldOrder, - RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, + Alias, Aliases, DecimalMetadata, DecimalSchema, EnumSchema, FixedSchema, Name, Names, + Namespace, Precision, RecordField, RecordFieldOrder, RecordSchema, ResolvedSchema, Scale, + Schema, SchemaKind, UnionSchema, }; + use crate::types::tokio::ValueExt; use crate::util::MapHelper; use crate::{ types::{Value, ValueKind}, - validator::{ - validate_enum_symbol_name, validate_record_field_name, - }, + validator::{validate_enum_symbol_name, validate_record_field_name}, }; use log::{debug, error, warn}; #[synca::cfg(sync)] @@ -1584,7 +1579,6 @@ mod schema { use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] use tokio::io::AsyncReadExt; - use crate::types::tokio::ValueExt; pub struct RecordFieldExt; @@ -1662,9 +1656,15 @@ mod schema { let mut resolved = false; for schema in schemas { - if ValueExt::resolve_internal(&avro_value, schema, names, &schema.namespace(), &None) - .await - .is_ok() + if ValueExt::resolve_internal( + &avro_value, + schema, + names, + &schema.namespace(), + &None, + ) + .await + .is_ok() { resolved = true; break; @@ -1684,9 +1684,15 @@ mod schema { } } _ => { - let resolved = ValueExt::resolve_internal(&avro_value, field_schema, names, &field_schema.namespace(), &None) - .await - .is_ok(); + let resolved = ValueExt::resolve_internal( + &avro_value, + field_schema, + names, + &field_schema.namespace(), + &None, + ) + .await + .is_ok(); if !resolved { return Err(Details::GetDefaultRecordField( @@ -1925,9 +1931,7 @@ mod schema { } /// Create a `Schema` from a reader which implements [`Read`]. - pub async fn parse_reader( - reader: &mut R, - ) -> AvroResult { + pub async fn parse_reader(reader: &mut R) -> AvroResult { let mut buf = String::new(); match reader.read_to_string(&mut buf).await { Ok(_) => Self::parse_str(&buf).await, @@ -2569,8 +2573,9 @@ mod schema { if let Some(ref json_value) = default { let value = Value::from(json_value.clone()); - let resolved = ValueExt::resolve_enum(&value, &symbols, &Some(json_value.to_string()), &None) - .is_ok(); + let resolved = + ValueExt::resolve_enum(&value, &symbols, &Some(json_value.to_string()), &None) + .is_ok(); if !resolved { return Err(Details::GetEnumDefault { symbol: json_value.to_string(), @@ -2751,6 +2756,8 @@ mod schema { #[cfg(test)] mod tests { use super::*; + #[synca::cfg(sync)] + use crate::writer::sync::SpecificSingleObjectWriter; use crate::{ de::tokio::from_value, error::Details, @@ -2759,14 +2766,12 @@ mod schema { ser::tokio::to_value, writer::tokio::{Writer, to_avro_datum}, }; - #[synca::cfg(sync)] - use crate::writer::sync::SpecificSingleObjectWriter; use apache_avro_test_helper::{ TestResult, logger::{assert_logged, assert_not_logged}, }; - use serde_json::json; use serde::{Deserialize, Serialize}; + use serde_json::json; #[synca::cfg(sync)] use serial_test::serial; #[synca::cfg(sync)] @@ -2874,7 +2879,9 @@ mod schema { .position(1) .build(); - assert!(!RecordFieldExt::is_nullable(&non_nullable_record_field.schema)); + assert!(!RecordFieldExt::is_nullable( + &non_nullable_record_field.schema + )); Ok(()) } @@ -5398,15 +5405,7 @@ mod schema { "#; #[derive( - Debug, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, - Clone, - Deserialize, - Serialize, + Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Deserialize, Serialize, )] pub enum Bar { #[serde(rename = "bar0")] @@ -5444,15 +5443,7 @@ mod schema { #[tokio::test] async fn avro_3755_deserialize() -> TestResult { #[derive( - Debug, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, - Clone, - Deserialize, - Serialize, + Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Deserialize, Serialize, )] pub enum Bar { #[serde(rename = "bar0")] diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 81f7da0b..44f1249f 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -470,8 +470,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { let ref_schema = self.names.get(full_name.as_ref()).clone(); - ref_schema - .ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) + ref_schema.ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) } fn write_bytes(&mut self, bytes: &[u8]) -> Result { @@ -483,11 +482,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { Ok(bytes_written) } - fn serialize_bool_with_schema( - &mut self, - value: bool, - schema: &Schema, - ) -> Result { + fn serialize_bool_with_schema(&mut self, value: bool, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "bool", @@ -520,11 +515,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_i32_with_schema( - &mut self, - value: i32, - schema: &Schema, - ) -> Result { + fn serialize_i32_with_schema(&mut self, value: i32, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "int (i8 | i16 | i32)", @@ -534,9 +525,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { }; match schema { - Schema::Int | Schema::TimeMillis | Schema::Date => { - encode_int(value, &mut self.writer) - } + Schema::Int | Schema::TimeMillis | Schema::Date => encode_int(value, &mut self.writer), Schema::Long | Schema::TimeMicros | Schema::TimestampMillis @@ -573,11 +562,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_i64_with_schema( - &mut self, - value: i64, - schema: &Schema, - ) -> Result { + fn serialize_i64_with_schema(&mut self, value: i64, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "i64", @@ -680,11 +665,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_u32_with_schema( - &mut self, - value: u32, - schema: &Schema, - ) -> Result { + fn serialize_u32_with_schema(&mut self, value: u32, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "unsigned int (u16 | u32)", @@ -735,11 +716,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_u64_with_schema( - &mut self, - value: u64, - schema: &Schema, - ) -> Result { + fn serialize_u64_with_schema(&mut self, value: u64, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "u64", @@ -795,11 +772,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_f32_with_schema( - &mut self, - value: f32, - schema: &Schema, - ) -> Result { + fn serialize_f32_with_schema(&mut self, value: f32, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "f32", @@ -836,11 +809,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_f64_with_schema( - &mut self, - value: f64, - schema: &Schema, - ) -> Result { + fn serialize_f64_with_schema(&mut self, value: f64, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "f64", @@ -877,11 +846,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_char_with_schema( - &mut self, - value: char, - schema: &Schema, - ) -> Result { + fn serialize_char_with_schema(&mut self, value: char, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "char", @@ -910,11 +875,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_str_with_schema( - &mut self, - value: &str, - schema: &Schema, - ) -> Result { + fn serialize_str_with_schema(&mut self, value: &str, schema: &Schema) -> Result { let create_error = |cause: String| { Error::new(Details::SerializeValueWithSchema { value_type: "string", @@ -1023,8 +984,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } Schema::Decimal(decimal_schema) => match decimal_schema.inner.as_ref() { Schema::Bytes => self.write_bytes(value), - Schema::Fixed(fixed_schema) => match fixed_schema.size.checked_sub(value.len()) - { + Schema::Fixed(fixed_schema) => match fixed_schema.size.checked_sub(value.len()) { Some(pad) => { let pad_val = match value.len() { 0 => 0, @@ -1108,11 +1068,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } } - fn serialize_some_with_schema( - &mut self, - value: &T, - schema: &Schema, - ) -> Result + fn serialize_some_with_schema(&mut self, value: &T, schema: &Schema) -> Result where T: ?Sized + ser::Serialize, { @@ -1180,13 +1136,11 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { match variant_schema { Schema::Record(record_schema) if record_schema.fields.is_empty() => { encode_int(i as i32, &mut *self.writer)?; - return self - .serialize_unit_struct_with_schema(name, variant_schema); + return self.serialize_unit_struct_with_schema(name, variant_schema); } Schema::Null | Schema::Ref { name: _ } => { encode_int(i as i32, &mut *self.writer)?; - return self - .serialize_unit_struct_with_schema(name, variant_schema); + return self.serialize_unit_struct_with_schema(name, variant_schema); } _ => { /* skip */ } } @@ -1245,12 +1199,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { } Schema::Ref { name: ref_name } => { let ref_schema = self.get_ref_schema(ref_name)?; - self.serialize_unit_variant_with_schema( - name, - variant_index, - variant, - ref_schema, - ) + self.serialize_unit_variant_with_schema(name, variant_index, variant, ref_schema) } unsupported => Err(create_error(format!( "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref" @@ -1478,13 +1427,13 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { match schema { Schema::Union(union_schema) => { let variant_schema = union_schema - .schemas - .get(variant_index as usize) - .ok_or_else(|| { - create_error(format!( - "Cannot find a variant at position {variant_index} in {union_schema:?}" - )) - })?; + .schemas + .get(variant_index as usize) + .ok_or_else(|| { + create_error(format!( + "Cannot find a variant at position {variant_index} in {union_schema:?}" + )) + })?; encode_int(variant_index as i32, &mut self.writer)?; self.serialize_tuple_struct_with_schema(variant, len, variant_schema) @@ -1567,19 +1516,11 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { if inner.fields.len() == len && inner.name.name == name => { encode_int(i as i32, &mut *self.writer)?; - return self.serialize_struct_with_schema( - name, - len, - variant_schema, - ); + return self.serialize_struct_with_schema(name, len, variant_schema); } Schema::Ref { name: _ } => { encode_int(i as i32, &mut *self.writer)?; - return self.serialize_struct_with_schema( - name, - len, - variant_schema, - ); + return self.serialize_struct_with_schema(name, len, variant_schema); } _ => { /* skip */ } } @@ -1613,13 +1554,13 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { match schema { Schema::Union(union_schema) => { let variant_schema = union_schema - .schemas - .get(variant_index as usize) - .ok_or_else(|| { - create_error(format!( - "Cannot find variant at position {variant_index} in {union_schema:?}" - )) - })?; + .schemas + .get(variant_index as usize) + .ok_or_else(|| { + create_error(format!( + "Cannot find variant at position {variant_index} in {union_schema:?}" + )) + })?; encode_int(variant_index as i32, &mut self.writer)?; self.serialize_struct_with_schema(variant, len, variant_schema) @@ -1826,8 +1767,7 @@ mod tests { use super::*; use crate::schema::sync::SchemaExt; use crate::{ - Days, Duration, Millis, Months, decimal::Decimal, error::Details, - schema::ResolvedSchema, + Days, Duration, Millis, Months, decimal::Decimal, error::Details, schema::ResolvedSchema, }; use apache_avro_test_helper::TestResult; use bigdecimal::BigDecimal; @@ -1847,8 +1787,7 @@ mod tests { let schema = Schema::Null; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); ().serialize(&mut serializer)?; None::<()>.serialize(&mut serializer)?; @@ -1867,8 +1806,7 @@ mod tests { let schema = Schema::Boolean; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); true.serialize(&mut serializer)?; false.serialize(&mut serializer)?; @@ -1885,8 +1823,7 @@ mod tests { let schema = Schema::Int; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 4u8.serialize(&mut serializer)?; 31u16.serialize(&mut serializer)?; @@ -1907,8 +1844,7 @@ mod tests { let schema = Schema::Long; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 4u8.serialize(&mut serializer)?; 31u16.serialize(&mut serializer)?; @@ -1934,8 +1870,7 @@ mod tests { let schema = Schema::Float; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 4.7f32.serialize(&mut serializer)?; (-14.1f64).serialize(&mut serializer)?; @@ -1952,8 +1887,7 @@ mod tests { let schema = Schema::Float; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 4.7f32.serialize(&mut serializer)?; (-14.1f64).serialize(&mut serializer)?; @@ -1970,8 +1904,7 @@ mod tests { let schema = Schema::Bytes; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 'a'.serialize(&mut serializer)?; "test".serialize(&mut serializer)?; @@ -1992,8 +1925,7 @@ mod tests { let schema = Schema::String; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 'a'.serialize(&mut serializer)?; "test".serialize(&mut serializer)?; @@ -2020,8 +1952,7 @@ mod tests { {"name": "intField", "type": "int"} ] }"#, - ) - ?; + )?; #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -2039,8 +1970,7 @@ mod tests { let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); let good_record = GoodTestRecord { string_field: String::from("test"), @@ -2070,13 +2000,11 @@ mod tests { "name": "EmptyRecord", "fields": [] }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); #[derive(Serialize)] struct EmptyRecord; @@ -2123,8 +2051,7 @@ mod tests { "name": "Suit", "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] }"#, - ) - ?; + )?; #[derive(Serialize)] enum Suit { @@ -2136,8 +2063,7 @@ mod tests { let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); Suit::Spades.serialize(&mut serializer)?; Suit::Hearts.serialize(&mut serializer)?; @@ -2171,13 +2097,11 @@ mod tests { "type": "array", "items": "long" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); let arr: Vec = vec![10, 5, 400]; arr.serialize(&mut serializer)?; @@ -2210,13 +2134,11 @@ mod tests { "type": "map", "values": "long" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); let mut map: BTreeMap = BTreeMap::new(); map.insert(String::from("item1"), 10); @@ -2242,8 +2164,8 @@ mod tests { assert_eq!( buffer.as_slice(), &[ - 4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, - 6, 0 + 4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, 6, + 0 ] ); @@ -2256,8 +2178,7 @@ mod tests { r#"{ "type": ["null", "long"] }"#, - ) - ?; + )?; #[derive(Serialize)] enum NullableLong { @@ -2267,8 +2188,7 @@ mod tests { let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); Some(10i64).serialize(&mut serializer)?; None::.serialize(&mut serializer)?; @@ -2305,8 +2225,7 @@ mod tests { r#"{ "type": ["null", "long", "string"] }"#, - ) - ?; + )?; #[derive(Serialize)] enum LongOrString { @@ -2317,8 +2236,7 @@ mod tests { let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); LongOrString::Null.serialize(&mut serializer)?; LongOrString::Long(400).serialize(&mut serializer)?; @@ -2359,13 +2277,11 @@ mod tests { "size": 8, "name": "LongVal" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?; @@ -2437,13 +2353,11 @@ mod tests { "precision": 16, "scale": 2 }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); let val = Decimal::from(&[251, 155]); val.serialize(&mut serializer)?; @@ -2477,13 +2391,11 @@ mod tests { "precision": 16, "scale": 2 }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]); val.serialize(&mut serializer)?; @@ -2514,14 +2426,12 @@ mod tests { "type": "bytes", "logicalType": "big-decimal" }"#, - ) - ?; + )?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2); val.serialize(&mut serializer)?; @@ -2539,14 +2449,12 @@ mod tests { "type": "string", "logicalType": "uuid" }"#, - ) - ?; + )?; crate::util::SERDE_HUMAN_READABLE.store(true, Ordering::Release); let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); "8c28da81-238c-4326-bddd-4e3d00cc5099" .parse::()? @@ -2568,9 +2476,9 @@ mod tests { assert_eq!( buffer.as_slice(), &[ - 72, b'8', b'c', b'2', b'8', b'd', b'a', b'8', b'1', b'-', b'2', b'3', b'8', - b'c', b'-', b'4', b'3', b'2', b'6', b'-', b'b', b'd', b'd', b'd', b'-', b'4', - b'e', b'3', b'd', b'0', b'0', b'c', b'c', b'5', b'0', b'9', b'9' + 72, b'8', b'c', b'2', b'8', b'd', b'a', b'8', b'1', b'-', b'2', b'3', b'8', b'c', + b'-', b'4', b'3', b'2', b'6', b'-', b'b', b'd', b'd', b'd', b'-', b'4', b'e', b'3', + b'd', b'0', b'0', b'c', b'c', b'5', b'0', b'9', b'9' ] ); @@ -2584,13 +2492,11 @@ mod tests { "type": "int", "logicalType": "date" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 100_u8.serialize(&mut serializer)?; 1000_u16.serialize(&mut serializer)?; @@ -2629,13 +2535,11 @@ mod tests { "type": "int", "logicalType": "time-millis" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 100_u8.serialize(&mut serializer)?; 1000_u16.serialize(&mut serializer)?; @@ -2674,13 +2578,11 @@ mod tests { "type": "long", "logicalType": "time-micros" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); 100_u8.serialize(&mut serializer)?; 1000_u16.serialize(&mut serializer)?; @@ -2723,8 +2625,7 @@ mod tests { "type": "long", "logicalType": "timestamp-{precision}" }}"# - )) - ?; + ))?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); @@ -2783,17 +2684,14 @@ mod tests { "name": "duration", "logicalType": "duration" }"#, - ) - ?; + )?; let mut buffer: Vec = Vec::new(); let names = HashMap::new(); - let mut serializer = - SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); + let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None); - let duration_bytes = ByteArray::new( - Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into(), - ); + let duration_bytes = + ByteArray::new(Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into()); duration_bytes.serialize(&mut serializer)?; match [1; 12] @@ -2872,12 +2770,12 @@ mod tests { assert_eq!( buffer.as_slice(), &[ - 8, 116, 101, 115, 116, 20, 10, 6, 0, 195, 104, 4, 72, 56, 99, 50, 56, 100, 97, - 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, - 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 56, 2, 20, 105, 110, 110, 101, 114, - 95, 116, 101, 115, 116, 200, 1, 8, 4, 78, 70, 4, 72, 56, 99, 50, 56, 100, 97, - 56, 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, - 101, 51, 100, 48, 48, 99, 99, 53, 48, 57, 57, 0 + 8, 116, 101, 115, 116, 20, 10, 6, 0, 195, 104, 4, 72, 56, 99, 50, 56, 100, 97, 56, + 49, 45, 50, 51, 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, 101, 51, + 100, 48, 48, 99, 99, 53, 48, 57, 56, 2, 20, 105, 110, 110, 101, 114, 95, 116, 101, + 115, 116, 200, 1, 8, 4, 78, 70, 4, 72, 56, 99, 50, 56, 100, 97, 56, 49, 45, 50, 51, + 56, 99, 45, 52, 51, 50, 54, 45, 98, 100, 100, 100, 45, 52, 101, 51, 100, 48, 48, + 99, 99, 53, 48, 57, 57, 0 ] ); diff --git a/avro/src/types.rs b/avro/src/types.rs index 52417278..06c86902 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -17,10 +17,10 @@ //! Logic handling the intermediate representation of Avro values. -use crate::{AvroResult, Decimal, Duration, Uuid}; use crate::bigdecimal::BigDecimal; -use crate::schema::{RecordSchema, Schema}; use crate::error::{Details, Error}; +use crate::schema::{RecordSchema, Schema}; +use crate::{AvroResult, Decimal, Duration, Uuid}; use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, @@ -279,7 +279,6 @@ to_value!(Decimal, Value::Decimal); to_value!(BigDecimal, Value::BigDecimal); to_value!(Duration, Value::Duration); - /// Convert Avro values to Json values impl TryFrom for serde_json::Value { type Error = Error; @@ -295,13 +294,13 @@ impl TryFrom for serde_json::Value { Value::Double(d) => serde_json::Number::from_f64(d) .map(Self::Number) .ok_or_else(|| Details::ConvertF64ToJson(d).into()), - Value::Bytes(bytes) => { - Ok(serde_json::Value::Array(bytes.into_iter().map(|b| b.into()).collect())) - } + Value::Bytes(bytes) => Ok(serde_json::Value::Array( + bytes.into_iter().map(|b| b.into()).collect(), + )), Value::String(s) => Ok(serde_json::Value::String(s)), - Value::Fixed(_size, items) => { - Ok(serde_json::Value::Array(items.into_iter().map(|v| v.into()).collect())) - } + Value::Fixed(_size, items) => Ok(serde_json::Value::Array( + items.into_iter().map(|v| v.into()).collect(), + )), Value::Enum(_i, s) => Ok(serde_json::Value::String(s)), Value::Union(_i, b) => Self::try_from(*b), Value::Array(items) => items @@ -323,8 +322,10 @@ impl TryFrom for serde_json::Value { Value::Decimal(ref d) => >::try_from(d) .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), Value::BigDecimal(ref bg) => { - let vec1: Vec = vec![];//crate::bigdecimal::sync::serialize_big_decimal(bg)?; - Ok(serde_json::Value::Array(vec1.into_iter().map(|b| b.into()).collect())) + let vec1: Vec = vec![]; //crate::bigdecimal::sync::serialize_big_decimal(bg)?; + Ok(serde_json::Value::Array( + vec1.into_iter().map(|b| b.into()).collect(), + )) } Value::TimeMillis(t) => Ok(serde_json::Value::Number(t.into())), Value::TimeMicros(t) => Ok(serde_json::Value::Number(t.into())), @@ -342,7 +343,6 @@ impl TryFrom for serde_json::Value { } } - #[synca::synca( #[cfg(feature = "tokio")] pub mod tokio { }, @@ -373,11 +373,11 @@ mod types { duration::Duration, error::Details, error::Error, + schema::tokio::{RecordFieldExt, UnionSchemaExt}, schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, }, - schema::tokio::{UnionSchemaExt, RecordFieldExt}, types::Value, }; use log::{debug, error}; @@ -390,7 +390,6 @@ mod types { Ok((2.0_f64.powi(8 * len - 1) - 1.0).log10().floor() as usize) } - pub struct ValueExt; impl ValueExt { @@ -548,25 +547,36 @@ mod types { // (&Value::Union(None), &Schema::Union(_)) => None, (&Value::Union(i, ref value), Schema::Union(inner)) => { if let Some(schema) = inner.variants().get(i as usize) { - Box::pin(ValueExt::validate_internal(&value, schema, names, enclosing_namespace)).await + Box::pin(ValueExt::validate_internal( + &value, + schema, + names, + enclosing_namespace, + )) + .await } else { Some(format!("No schema in the union at position '{i}'")) } } - (v, Schema::Union(inner)) => { - match UnionSchemaExt::find_schema_with_known_schemata(inner, v, names, enclosing_namespace) - .await - { - Some(_) => None, - None => Some("Could not find matching type in union".to_string()), - } - } + (v, Schema::Union(inner)) => match UnionSchemaExt::find_schema_with_known_schemata( + inner, + v, + names, + enclosing_namespace, + ) + .await + { + Some(_) => None, + None => Some("Could not find matching type in union".to_string()), + }, (Value::Array(items), Schema::Array(inner)) => { let mut acc = None; for item in items.iter() { acc = ValueExt::accumulate( acc, - Box::pin(ValueExt::validate_internal(&item, &inner.items, + Box::pin(ValueExt::validate_internal( + &item, + &inner.items, names, enclosing_namespace, )) @@ -586,8 +596,9 @@ mod types { for (_, value) in items.iter() { acc = ValueExt::accumulate( acc, - Box::pin(ValueExt::validate_internal(&value, - &inner.types, + Box::pin(ValueExt::validate_internal( + &value, + &inner.types, names, enclosing_namespace, )) @@ -611,8 +622,10 @@ mod types { .. }), ) => { - let non_nullable_fields_count = - fields.iter().filter(|&rf| !RecordFieldExt::is_nullable(&rf.schema)).count(); + let non_nullable_fields_count = fields + .iter() + .filter(|&rf| !RecordFieldExt::is_nullable(&rf.schema)) + .count(); // If the record contains fewer fields as required fields by the schema, it is invalid. if record_fields.len() < non_nullable_fields_count { @@ -641,8 +654,9 @@ mod types { let field = &fields[*idx]; ValueExt::accumulate( acc, - Box::pin(ValueExt::validate_internal(&record_field, - &field.schema, + Box::pin(ValueExt::validate_internal( + &record_field, + &field.schema, names, record_namespace, )) @@ -714,7 +728,8 @@ mod types { } else { ResolvedSchema::try_from(schemata)? }; - ValueExt::resolve_internal(&value, schema, rs.get_names(), &enclosing_namespace, &None).await + ValueExt::resolve_internal(&value, schema, rs.get_names(), &enclosing_namespace, &None) + .await } pub(crate) async fn resolve_internal( @@ -765,8 +780,14 @@ mod types { Schema::String => ValueExt::resolve_string(value), Schema::Fixed(FixedSchema { size, .. }) => ValueExt::resolve_fixed(value, size), Schema::Union(ref inner) => { - Box::pin(ValueExt::resolve_union(value, inner, names, enclosing_namespace, field_default)) - .await + Box::pin(ValueExt::resolve_union( + value, + inner, + names, + enclosing_namespace, + field_default, + )) + .await } Schema::Enum(EnumSchema { ref symbols, @@ -774,13 +795,31 @@ mod types { .. }) => ValueExt::resolve_enum(value, symbols, default, field_default), Schema::Array(ref inner) => { - Box::pin(ValueExt::resolve_array(value, &inner.items, names, enclosing_namespace)).await + Box::pin(ValueExt::resolve_array( + value, + &inner.items, + names, + enclosing_namespace, + )) + .await } Schema::Map(ref inner) => { - Box::pin(ValueExt::resolve_map(value, &inner.types, names, enclosing_namespace)).await + Box::pin(ValueExt::resolve_map( + value, + &inner.types, + names, + enclosing_namespace, + )) + .await } Schema::Record(RecordSchema { ref fields, .. }) => { - Box::pin(ValueExt::resolve_record(value, fields, names, enclosing_namespace)).await + Box::pin(ValueExt::resolve_record( + value, + fields, + names, + enclosing_namespace, + )) + .await } Schema::Decimal(DecimalSchema { scale, @@ -1127,18 +1166,29 @@ mod types { // Reader is a union, but writer is not. v => v, }; - let (i, inner) = UnionSchemaExt::find_schema_with_known_schemata(schema, &v, names, enclosing_namespace) - .await - .ok_or_else(|| Details::FindUnionVariant { - schema: schema.clone(), - value: v.clone(), - })?; + let (i, inner) = UnionSchemaExt::find_schema_with_known_schemata( + schema, + &v, + names, + enclosing_namespace, + ) + .await + .ok_or_else(|| Details::FindUnionVariant { + schema: schema.clone(), + value: v.clone(), + })?; Ok(Value::Union( i as u32, Box::new( - ValueExt::resolve_internal(&v, inner, names, enclosing_namespace, field_default) - .await?, + ValueExt::resolve_internal( + &v, + inner, + names, + enclosing_namespace, + field_default, + ) + .await?, ), )) } @@ -1509,8 +1559,8 @@ mod types { ]; for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { - let err_message = ValueExt::validate_internal(&value, &schema, &HashMap::default(), &None) - .await; + let err_message = + ValueExt::validate_internal(&value, &schema, &HashMap::default(), &None).await; assert_eq!(valid, err_message.is_none()); if !valid { let full_err_message = format!( @@ -1701,10 +1751,7 @@ mod types { ("a".to_string(), Value::Long(42i64)), ("b".to_string(), Value::String("foo".to_string())), ]); - assert!( - ValueExt::validate(&value, &schema) - .await - ); + assert!(ValueExt::validate(&value, &schema).await); let value = Value::Record(vec![ ("b".to_string(), Value::String("foo".to_string())), @@ -1758,23 +1805,17 @@ mod types { ("a".to_string(), Value::Long(42i64)), ("b".to_string(), Value::String("foo".to_string())), ] - .into_iter() - .collect() - ); - assert!( - ValueExt::validate(&value, &schema) - .await + .into_iter() + .collect(), ); + assert!(ValueExt::validate(&value, &schema).await); let value = Value::Map( - vec![("d".to_string(), Value::Long(123_i64)),] + vec![("d".to_string(), Value::Long(123_i64))] .into_iter() - .collect() - ); - assert!( - !ValueExt::validate(&value, &schema) - .await + .collect(), ); + assert!(!ValueExt::validate(&value, &schema).await); assert_logged( r#"Invalid value: Map({"d": Long(123)}) for schema: Record(RecordSchema { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases: None, default: None, schema: Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc: None, aliases: None, default: Some(Null), schema: Union(UnionSchema { schemas: [Null, Int], variant_index: {Null: 0, Int: 1} }), order: Ascending, position: 2, custom_attributes: {} }], lookup: {"a": 0, "b": 1, "c": 2}, attributes: {} }). Reason: Field with name '"a"' is not a member of the map items Field with name '"b"' is not a member of the map items"#, @@ -1787,12 +1828,9 @@ Field with name '"b"' is not a member of the map items"#, Box::new(Value::Record(vec![ ("a".to_string(), Value::Long(42i64)), ("b".to_string(), Value::String("foo".to_string())), - ])) - ); - assert!( - ValueExt::validate(&value, &union_schema) - .await + ])), ); + assert!(ValueExt::validate(&value, &union_schema).await); let value = Value::Union( 1, @@ -1801,14 +1839,11 @@ Field with name '"b"' is not a member of the map items"#, ("a".to_string(), Value::Long(42i64)), ("b".to_string(), Value::String("foo".to_string())), ] - .into_iter() - .collect() - )) - ); - assert!( - ValueExt::validate(&value, &union_schema) - .await + .into_iter() + .collect(), + )), ); + assert!(ValueExt::validate(&value, &union_schema).await); Ok(()) } @@ -1855,12 +1890,15 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn resolve_decimal_bytes() -> TestResult { let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); - ValueExt::resolve(value.clone(), &Schema::Decimal(DecimalSchema { + ValueExt::resolve( + value.clone(), + &Schema::Decimal(DecimalSchema { precision: 10, scale: 4, inner: Box::new(Schema::Bytes), - })) - .await?; + }), + ) + .await?; assert!(ValueExt::resolve(value, &Schema::String).await.is_err()); Ok(()) @@ -1870,13 +1908,16 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_decimal_invalid_scale() { let value = Value::Decimal(Decimal::from(vec![1, 2])); assert!( - ValueExt::resolve(value, &Schema::Decimal(DecimalSchema { + ValueExt::resolve( + value, + &Schema::Decimal(DecimalSchema { precision: 2, scale: 3, inner: Box::new(Schema::Bytes), - })) - .await - .is_err() + }) + ) + .await + .is_err() ); } @@ -1884,13 +1925,16 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_decimal_invalid_precision_for_length() { let value = Value::Decimal(Decimal::from((1u8..=8u8).rev().collect::>())); assert!( - ValueExt::resolve(value, &Schema::Decimal(DecimalSchema { + ValueExt::resolve( + value, + &Schema::Decimal(DecimalSchema { precision: 1, scale: 0, inner: Box::new(Schema::Bytes), - })) - .await - .is_ok() + }) + ) + .await + .is_ok() ); } @@ -1898,7 +1942,9 @@ Field with name '"b"' is not a member of the map items"#, async fn resolve_decimal_fixed() { let value = Value::Decimal(Decimal::from(vec![1, 2, 3, 4, 5])); assert!( - ValueExt::resolve(value.clone(), &Schema::Decimal(DecimalSchema { + ValueExt::resolve( + value.clone(), + &Schema::Decimal(DecimalSchema { precision: 10, scale: 1, inner: Box::new(Schema::Fixed(FixedSchema { @@ -1909,9 +1955,10 @@ Field with name '"b"' is not a member of the map items"#, default: None, attributes: Default::default(), })) - })) - .await - .is_ok() + }) + ) + .await + .is_ok() ); assert!(ValueExt::resolve(value, &Schema::String).await.is_err()); } @@ -1919,21 +1966,33 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn resolve_date() { let value = Value::Date(2345); - assert!(ValueExt::resolve(value.clone(), &Schema::Date).await.is_ok()); + assert!( + ValueExt::resolve(value.clone(), &Schema::Date) + .await + .is_ok() + ); assert!(ValueExt::resolve(value, &Schema::String).await.is_err()); } #[tokio::test] async fn resolve_time_millis() { let value = Value::TimeMillis(10); - assert!(ValueExt::resolve(value.clone(), &Schema::TimeMillis).await.is_ok()); + assert!( + ValueExt::resolve(value.clone(), &Schema::TimeMillis) + .await + .is_ok() + ); assert!(ValueExt::resolve(value, &Schema::TimeMicros).await.is_err()); } #[tokio::test] async fn resolve_time_micros() { let value = Value::TimeMicros(10); - assert!(ValueExt::resolve(value.clone(), &Schema::TimeMicros).await.is_ok()); + assert!( + ValueExt::resolve(value.clone(), &Schema::TimeMicros) + .await + .is_ok() + ); assert!(ValueExt::resolve(value, &Schema::TimeMillis).await.is_err()); } @@ -1948,75 +2007,101 @@ Field with name '"b"' is not a member of the map items"#, assert!(ValueExt::resolve(value, &Schema::Float).await.is_err()); let value = Value::Float(10.0f32); - assert!(ValueExt::resolve(value, &Schema::TimestampMillis).await.is_err()); + assert!( + ValueExt::resolve(value, &Schema::TimestampMillis) + .await + .is_err() + ); } #[tokio::test] async fn resolve_timestamp_micros() { let value = Value::TimestampMicros(10); assert!( - ValueExt::resolve(value - .clone(), &Schema::TimestampMicros) + ValueExt::resolve(value.clone(), &Schema::TimestampMicros) .await .is_ok() ); assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(ValueExt::resolve(value, &Schema::TimestampMicros).await.is_err()); + assert!( + ValueExt::resolve(value, &Schema::TimestampMicros) + .await + .is_err() + ); } #[tokio::test] async fn test_avro_3914_resolve_timestamp_nanos() { let value = Value::TimestampNanos(10); - assert!(ValueExt::resolve(value.clone(), &Schema::TimestampNanos).await.is_ok()); + assert!( + ValueExt::resolve(value.clone(), &Schema::TimestampNanos) + .await + .is_ok() + ); assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(ValueExt::resolve(value, &Schema::TimestampNanos).await.is_err()); + assert!( + ValueExt::resolve(value, &Schema::TimestampNanos) + .await + .is_err() + ); } #[tokio::test] async fn test_avro_3853_resolve_timestamp_millis() { let value = Value::LocalTimestampMillis(10); assert!( - ValueExt::resolve(value - .clone(), &Schema::LocalTimestampMillis) + ValueExt::resolve(value.clone(), &Schema::LocalTimestampMillis) .await .is_ok() ); assert!(ValueExt::resolve(value, &Schema::Float).await.is_err()); let value = Value::Float(10.0f32); - assert!(ValueExt::resolve(value, &Schema::LocalTimestampMillis).await.is_err()); + assert!( + ValueExt::resolve(value, &Schema::LocalTimestampMillis) + .await + .is_err() + ); } #[tokio::test] async fn test_avro_3853_resolve_timestamp_micros() { let value = Value::LocalTimestampMicros(10); assert!( - ValueExt::resolve(value.clone(), &Schema::LocalTimestampMicros) + ValueExt::resolve(value.clone(), &Schema::LocalTimestampMicros) .await .is_ok() ); assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(ValueExt::resolve(value, &Schema::LocalTimestampMicros).await.is_err()); + assert!( + ValueExt::resolve(value, &Schema::LocalTimestampMicros) + .await + .is_err() + ); } #[tokio::test] async fn test_avro_3916_resolve_timestamp_nanos() { let value = Value::LocalTimestampNanos(10); assert!( - ValueExt::resolve(value.clone(), &Schema::LocalTimestampNanos) + ValueExt::resolve(value.clone(), &Schema::LocalTimestampNanos) .await .is_ok() ); assert!(ValueExt::resolve(value, &Schema::Int).await.is_err()); let value = Value::Double(10.0); - assert!(ValueExt::resolve(value, &Schema::LocalTimestampNanos).await.is_err()); + assert!( + ValueExt::resolve(value, &Schema::LocalTimestampNanos) + .await + .is_err() + ); } #[tokio::test] @@ -2026,16 +2111,36 @@ Field with name '"b"' is not a member of the map items"#, Days::new(5), Millis::new(3000), )); - assert!(ValueExt::resolve(value.clone(), &Schema::Duration).await.is_ok()); - assert!(ValueExt::resolve(value, &Schema::TimestampMicros).await.is_err()); - assert!(ValueExt::resolve(Value::Long(1i64), &Schema::Duration).await.is_err()); + assert!( + ValueExt::resolve(value.clone(), &Schema::Duration) + .await + .is_ok() + ); + assert!( + ValueExt::resolve(value, &Schema::TimestampMicros) + .await + .is_err() + ); + assert!( + ValueExt::resolve(Value::Long(1i64), &Schema::Duration) + .await + .is_err() + ); } #[tokio::test] async fn resolve_uuid() -> TestResult { let value = Value::Uuid(Uuid::parse_str("1481531d-ccc9-46d9-a56f-5b67459c0537")?); - assert!(ValueExt::resolve(value.clone(), &Schema::Uuid).await.is_ok()); - assert!(ValueExt::resolve(value, &Schema::TimestampMicros).await.is_err()); + assert!( + ValueExt::resolve(value.clone(), &Schema::Uuid) + .await + .is_ok() + ); + assert!( + ValueExt::resolve(value, &Schema::TimestampMicros) + .await + .is_err() + ); Ok(()) } @@ -2545,7 +2650,7 @@ Field with name '"b"' is not a member of the map items"#, ("a".into(), inner_value1), ("b".into(), inner_value2.clone()), ]); - ValueExt::resolve(outer1, &schema) + ValueExt::resolve(outer1, &schema) .await .expect("Record definition defined in union must be resolved in other field"); let outer2 = Value::Record(vec![("a".into(), Value::Null), ("b".into(), inner_value2)]); @@ -3172,8 +3277,7 @@ Field with name '"b"' is not a member of the map items"#, let schemata: Vec<_> = schemas.iter().skip(1).collect(); let resolve_result = - ValueExt::resolve_schemata(avro_value.clone(), main_schema, schemata) - .await; + ValueExt::resolve_schemata(avro_value.clone(), main_schema, schemata).await; assert!( resolve_result.is_ok(), @@ -3214,8 +3318,8 @@ Field with name '"b"' is not a member of the map items"#, let main_schema = schemata.last().unwrap(); let other_schemata: Vec<&Schema> = schemata.iter().take(2).collect(); - let resolve_result = ValueExt::resolve_schemata(avro_value, main_schema, other_schemata) - .await; + let resolve_result = + ValueExt::resolve_schemata(avro_value, main_schema, other_schemata).await; assert!( resolve_result.is_ok(), @@ -3223,8 +3327,7 @@ Field with name '"b"' is not a member of the map items"#, ); assert!( - ValueExt::validate_schemata(&resolve_result?, schemata.iter().collect()) - .await, + ValueExt::validate_schemata(&resolve_result?, schemata.iter().collect()).await, "result of validation with schemata should be true" ); @@ -3268,44 +3371,53 @@ Field with name '"b"' is not a member of the map items"#, async fn test_avro_3892_resolve_fixed_from_bytes() -> TestResult { let value = Value::Bytes(vec![97, 98, 99]); assert_eq!( - ValueExt::resolve(value, &Schema::Fixed(FixedSchema { + ValueExt::resolve( + value, + &Schema::Fixed(FixedSchema { name: "test".into(), aliases: None, doc: None, size: 3, default: None, attributes: Default::default() - })) - .await?, + }) + ) + .await?, Value::Fixed(3, vec![97, 98, 99]) ); let value = Value::Bytes(vec![97, 99]); assert!( - ValueExt::resolve(value, &Schema::Fixed(FixedSchema { + ValueExt::resolve( + value, + &Schema::Fixed(FixedSchema { name: "test".into(), aliases: None, doc: None, size: 3, default: None, attributes: Default::default() - })) - .await - .is_err(), + }) + ) + .await + .is_err(), ); let value = Value::Bytes(vec![97, 98, 99, 100]); assert!( - ValueExt::resolve(value, &Schema::Fixed(FixedSchema { + ValueExt::resolve( + value, + &Schema::Fixed(FixedSchema { name: "test".into(), aliases: None, doc: None, size: 3, default: None, attributes: Default::default() - })) - .await - .is_err(), + }) + ) + .await + .is_err(), ); Ok(()) @@ -3360,7 +3472,10 @@ Field with name '"b"' is not a member of the map items"#, async fn avro_4024_resolve_double_from_unknown_string_err() -> TestResult { let schema = SchemaExt::parse_str(r#"{"type": "double"}"#).await?; let value = Value::String("unknown".to_owned()); - match ValueExt::resolve(value, &schema).await.map_err(Error::into_details) { + match ValueExt::resolve(value, &schema) + .await + .map_err(Error::into_details) + { Err(err @ Details::GetDouble(_)) => { assert_eq!( format!("{err:?}"), @@ -3378,7 +3493,10 @@ Field with name '"b"' is not a member of the map items"#, async fn avro_4024_resolve_float_from_unknown_string_err() -> TestResult { let schema = SchemaExt::parse_str(r#"{"type": "float"}"#).await?; let value = Value::String("unknown".to_owned()); - match ValueExt::resolve(value, &schema).await.map_err(Error::into_details) { + match ValueExt::resolve(value, &schema) + .await + .map_err(Error::into_details) + { Err(err @ Details::GetFloat(_)) => { assert_eq!( format!("{err:?}"), diff --git a/avro/src/util.rs b/avro/src/util.rs index 5f3d62ff..e9c303b5 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -185,7 +185,8 @@ mod util { } } writer - .write(&buffer[..i]).await + .write(&buffer[..i]) + .await .map_err(|e| Details::WriteBytes(e).into()) } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 9a1f43a8..4f6fc06d 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -51,6 +51,11 @@ mod writer { use tokio::io::AsyncWriteExt; use crate::AvroResult; + #[synca::cfg(sync)] + use crate::schema::Name; + #[synca::cfg(sync)] + use crate::ser_schema::SchemaAwareWriteSerializer; + use crate::types::tokio::ValueExt; use crate::{ codec::tokio::Codec, encode::tokio::{encode, encode_internal, encode_to_vec}, @@ -61,16 +66,8 @@ mod writer { types::Value, }; #[synca::cfg(sync)] - use crate::schema::Name; - #[synca::cfg(sync)] - use crate::ser_schema::SchemaAwareWriteSerializer; - #[synca::cfg(sync)] use serde::Serialize; - use std::{ - collections::HashMap, marker::PhantomData, mem::ManuallyDrop, - ops::RangeInclusive, - }; - use crate::types::tokio::ValueExt; + use std::{collections::HashMap, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive}; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; @@ -387,7 +384,8 @@ mod writer { + self.append_raw(&stream_len.into(), &Schema::Long).await? + self .writer - .write(self.buffer.as_ref()).await + .write(self.buffer.as_ref()) + .await .map_err(Details::WriteBytes)? + self.append_marker().await?; @@ -443,19 +441,22 @@ mod writer { // using .writer.write directly to avoid mutable borrow of self // with ref borrowing of self.marker self.writer - .write(&self.marker).await + .write(&self.marker) + .await .map_err(|e| Details::WriteMarker(e).into()) } /// Append a raw Avro Value to the payload avoiding to encode it again. async fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult { - self.append_bytes(encode_to_vec(value, schema).await?.as_ref()).await + self.append_bytes(encode_to_vec(value, schema).await?.as_ref()) + .await } /// Append pure bytes to the payload. async fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult { self.writer - .write(bytes).await + .write(bytes) + .await .map_err(|e| Details::WriteBytes(e).into()) } @@ -573,8 +574,7 @@ mod writer { let names = rs.get_names(); let enclosing_namespace = schema.namespace(); if let Some(_err) = - ValueExt::validate_internal(&avro, schema, names, &enclosing_namespace) - .await + ValueExt::validate_internal(&avro, schema, names, &enclosing_namespace).await { return Err(Details::Validation.into()); } @@ -627,7 +627,8 @@ mod writer { } else { write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer).await?; writer - .write_all(&self.buffer).await + .write_all(&self.buffer) + .await .map_err(Details::WriteBytes)?; let len = self.buffer.len(); self.buffer.truncate(original_length); @@ -694,12 +695,17 @@ mod writer { { /// Write the referenced `Serialize` object to the provided `Write` object. Returns a result with /// the number of bytes written including the header - pub async fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { + pub async fn write_ref( + &mut self, + data: &T, + writer: &mut W, + ) -> AvroResult { let mut bytes_written: usize = 0; if !self.header_written { bytes_written += writer - .write(self.inner.buffer.as_slice()).await + .write(self.inner.buffer.as_slice()) + .await .map_err(Details::WriteBytes)?; self.header_written = true; } @@ -711,7 +717,11 @@ mod writer { /// Write the Serialize object to the provided Write object. Returns a result with the number /// of bytes written including the header - pub async fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + pub async fn write( + &mut self, + data: T, + writer: &mut W, + ) -> AvroResult { self.write_ref(&data, writer).await } } @@ -722,8 +732,13 @@ mod writer { value: &Value, buffer: &mut Vec, ) -> AvroResult { - match ValueExt::validate_internal(&value, schema, resolved_schema.get_names(), &schema.namespace()) - .await + match ValueExt::validate_internal( + &value, + schema, + resolved_schema.get_names(), + &schema.namespace(), + ) + .await { Some(reason) => Err(Details::ValidationWithReason { value: value.clone(), @@ -751,12 +766,12 @@ mod writer { ) -> AvroResult<()> { let root_schema = resolved_schema.get_root_schema(); if let Some(reason) = ValueExt::validate_internal( - &value, - root_schema, - resolved_schema.get_names(), - &root_schema.namespace(), - ) - .await + &value, + root_schema, + resolved_schema.get_names(), + &root_schema.namespace(), + ) + .await { return Err(Details::ValidationWithReason { value: value.clone(), @@ -861,9 +876,9 @@ mod writer { use serde::{Deserialize, Serialize}; use uuid::Uuid; + use crate::schema::AvroSchema; use crate::{codec::tokio::DeflateSettings, error::Details}; use apache_avro_test_helper::TestResult; - use crate::schema::AvroSchema; const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); @@ -1015,7 +1030,7 @@ mod writer { &Schema::Int, 1_i32, ) - .await + .await } #[tokio::test] @@ -1027,7 +1042,7 @@ mod writer { &Schema::Int, 1_i32, ) - .await + .await } #[tokio::test] @@ -1039,7 +1054,7 @@ mod writer { &Schema::Long, 1_i64, ) - .await + .await } #[tokio::test] @@ -1051,7 +1066,7 @@ mod writer { &Schema::Long, 1_i64, ) - .await + .await } #[tokio::test] @@ -1063,7 +1078,7 @@ mod writer { &Schema::Long, 1_i64, ) - .await + .await } #[tokio::test] @@ -1106,7 +1121,7 @@ mod writer { &inner, value, ) - .await + .await } #[tokio::test] @@ -1589,7 +1604,7 @@ mod writer { &TestSingleObjectWriter::get_schema(), 1024, ) - .expect("Should resolve schema"); + .expect("Should resolve schema"); let value = obj.into(); let written_bytes = writer .write_value_ref(&value, &mut buf) @@ -1611,7 +1626,7 @@ mod writer { &TestSingleObjectWriter::get_schema(), &mut msg_binary, ) - .expect("encode should have failed by here as a dependency of any writing"); + .expect("encode should have failed by here as a dependency of any writing"); assert_eq!(&buf[10..], &msg_binary[..]); Ok(()) @@ -1633,7 +1648,7 @@ mod writer { 1024, header_builder, ) - .expect("Should resolve schema"); + .expect("Should resolve schema"); let value = obj.into(); writer .write_value_ref(&value, &mut buf) @@ -1662,7 +1677,7 @@ mod writer { &TestSingleObjectWriter::get_schema(), 1024, ) - .expect("Should resolve schema"); + .expect("Should resolve schema"); let mut specific_writer = SpecificSingleObjectWriter::::with_capacity(1024) .await diff --git a/avro/tests/append_to_existing.rs b/avro/tests/append_to_existing.rs index a1eafd2b..e756422e 100644 --- a/avro/tests/append_to_existing.rs +++ b/avro/tests/append_to_existing.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{AvroResult, Reader, Schema, Writer, read_marker, types::{Record, Value}, SchemaExt}; +use apache_avro::{ + AvroResult, Reader, Schema, SchemaExt, Writer, read_marker, + types::{Record, Value}, +}; use apache_avro_test_helper::TestResult; const SCHEMA: &str = r#"{ diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index b3a9e78d..91ef5b07 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -16,8 +16,8 @@ // under the License. use apache_avro::types::Value; -use apache_avro::{from_avro_datum, to_avro_datum, to_value, SchemaExt}; use apache_avro::types::sync::ValueExt; +use apache_avro::{SchemaExt, from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; #[test] diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index bc371ebd..85a4e690 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -17,8 +17,8 @@ use apache_avro::SchemaExt; use apache_avro::types::Value; -use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro::types::sync::ValueExt; +use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; #[test] diff --git a/avro/tests/codecs.rs b/avro/tests/codecs.rs index f9fca887..7198e6ef 100644 --- a/avro/tests/codecs.rs +++ b/avro/tests/codecs.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, DeflateSettings, Reader, Writer, types::{Record, Value}, SchemaExt}; +use apache_avro::{ + Codec, DeflateSettings, Reader, SchemaExt, Writer, + types::{Record, Value}, +}; use apache_avro_test_helper::TestResult; use miniz_oxide::deflate::CompressionLevel; diff --git a/avro/tests/io.rs b/avro/tests/io.rs index a9a94de7..4a885068 100644 --- a/avro/tests/io.rs +++ b/avro/tests/io.rs @@ -16,11 +16,13 @@ // under the License. //! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py -use apache_avro::{Error, Schema, error::Details, from_avro_datum, to_avro_datum, types::Value, SchemaExt}; +use apache_avro::types::sync::ValueExt; +use apache_avro::{ + Error, Schema, SchemaExt, error::Details, from_avro_datum, to_avro_datum, types::Value, +}; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use std::{io::Cursor, sync::OnceLock}; -use apache_avro::types::sync::ValueExt; fn schemas_to_validate() -> &'static Vec<(&'static str, Value)> { static SCHEMAS_TO_VALIDATE_ONCE: OnceLock> = OnceLock::new(); diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index de7797e3..6ed97b1a 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -15,18 +15,22 @@ // specific language governing permissions and limitations // under the License. -use std::{ - collections::HashMap, - io::Cursor, -}; -use std::io::Read; -use apache_avro::{Codec, Reader, Writer, error::{Details, Error}, from_avro_datum, from_value, schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, to_avro_datum, to_value, types::{Record, Value}, SchemaExt}; use apache_avro::types::sync::ValueExt; +use apache_avro::{ + Codec, Reader, SchemaExt, Writer, + error::{Details, Error}, + from_avro_datum, from_value, + schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, + to_avro_datum, to_value, + types::{Record, Value}, +}; use apache_avro_test_helper::{ TestResult, data::{DOC_EXAMPLES, examples, valid_examples}, init, }; +use std::io::Read; +use std::{collections::HashMap, io::Cursor}; #[test] fn test_correct_recursive_extraction() -> TestResult { diff --git a/avro/tests/shared.rs b/avro/tests/shared.rs index c40a10e8..99cc467c 100644 --- a/avro/tests/shared.rs +++ b/avro/tests/shared.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, Reader, Schema, Writer, types::Value, SchemaExt}; +use apache_avro::{Codec, Reader, Schema, SchemaExt, Writer, types::Value}; use apache_avro_test_helper::TestResult; use std::{ fmt, diff --git a/avro/tests/to_from_avro_datum_schemata.rs b/avro/tests/to_from_avro_datum_schemata.rs index b204e79e..e9ef51a1 100644 --- a/avro/tests/to_from_avro_datum_schemata.rs +++ b/avro/tests/to_from_avro_datum_schemata.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{Codec, Reader, Schema, Writer, from_avro_datum_reader_schemata, from_avro_datum_schemata, to_avro_datum_schemata, types::Value, SchemaExt}; +use apache_avro::{ + Codec, Reader, Schema, SchemaExt, Writer, from_avro_datum_reader_schemata, + from_avro_datum_schemata, to_avro_datum_schemata, types::Value, +}; use apache_avro_test_helper::{TestResult, init}; static SCHEMA_A_STR: &str = r#"{ diff --git a/avro/tests/union_schema.rs b/avro/tests/union_schema.rs index 3dbaaff4..3e55f024 100644 --- a/avro/tests/union_schema.rs +++ b/avro/tests/union_schema.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::{AvroResult, Codec, Reader, Schema, Writer, from_value, SchemaExt}; +use apache_avro::{AvroResult, Codec, Reader, Schema, SchemaExt, Writer, from_value}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; static SCHEMA_A_STR: &str = r#"{ diff --git a/avro_derive/tests/derive.rs b/avro_derive/tests/derive.rs index f01ec10d..a1664122 100644 --- a/avro_derive/tests/derive.rs +++ b/avro_derive/tests/derive.rs @@ -19,8 +19,8 @@ mod test_derive { use apache_avro::{ - Reader, Schema, Writer, AvroSchema, from_value, - schema::sync::AvroSchemaComponent, schema::sync::SchemaExt, + AvroSchema, Reader, Schema, Writer, from_value, schema::sync::AvroSchemaComponent, + schema::sync::SchemaExt, }; use apache_avro_derive::*; use proptest::prelude::*; From 504368339bd313662f4b39c07a9872731a6149a7 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 14:17:24 +0300 Subject: [PATCH 29/47] Remove synca from codec.rs since there is nothing async in it Fix clippy and formatting Signed-off-by: Martin Tzvetanov Grigorov --- avro/benches/serde.rs | 2 +- avro/benches/single.rs | 1 + avro/src/bigdecimal.rs | 3 +- avro/src/codec.rs | 548 +++++++++++++++---------------- avro/src/lib.rs | 12 +- avro/src/reader.rs | 26 +- avro/src/schema.rs | 6 +- avro/src/schema_compatibility.rs | 3 +- avro/src/ser_schema.rs | 2 +- avro/src/types.rs | 12 +- avro/src/writer.rs | 28 +- avro/tests/io.rs | 2 +- avro_derive/Cargo.toml | 4 +- avro_derive/tests/derive.rs | 5 +- 14 files changed, 317 insertions(+), 337 deletions(-) diff --git a/avro/benches/serde.rs b/avro/benches/serde.rs index aa714fd1..b3ae80cb 100644 --- a/avro/benches/serde.rs +++ b/avro/benches/serde.rs @@ -16,7 +16,7 @@ // under the License. use apache_avro::{ - AvroResult, Reader, Writer, + AvroResult, Reader, SchemaExt, Writer, schema::Schema, types::{Record, Value}, }; diff --git a/avro/benches/single.rs b/avro/benches/single.rs index f95caf82..0b21a38f 100644 --- a/avro/benches/single.rs +++ b/avro/benches/single.rs @@ -16,6 +16,7 @@ // under the License. use apache_avro::{ + SchemaExt, schema::Schema, to_avro_datum, types::{Record, Value}, diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index 44c5b584..a0a1429c 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -25,7 +25,6 @@ pub use bigdecimal::BigDecimal; sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::codec::tokio => crate::codec::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -101,7 +100,7 @@ mod bigdecimal { mod tests { use super::*; use crate::{ - codec::tokio::Codec, error::Error, reader::tokio::Reader, schema::tokio::SchemaExt, + codec::Codec, error::Error, reader::tokio::Reader, schema::tokio::SchemaExt, types::Record, writer::tokio::Writer, }; use apache_avro_test_helper::TestResult; diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 0b5e7234..6a8e8eac 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -17,151 +17,130 @@ //! Logic for all supported compression codecs in Avro. -#[synca::synca( - #[cfg(feature = "tokio")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { - sync!(); - replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - #[tokio::test] => #[test] - ); - } -)] -mod codec { - - use crate::AvroResult; - use crate::{error::Details, error::Error, types::Value}; - use strum_macros::{EnumIter, EnumString, IntoStaticStr}; - - /// Settings for the `Deflate` codec. - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct DeflateSettings { - compression_level: miniz_oxide::deflate::CompressionLevel, - } - - impl DeflateSettings { - pub fn new(compression_level: miniz_oxide::deflate::CompressionLevel) -> Self { - DeflateSettings { compression_level } - } +use crate::AvroResult; +use crate::{error::Details, error::Error, types::Value}; +use strum_macros::{EnumIter, EnumString, IntoStaticStr}; + +/// Settings for the `Deflate` codec. +#[derive(Clone, Copy, Eq, PartialEq, Debug)] +pub struct DeflateSettings { + compression_level: miniz_oxide::deflate::CompressionLevel, +} - fn compression_level(&self) -> u8 { - self.compression_level as u8 - } +impl DeflateSettings { + pub fn new(compression_level: miniz_oxide::deflate::CompressionLevel) -> Self { + DeflateSettings { compression_level } } - impl Default for DeflateSettings { - /// Default compression level is `miniz_oxide::deflate::CompressionLevel::DefaultCompression`. - fn default() -> Self { - Self::new(miniz_oxide::deflate::CompressionLevel::DefaultCompression) - } + fn compression_level(&self) -> u8 { + self.compression_level as u8 } +} - /// The compression codec used to compress blocks. - #[derive(Clone, Copy, Debug, Eq, PartialEq, EnumIter, EnumString, IntoStaticStr)] - #[strum(serialize_all = "kebab_case")] - pub enum Codec { - /// The `Null` codec simply passes through data uncompressed. - Null, - /// The `Deflate` codec writes the data block using the deflate algorithm - /// as specified in RFC 1951, and typically implemented using the zlib library. - /// Note that this format (unlike the "zlib format" in RFC 1950) does not have a checksum. - Deflate(DeflateSettings), - #[cfg(feature = "snappy")] - /// The `Snappy` codec uses Google's [Snappy](http://google.github.io/snappy/) - /// compression library. Each compressed block is followed by the 4-byte, big-endian - /// CRC32 checksum of the uncompressed data in the block. - Snappy, - #[cfg(feature = "zstandard")] - /// The `Zstandard` codec uses Facebook's [Zstandard](https://facebook.github.io/zstd/) - Zstandard(zstandard::ZstandardSettings), - #[cfg(feature = "bzip")] - /// The `BZip2` codec uses [BZip2](https://sourceware.org/bzip2/) - /// compression library. - Bzip2(bzip::Bzip2Settings), - #[cfg(feature = "xz")] - /// The `Xz` codec uses [Xz utils](https://tukaani.org/xz/) - /// compression library. - Xz(xz::XzSettings), +impl Default for DeflateSettings { + /// Default compression level is `miniz_oxide::deflate::CompressionLevel::DefaultCompression`. + fn default() -> Self { + Self::new(miniz_oxide::deflate::CompressionLevel::DefaultCompression) } +} - impl From for Value { - fn from(value: Codec) -> Self { - Self::Bytes(<&str>::from(value).as_bytes().to_vec()) - } +/// The compression codec used to compress blocks. +#[derive(Clone, Copy, Debug, Eq, PartialEq, EnumIter, EnumString, IntoStaticStr)] +#[strum(serialize_all = "kebab_case")] +pub enum Codec { + /// The `Null` codec simply passes through data uncompressed. + Null, + /// The `Deflate` codec writes the data block using the deflate algorithm + /// as specified in RFC 1951, and typically implemented using the zlib library. + /// Note that this format (unlike the "zlib format" in RFC 1950) does not have a checksum. + Deflate(DeflateSettings), + #[cfg(feature = "snappy")] + /// The `Snappy` codec uses Google's [Snappy](http://google.github.io/snappy/) + /// compression library. Each compressed block is followed by the 4-byte, big-endian + /// CRC32 checksum of the uncompressed data in the block. + Snappy, + #[cfg(feature = "zstandard")] + /// The `Zstandard` codec uses Facebook's [Zstandard](https://facebook.github.io/zstd/) + Zstandard(zstandard::ZstandardSettings), + #[cfg(feature = "bzip")] + /// The `BZip2` codec uses [BZip2](https://sourceware.org/bzip2/) + /// compression library. + Bzip2(bzip::Bzip2Settings), + #[cfg(feature = "xz")] + /// The `Xz` codec uses [Xz utils](https://tukaani.org/xz/) + /// compression library. + Xz(xz::XzSettings), +} + +impl From for Value { + fn from(value: Codec) -> Self { + Self::Bytes(<&str>::from(value).as_bytes().to_vec()) } +} - impl Codec { - /// Compress a stream of bytes in-place. - pub fn compress(self, stream: &mut Vec) -> AvroResult<()> { - match self { - Codec::Null => (), - Codec::Deflate(settings) => { - let compressed = - miniz_oxide::deflate::compress_to_vec(stream, settings.compression_level()); - *stream = compressed; - } - #[cfg(feature = "snappy")] - Codec::Snappy => { - let mut encoded: Vec = vec![0; snap::raw::max_compress_len(stream.len())]; - let compressed_size = snap::raw::Encoder::new() - .compress(&stream[..], &mut encoded[..]) - .map_err(Details::SnappyCompress)?; - - let mut hasher = crc32fast::Hasher::new(); - hasher.update(&stream[..]); - let checksum = hasher.finalize(); - let checksum_as_bytes = checksum.to_be_bytes(); - let checksum_len = checksum_as_bytes.len(); - encoded.truncate(compressed_size + checksum_len); - encoded[compressed_size..].copy_from_slice(&checksum_as_bytes); - - *stream = encoded; - } - #[cfg(feature = "zstandard")] - Codec::Zstandard(settings) => { - use std::io::Write; - let mut encoder = - zstd::Encoder::new(Vec::new(), settings.compression_level as i32).unwrap(); - encoder.write_all(stream).map_err(Details::ZstdCompress)?; - *stream = encoder.finish().unwrap(); - } - #[cfg(feature = "bzip")] - Codec::Bzip2(settings) => { - use bzip2::read::BzEncoder; - use std::io::Read; - - let mut encoder = BzEncoder::new(&stream[..], settings.compression()); - let mut buffer = Vec::new(); - encoder.read_to_end(&mut buffer).unwrap(); - *stream = buffer; - } - #[cfg(feature = "xz")] - Codec::Xz(settings) => { - use std::io::Read; - use xz2::read::XzEncoder; - - let mut encoder = - XzEncoder::new(&stream[..], settings.compression_level as u32); - let mut buffer = Vec::new(); - encoder.read_to_end(&mut buffer).unwrap(); - *stream = buffer; - } - }; +impl Codec { + /// Compress a stream of bytes in-place. + pub fn compress(self, stream: &mut Vec) -> AvroResult<()> { + match self { + Codec::Null => (), + Codec::Deflate(settings) => { + let compressed = + miniz_oxide::deflate::compress_to_vec(stream, settings.compression_level()); + *stream = compressed; + } + #[cfg(feature = "snappy")] + Codec::Snappy => { + let mut encoded: Vec = vec![0; snap::raw::max_compress_len(stream.len())]; + let compressed_size = snap::raw::Encoder::new() + .compress(&stream[..], &mut encoded[..]) + .map_err(Details::SnappyCompress)?; - Ok(()) - } + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&stream[..]); + let checksum = hasher.finalize(); + let checksum_as_bytes = checksum.to_be_bytes(); + let checksum_len = checksum_as_bytes.len(); + encoded.truncate(compressed_size + checksum_len); + encoded[compressed_size..].copy_from_slice(&checksum_as_bytes); + + *stream = encoded; + } + #[cfg(feature = "zstandard")] + Codec::Zstandard(settings) => { + use std::io::Write; + let mut encoder = + zstd::Encoder::new(Vec::new(), settings.compression_level as i32).unwrap(); + encoder.write_all(stream).map_err(Details::ZstdCompress)?; + *stream = encoder.finish().unwrap(); + } + #[cfg(feature = "bzip")] + Codec::Bzip2(settings) => { + use bzip2::read::BzEncoder; + use std::io::Read; - /// Decompress a stream of bytes in-place. - pub fn decompress(self, stream: &mut Vec) -> AvroResult<()> { - *stream = match self { + let mut encoder = BzEncoder::new(&stream[..], settings.compression()); + let mut buffer = Vec::new(); + encoder.read_to_end(&mut buffer).unwrap(); + *stream = buffer; + } + #[cfg(feature = "xz")] + Codec::Xz(settings) => { + use std::io::Read; + use xz2::read::XzEncoder; + + let mut encoder = XzEncoder::new(&stream[..], settings.compression_level as u32); + let mut buffer = Vec::new(); + encoder.read_to_end(&mut buffer).unwrap(); + *stream = buffer; + } + }; + + Ok(()) + } + + /// Decompress a stream of bytes in-place. + pub fn decompress(self, stream: &mut Vec) -> AvroResult<()> { + *stream = match self { Codec::Null => return Ok(()), Codec::Deflate(_settings) => miniz_oxide::inflate::decompress_to_vec(stream).map_err(|e| { let err = { @@ -234,196 +213,195 @@ mod codec { decoded } }; - Ok(()) - } + Ok(()) } +} - #[cfg(feature = "bzip")] - pub mod bzip { - use bzip2::Compression; - - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct Bzip2Settings { - pub compression_level: u8, - } +#[cfg(feature = "bzip")] +pub mod bzip { + use bzip2::Compression; - impl Bzip2Settings { - pub fn new(compression_level: u8) -> Self { - Self { compression_level } - } + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct Bzip2Settings { + pub compression_level: u8, + } - pub(crate) fn compression(&self) -> Compression { - Compression::new(self.compression_level as u32) - } + impl Bzip2Settings { + pub fn new(compression_level: u8) -> Self { + Self { compression_level } } - impl Default for Bzip2Settings { - fn default() -> Self { - Bzip2Settings::new(Compression::best().level() as u8) - } + pub(crate) fn compression(&self) -> Compression { + Compression::new(self.compression_level as u32) } } - #[cfg(feature = "zstandard")] - pub mod zstandard { - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct ZstandardSettings { - pub compression_level: u8, + impl Default for Bzip2Settings { + fn default() -> Self { + Bzip2Settings::new(Compression::best().level() as u8) } + } +} - impl ZstandardSettings { - pub fn new(compression_level: u8) -> Self { - Self { compression_level } - } - } +#[cfg(feature = "zstandard")] +pub mod zstandard { + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct ZstandardSettings { + pub compression_level: u8, + } - impl Default for ZstandardSettings { - fn default() -> Self { - Self::new(0) - } + impl ZstandardSettings { + pub fn new(compression_level: u8) -> Self { + Self { compression_level } } } - #[cfg(feature = "xz")] - pub mod xz { - #[derive(Clone, Copy, Eq, PartialEq, Debug)] - pub struct XzSettings { - pub compression_level: u8, + impl Default for ZstandardSettings { + fn default() -> Self { + Self::new(0) } + } +} - impl XzSettings { - pub fn new(compression_level: u8) -> Self { - Self { compression_level } - } - } +#[cfg(feature = "xz")] +pub mod xz { + #[derive(Clone, Copy, Eq, PartialEq, Debug)] + pub struct XzSettings { + pub compression_level: u8, + } - impl Default for XzSettings { - fn default() -> Self { - XzSettings::new(9) - } + impl XzSettings { + pub fn new(compression_level: u8) -> Self { + Self { compression_level } } } - #[cfg(test)] - mod tests { - use super::*; - use apache_avro_test_helper::TestResult; - use miniz_oxide::deflate::CompressionLevel; - use pretty_assertions::{assert_eq, assert_ne}; - - const INPUT: &[u8] = b"theanswertolifetheuniverseandeverythingis42theanswertolifetheuniverseandeverythingis4theanswertolifetheuniverseandeverythingis2"; - - #[test] - fn null_compress_and_decompress() -> TestResult { - let codec = Codec::Null; - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream)?; - assert_eq!(INPUT, stream.as_slice()); - codec.decompress(&mut stream)?; - assert_eq!(INPUT, stream.as_slice()); - Ok(()) + impl Default for XzSettings { + fn default() -> Self { + XzSettings::new(9) } + } +} - #[test] - fn deflate_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Deflate(DeflateSettings::new( - CompressionLevel::BestCompression, - ))) - } +#[cfg(test)] +mod tests { + use super::*; + use apache_avro_test_helper::TestResult; + use miniz_oxide::deflate::CompressionLevel; + use pretty_assertions::{assert_eq, assert_ne}; + + const INPUT: &[u8] = b"theanswertolifetheuniverseandeverythingis42theanswertolifetheuniverseandeverythingis4theanswertolifetheuniverseandeverythingis2"; + + #[test] + fn null_compress_and_decompress() -> TestResult { + let codec = Codec::Null; + let mut stream = INPUT.to_vec(); + codec.compress(&mut stream)?; + assert_eq!(INPUT, stream.as_slice()); + codec.decompress(&mut stream)?; + assert_eq!(INPUT, stream.as_slice()); + Ok(()) + } - #[cfg(feature = "snappy")] - #[test] - fn snappy_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Snappy) - } + #[test] + fn deflate_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Deflate(DeflateSettings::new( + CompressionLevel::BestCompression, + ))) + } - #[cfg(feature = "zstandard")] - #[test] - fn zstd_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Zstandard(zstandard::ZstandardSettings::default())) - } + #[cfg(feature = "snappy")] + #[test] + fn snappy_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Snappy) + } - #[cfg(feature = "bzip")] - #[test] - fn bzip_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Bzip2(bzip::Bzip2Settings::default())) - } + #[cfg(feature = "zstandard")] + #[test] + fn zstd_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Zstandard(zstandard::ZstandardSettings::default())) + } - #[cfg(feature = "xz")] - #[test] - fn xz_compress_and_decompress() -> TestResult { - compress_and_decompress(Codec::Xz(xz::XzSettings::default())) - } + #[cfg(feature = "bzip")] + #[test] + fn bzip_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Bzip2(bzip::Bzip2Settings::default())) + } - fn compress_and_decompress(codec: Codec) -> TestResult { - let mut stream = INPUT.to_vec(); - codec.compress(&mut stream)?; - assert_ne!(INPUT, stream.as_slice()); - assert!(INPUT.len() > stream.len()); - codec.decompress(&mut stream)?; - assert_eq!(INPUT, stream.as_slice()); - Ok(()) - } + #[cfg(feature = "xz")] + #[test] + fn xz_compress_and_decompress() -> TestResult { + compress_and_decompress(Codec::Xz(xz::XzSettings::default())) + } - #[test] - fn codec_to_str() { - assert_eq!(<&str>::from(Codec::Null), "null"); - assert_eq!( - <&str>::from(Codec::Deflate(DeflateSettings::default())), - "deflate" - ); + fn compress_and_decompress(codec: Codec) -> TestResult { + let mut stream = INPUT.to_vec(); + codec.compress(&mut stream)?; + assert_ne!(INPUT, stream.as_slice()); + assert!(INPUT.len() > stream.len()); + codec.decompress(&mut stream)?; + assert_eq!(INPUT, stream.as_slice()); + Ok(()) + } - #[cfg(feature = "snappy")] - assert_eq!(<&str>::from(Codec::Snappy), "snappy"); + #[test] + fn codec_to_str() { + assert_eq!(<&str>::from(Codec::Null), "null"); + assert_eq!( + <&str>::from(Codec::Deflate(DeflateSettings::default())), + "deflate" + ); - #[cfg(feature = "zstandard")] - assert_eq!( - <&str>::from(Codec::Zstandard(zstandard::ZstandardSettings::default())), - "zstandard" - ); + #[cfg(feature = "snappy")] + assert_eq!(<&str>::from(Codec::Snappy), "snappy"); - #[cfg(feature = "bzip")] - assert_eq!( - <&str>::from(Codec::Bzip2(bzip::Bzip2Settings::default())), - "bzip2" - ); + #[cfg(feature = "zstandard")] + assert_eq!( + <&str>::from(Codec::Zstandard(zstandard::ZstandardSettings::default())), + "zstandard" + ); - #[cfg(feature = "xz")] - assert_eq!(<&str>::from(Codec::Xz(xz::XzSettings::default())), "xz"); - } + #[cfg(feature = "bzip")] + assert_eq!( + <&str>::from(Codec::Bzip2(bzip::Bzip2Settings::default())), + "bzip2" + ); - #[test] - fn codec_from_str() { - use std::str::FromStr; + #[cfg(feature = "xz")] + assert_eq!(<&str>::from(Codec::Xz(xz::XzSettings::default())), "xz"); + } - assert_eq!(Codec::from_str("null").unwrap(), Codec::Null); - assert_eq!( - Codec::from_str("deflate").unwrap(), - Codec::Deflate(DeflateSettings::default()) - ); + #[test] + fn codec_from_str() { + use std::str::FromStr; - #[cfg(feature = "snappy")] - assert_eq!(Codec::from_str("snappy").unwrap(), Codec::Snappy); + assert_eq!(Codec::from_str("null").unwrap(), Codec::Null); + assert_eq!( + Codec::from_str("deflate").unwrap(), + Codec::Deflate(DeflateSettings::default()) + ); - #[cfg(feature = "zstandard")] - assert_eq!( - Codec::from_str("zstandard").unwrap(), - Codec::Zstandard(zstandard::ZstandardSettings::default()) - ); + #[cfg(feature = "snappy")] + assert_eq!(Codec::from_str("snappy").unwrap(), Codec::Snappy); - #[cfg(feature = "bzip")] - assert_eq!( - Codec::from_str("bzip2").unwrap(), - Codec::Bzip2(bzip::Bzip2Settings::default()) - ); + #[cfg(feature = "zstandard")] + assert_eq!( + Codec::from_str("zstandard").unwrap(), + Codec::Zstandard(zstandard::ZstandardSettings::default()) + ); - #[cfg(feature = "xz")] - assert_eq!( - Codec::from_str("xz").unwrap(), - Codec::Xz(xz::XzSettings::default()) - ); + #[cfg(feature = "bzip")] + assert_eq!( + Codec::from_str("bzip2").unwrap(), + Codec::Bzip2(bzip::Bzip2Settings::default()) + ); - assert!(Codec::from_str("not a codec").is_err()); - } + #[cfg(feature = "xz")] + assert_eq!( + Codec::from_str("xz").unwrap(), + Codec::Xz(xz::XzSettings::default()) + ); + + assert!(Codec::from_str("not a codec").is_err()); } } diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 45781430..01ef0d57 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -902,14 +902,11 @@ pub use crate::bytes::{ }; #[cfg(feature = "bzip")] pub use codec::bzip::Bzip2Settings; -#[cfg(feature = "sync")] -pub use codec::sync::{Codec, DeflateSettings}; -#[cfg(feature = "tokio")] -pub use codec::tokio::{Codec as AsyncCodec, DeflateSettings as AsyncDeflateSettings}; #[cfg(feature = "xz")] pub use codec::xz::XzSettings; #[cfg(feature = "zstandard")] pub use codec::zstandard::ZstandardSettings; +pub use codec::{Codec, DeflateSettings}; #[cfg(feature = "sync")] pub use de::sync::from_value; #[cfg(feature = "tokio")] @@ -967,9 +964,8 @@ pub type AvroResult = Result; pub mod sync { sync!(); replace!( - crate::codec::tokio => crate::codec::sync, - crate::reader::tokio => crate::reader::sync, - crate::schema::tokio => crate::schema::sync, + crate::reader::tokio => crate::reader::sync, + crate::schema::tokio => crate::schema::sync, crate::types::tokio => crate::types::sync, crate::writer::tokio => crate::writer::sync, #[tokio::test] => #[test] @@ -979,7 +975,7 @@ pub type AvroResult = Result; #[cfg(test)] mod tests { use crate::{ - codec::tokio::Codec, + codec::Codec, reader::tokio::{Reader, from_avro_datum}, schema::tokio::SchemaExt, types::{Record, Value}, diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 4f3e2702..d6f66774 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -23,7 +23,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::codec::tokio => crate::codec::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -46,7 +45,6 @@ mod reader { use crate::from_value; #[synca::cfg(sync)] use std::io::Read as AvroRead; - use std::str::FromStr; #[synca::cfg(tokio)] use tokio::io::AsyncRead as AvroRead; #[cfg(feature = "tokio")] @@ -54,7 +52,7 @@ mod reader { use crate::util::tokio::safe_len; use crate::{ - codec::tokio::Codec, + codec::Codec, error::Details, error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, @@ -70,10 +68,10 @@ mod reader { use log::warn; use serde::de::DeserializeOwned; use serde_json::from_slice; + use std::str::FromStr; use std::{collections::HashMap, io::ErrorKind, marker::PhantomData}; - /// Internal Block reader. - + // Internal Block reader. #[derive(Debug, Clone)] struct Block<'r, R> { reader: R, @@ -701,16 +699,20 @@ mod reader { #[cfg(test)] mod tests { use super::*; - use crate::{ - encode::tokio::encode, headers::tokio::GlueSchemaUuidHeader, rabin::Rabin, - types::Record, - }; + #[synca::cfg(sync)] + use crate::encode::tokio::encode; + #[synca::cfg(sync)] + use crate::headers::sync::GlueSchemaUuidHeader; + #[synca::cfg(sync)] + use crate::rabin::Rabin; + use crate::types::Record; use apache_avro_test_helper::TestResult; #[synca::cfg(tokio)] use futures::StreamExt; use pretty_assertions::assert_eq; use serde::Deserialize; use std::io::Cursor; + #[synca::cfg(sync)] use uuid::Uuid; const SCHEMA: &str = r#" @@ -863,6 +865,8 @@ mod reader { let expected = [record1.into(), record2.into()]; let mut i = 0; + + #[allow(clippy::while_let_on_iterator)] while let Some(value) = reader.next().await { assert_eq!(value?, expected[i]); i += 1; @@ -893,6 +897,8 @@ mod reader { .rev() .collect::>(); let mut reader = Reader::with_schema(&schema, &invalid[..]).await?; + + #[allow(clippy::while_let_on_iterator)] while let Some(value) = reader.next().await { assert!(value.is_err()); } @@ -912,6 +918,8 @@ mod reader { async fn test_reader_only_header() -> TestResult { let invalid = ENCODED.iter().copied().take(165).collect::>(); let mut reader = Reader::new(&invalid[..]).await?; + + #[allow(clippy::while_let_on_iterator)] while let Some(value) = reader.next().await { assert!(value.is_err()); } diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 9229043a..c1adcef3 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -271,7 +271,7 @@ impl<'s> ResolvedSchema<'s> { /// Creates `ResolvedSchema` with some already known schemas. /// /// Those schemata would be used to resolve references if needed. - pub fn new_with_known_schemata<'n>( + pub fn new_with_known_schemata( schemata_to_resolve: Vec<&'s Schema>, enclosing_namespace: &Namespace, known_schemata: &Names, @@ -285,7 +285,7 @@ impl<'s> ResolvedSchema<'s> { Ok(rs) } - fn resolve<'n>( + fn resolve( &mut self, schemata: Vec<&'s Schema>, enclosing_namespace: &Namespace, @@ -1782,7 +1782,7 @@ mod schema { { return Some((i, schema)); } else { - i = i + 1; + i += 1; } } None diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index b304752c..f7b42b95 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -25,7 +25,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::codec::tokio => crate::codec::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -556,7 +555,7 @@ mod schema_compatibility { mod tests { use super::*; use crate::{ - codec::tokio::Codec, + codec::Codec, reader::tokio::Reader, schema::tokio::SchemaExt, types::{Record, Value}, diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index 44f1249f..a095614f 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -468,7 +468,7 @@ impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> { }), }; - let ref_schema = self.names.get(full_name.as_ref()).clone(); + let ref_schema = self.names.get(full_name.as_ref()); ref_schema.ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into()) } diff --git a/avro/src/types.rs b/avro/src/types.rs index 06c86902..d1bda68e 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -548,7 +548,7 @@ mod types { (&Value::Union(i, ref value), Schema::Union(inner)) => { if let Some(schema) = inner.variants().get(i as usize) { Box::pin(ValueExt::validate_internal( - &value, + value, schema, names, enclosing_namespace, @@ -575,7 +575,7 @@ mod types { acc = ValueExt::accumulate( acc, Box::pin(ValueExt::validate_internal( - &item, + item, &inner.items, names, enclosing_namespace, @@ -597,7 +597,7 @@ mod types { acc = ValueExt::accumulate( acc, Box::pin(ValueExt::validate_internal( - &value, + value, &inner.types, names, enclosing_namespace, @@ -655,7 +655,7 @@ mod types { ValueExt::accumulate( acc, Box::pin(ValueExt::validate_internal( - &record_field, + record_field, &field.schema, names, record_namespace, @@ -676,7 +676,7 @@ mod types { for field in fields.iter() { if let Some(item) = items.get(&field.name) { let res = Box::pin(ValueExt::validate_internal( - &item, + item, &field.schema, names, enclosing_namespace, @@ -749,7 +749,7 @@ mod types { Value::Union(_i, b) => b, _ => unreachable!(), }; - value = &v; + value = v; } match *schema { Schema::Ref { ref name } => { diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 4f6fc06d..8e4fa3d8 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -25,7 +25,6 @@ sync!(); replace!( crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::codec::tokio => crate::codec::sync, crate::decode::tokio => crate::decode::sync, crate::encode::tokio => crate::encode::sync, crate::error::tokio => crate::error::sync, @@ -57,7 +56,7 @@ mod writer { use crate::ser_schema::SchemaAwareWriteSerializer; use crate::types::tokio::ValueExt; use crate::{ - codec::tokio::Codec, + codec::Codec, encode::tokio::{encode, encode_internal, encode_to_vec}, error::Details, error::Error, @@ -537,6 +536,7 @@ mod writer { } } + #[synca::cfg(sync)] impl Drop for Writer<'_, W> { /// Drop the writer, will try to flush ignoring any errors. fn drop(&mut self) { @@ -695,7 +695,7 @@ mod writer { { /// Write the referenced `Serialize` object to the provided `Write` object. Returns a result with /// the number of bytes written including the header - pub async fn write_ref( + pub fn write_ref( &mut self, data: &T, writer: &mut W, @@ -705,7 +705,6 @@ mod writer { if !self.header_written { bytes_written += writer .write(self.inner.buffer.as_slice()) - .await .map_err(Details::WriteBytes)?; self.header_written = true; } @@ -717,12 +716,12 @@ mod writer { /// Write the Serialize object to the provided Write object. Returns a result with the number /// of bytes written including the header - pub async fn write( + pub fn write( &mut self, data: T, writer: &mut W, ) -> AvroResult { - self.write_ref(&data, writer).await + self.write_ref(&data, writer) } } @@ -733,7 +732,7 @@ mod writer { buffer: &mut Vec, ) -> AvroResult { match ValueExt::validate_internal( - &value, + value, schema, resolved_schema.get_names(), &schema.namespace(), @@ -766,7 +765,7 @@ mod writer { ) -> AvroResult<()> { let root_schema = resolved_schema.get_root_schema(); if let Some(reason) = ValueExt::validate_internal( - &value, + value, root_schema, resolved_schema.get_names(), &root_schema.namespace(), @@ -859,27 +858,28 @@ mod writer { #[cfg(test)] mod tests { use super::*; + #[synca::cfg(sync)] + use crate::schema::AvroSchema; + use crate::{codec::DeflateSettings, error::Details}; use crate::{ decimal::Decimal, duration::{Days, Duration, Millis, Months}, - headers::tokio::GlueSchemaUuidHeader, - rabin::Rabin, reader::tokio::{Reader, from_avro_datum}, schema::tokio::SchemaExt, schema::{DecimalSchema, FixedSchema, Name}, types::Record, util::tokio::zig_i64, }; + #[synca::cfg(sync)] + use crate::{headers::tokio::GlueSchemaUuidHeader, rabin::Rabin}; + use apache_avro_test_helper::TestResult; #[synca::cfg(tokio)] use futures::StreamExt; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; + #[synca::cfg(sync)] use uuid::Uuid; - use crate::schema::AvroSchema; - use crate::{codec::tokio::DeflateSettings, error::Details}; - use apache_avro_test_helper::TestResult; - const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); const SCHEMA: &str = r#" diff --git a/avro/tests/io.rs b/avro/tests/io.rs index 4a885068..281c7fbb 100644 --- a/avro/tests/io.rs +++ b/avro/tests/io.rs @@ -222,7 +222,7 @@ fn test_validate() -> TestResult { for (raw_schema, value) in schemas_to_validate().iter() { let schema = SchemaExt::parse_str(raw_schema)?; assert!( - ValueExt::validate(&value, &schema), + ValueExt::validate(value, &schema), "value {value:?} does not validate schema: {raw_schema}" ); } diff --git a/avro_derive/Cargo.toml b/avro_derive/Cargo.toml index 2bd721d2..48faa8db 100644 --- a/avro_derive/Cargo.toml +++ b/avro_derive/Cargo.toml @@ -43,7 +43,7 @@ syn = { default-features = false, version = "2.0.104", features = [ [dev-dependencies] apache-avro = { default-features = false, path = "../avro", features = [ - "derive", + "derive", "sync" ] } proptest = { default-features = false, version = "1.7.0", features = ["std"] } serde = { workspace = true } @@ -55,4 +55,4 @@ rustdoc-args = ["--cfg", "docsrs"] [[test]] name = "derive" path = "tests/derive.rs" -required-features = ["sync"] +#required-features = ["sync"] diff --git a/avro_derive/tests/derive.rs b/avro_derive/tests/derive.rs index a1664122..ccf0ef7b 100644 --- a/avro_derive/tests/derive.rs +++ b/avro_derive/tests/derive.rs @@ -19,10 +19,10 @@ mod test_derive { use apache_avro::{ - AvroSchema, Reader, Schema, Writer, from_value, schema::sync::AvroSchemaComponent, + AvroSchema, Reader, Schema, Writer, from_value, schema::derive::AvroSchemaComponent, schema::sync::SchemaExt, }; - use apache_avro_derive::*; + // use apache_avro_derive::*; use proptest::prelude::*; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use std::collections::HashMap; @@ -30,7 +30,6 @@ mod test_derive { use apache_avro::schema::{Alias, EnumSchema, RecordSchema}; use std::{borrow::Cow, sync::Mutex}; - use super::*; use pretty_assertions::assert_eq; /// Takes in a type that implements the right combination of traits and runs it through a Serde Cycle and asserts the result is the same From 58870397c6bd4b11097e920787885a74e0feca81 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 13 Aug 2025 14:50:31 +0300 Subject: [PATCH 30/47] Rework TryFrom for serde_json::Value to ValueExt::try_from() Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/types.rs | 208 ++++++++++++++++++++++++---------------------- 1 file changed, 109 insertions(+), 99 deletions(-) diff --git a/avro/src/types.rs b/avro/src/types.rs index d1bda68e..edf70336 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -18,9 +18,8 @@ //! Logic handling the intermediate representation of Avro values. use crate::bigdecimal::BigDecimal; -use crate::error::{Details, Error}; use crate::schema::{RecordSchema, Schema}; -use crate::{AvroResult, Decimal, Duration, Uuid}; +use crate::{Decimal, Duration, Uuid}; use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, @@ -279,70 +278,6 @@ to_value!(Decimal, Value::Decimal); to_value!(BigDecimal, Value::BigDecimal); to_value!(Duration, Value::Duration); -/// Convert Avro values to Json values -impl TryFrom for serde_json::Value { - type Error = Error; - fn try_from(value: Value) -> AvroResult { - match value { - Value::Null => Ok(serde_json::Value::Null), - Value::Boolean(b) => Ok(serde_json::Value::Bool(b)), - Value::Int(i) => Ok(serde_json::Value::Number(i.into())), - Value::Long(l) => Ok(serde_json::Value::Number(l.into())), - Value::Float(f) => serde_json::Number::from_f64(f.into()) - .map(Self::Number) - .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), - Value::Double(d) => serde_json::Number::from_f64(d) - .map(Self::Number) - .ok_or_else(|| Details::ConvertF64ToJson(d).into()), - Value::Bytes(bytes) => Ok(serde_json::Value::Array( - bytes.into_iter().map(|b| b.into()).collect(), - )), - Value::String(s) => Ok(serde_json::Value::String(s)), - Value::Fixed(_size, items) => Ok(serde_json::Value::Array( - items.into_iter().map(|v| v.into()).collect(), - )), - Value::Enum(_i, s) => Ok(serde_json::Value::String(s)), - Value::Union(_i, b) => Self::try_from(*b), - Value::Array(items) => items - .into_iter() - .map(Self::try_from) - .collect::, _>>() - .map(Self::Array), - Value::Map(items) => items - .into_iter() - .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) - .collect::, _>>() - .map(|v| Self::Object(v.into_iter().collect())), - Value::Record(items) => items - .into_iter() - .map(|(key, value)| Self::try_from(value).map(|v| (key, v))) - .collect::, _>>() - .map(|v| Self::Object(v.into_iter().collect())), - Value::Date(d) => Ok(serde_json::Value::Number(d.into())), - Value::Decimal(ref d) => >::try_from(d) - .map(|vec| Self::Array(vec.into_iter().map(|v| v.into()).collect())), - Value::BigDecimal(ref bg) => { - let vec1: Vec = vec![]; //crate::bigdecimal::sync::serialize_big_decimal(bg)?; - Ok(serde_json::Value::Array( - vec1.into_iter().map(|b| b.into()).collect(), - )) - } - Value::TimeMillis(t) => Ok(serde_json::Value::Number(t.into())), - Value::TimeMicros(t) => Ok(serde_json::Value::Number(t.into())), - Value::TimestampMillis(t) => Ok(serde_json::Value::Number(t.into())), - Value::TimestampMicros(t) => Ok(serde_json::Value::Number(t.into())), - Value::TimestampNanos(t) => Ok(serde_json::Value::Number(t.into())), - Value::LocalTimestampMillis(t) => Ok(serde_json::Value::Number(t.into())), - Value::LocalTimestampMicros(t) => Ok(serde_json::Value::Number(t.into())), - Value::LocalTimestampNanos(t) => Ok(serde_json::Value::Number(t.into())), - Value::Duration(d) => Ok(serde_json::Value::Array( - <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), - )), - Value::Uuid(uuid) => Ok(serde_json::Value::String(uuid.as_hyphenated().to_string())), - } - } -} - #[synca::synca( #[cfg(feature = "tokio")] pub mod tokio { }, @@ -368,7 +303,7 @@ mod types { use crate::AvroResult; use crate::{ Uuid, - bigdecimal::tokio::deserialize_big_decimal, + bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, decimal::Decimal, duration::Duration, error::Details, @@ -393,6 +328,81 @@ mod types { pub struct ValueExt; impl ValueExt { + + /// Convert Avro values to Json values + async fn try_from(value: Value) -> AvroResult { + match value { + Value::Null => Ok(serde_json::Value::Null), + Value::Boolean(b) => Ok(serde_json::Value::Bool(b)), + Value::Int(i) => Ok(serde_json::Value::Number(i.into())), + Value::Long(l) => Ok(serde_json::Value::Number(l.into())), + Value::Float(f) => serde_json::Number::from_f64(f.into()) + .map(serde_json::Value::Number) + .ok_or_else(|| Details::ConvertF64ToJson(f.into()).into()), + Value::Double(d) => serde_json::Number::from_f64(d) + .map(serde_json::Value::Number) + .ok_or_else(|| Details::ConvertF64ToJson(d).into()), + Value::Bytes(bytes) => Ok(serde_json::Value::Array( + bytes.into_iter().map(|b| b.into()).collect(), + )), + Value::String(s) => Ok(serde_json::Value::String(s)), + Value::Fixed(_size, items) => Ok(serde_json::Value::Array( + items.into_iter().map(|v| v.into()).collect(), + )), + Value::Enum(_i, s) => Ok(serde_json::Value::String(s)), + Value::Union(_i, b) => Box::pin(Self::try_from(*b)).await, + Value::Array(items) => { + let mut result = Vec::with_capacity(items.len()); + for item in items { + let json = Box::pin(Self::try_from(item)).await?; + result.push(json); + } + Ok(serde_json::Value::Array(result)) + }, + Value::Map(items) => { + let mut result = serde_json::map::Map::with_capacity(items.len()); + for (key, value) in items { + let v = Box::pin(Self::try_from(value)).await?; + result.insert(key, v); + } + Ok(serde_json::Value::Object(result)) + }, + Value::Record(items) => { + let mut result = serde_json::map::Map::with_capacity(items.len()); + for (key, value) in items { + let v = Box::pin(Self::try_from(value)).await?; + result.insert(key, v); + } + Ok(serde_json::Value::Object(result)) + }, + // Value::Record(items) => items + // .into_iter() + // .map(|(key, value)| Self::try_from(value).await.map(|v| (key, v))).await + // .collect::, _>>() + // .map(|v| serde_json::Value::Object(v.into_iter().collect())), + Value::Date(d) => Ok(serde_json::Value::Number(d.into())), + Value::Decimal(ref d) => >::try_from(d) + .map(|vec| serde_json::Value::Array(vec.into_iter().map(|v| v.into()).collect())), + Value::BigDecimal(ref bg) => { + let vec1: Vec = serialize_big_decimal(bg).await?; + Ok(serde_json::Value::Array( + vec1.into_iter().map(|b| b.into()).collect(), + )) + } + Value::TimeMillis(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimeMicros(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimestampMillis(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimestampMicros(t) => Ok(serde_json::Value::Number(t.into())), + Value::TimestampNanos(t) => Ok(serde_json::Value::Number(t.into())), + Value::LocalTimestampMillis(t) => Ok(serde_json::Value::Number(t.into())), + Value::LocalTimestampMicros(t) => Ok(serde_json::Value::Number(t.into())), + Value::LocalTimestampNanos(t) => Ok(serde_json::Value::Number(t.into())), + Value::Duration(d) => Ok(serde_json::Value::Array( + <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), + )), + Value::Uuid(uuid) => Ok(serde_json::Value::String(uuid.as_hyphenated().to_string())), + } + } /// Validate the value against the given [Schema](../schema/enum.Schema.html). /// /// See the [Avro specification](https://avro.apache.org/docs/current/specification) @@ -2206,31 +2216,31 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn json_from_avro() -> TestResult { assert_eq!( - serde_json::Value::try_from(Value::Null)?, + ValueExt::try_from(Value::Null).await?, serde_json::Value::Null ); assert_eq!( - serde_json::Value::try_from(Value::Boolean(true))?, + ValueExt::try_from(Value::Boolean(true)).await?, serde_json::Value::Bool(true) ); assert_eq!( - serde_json::Value::try_from(Value::Int(1))?, + ValueExt::try_from(Value::Int(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::Long(1))?, + ValueExt::try_from(Value::Long(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::Float(1.0))?, + ValueExt::try_from(Value::Float(1.0)).await?, serde_json::Value::Number(serde_json::Number::from_f64(1.0).unwrap()) ); assert_eq!( - serde_json::Value::try_from(Value::Double(1.0))?, + ValueExt::try_from(Value::Double(1.0)).await?, serde_json::Value::Number(serde_json::Number::from_f64(1.0).unwrap()) ); assert_eq!( - serde_json::Value::try_from(Value::Bytes(vec![1, 2, 3]))?, + ValueExt::try_from(Value::Bytes(vec![1, 2, 3])).await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2238,11 +2248,11 @@ Field with name '"b"' is not a member of the map items"#, ]) ); assert_eq!( - serde_json::Value::try_from(Value::String("test".into()))?, + ValueExt::try_from(Value::String("test".into())).await?, serde_json::Value::String("test".into()) ); assert_eq!( - serde_json::Value::try_from(Value::Fixed(3, vec![1, 2, 3]))?, + ValueExt::try_from(Value::Fixed(3, vec![1, 2, 3])).await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2250,22 +2260,22 @@ Field with name '"b"' is not a member of the map items"#, ]) ); assert_eq!( - serde_json::Value::try_from(Value::Enum(1, "test_enum".into()))?, + ValueExt::try_from(Value::Enum(1, "test_enum".into())).await?, serde_json::Value::String("test_enum".into()) ); assert_eq!( - serde_json::Value::try_from(Value::Union( + ValueExt::try_from(Value::Union( 1, Box::new(Value::String("test_enum".into())) - ))?, + )).await?, serde_json::Value::String("test_enum".into()) ); assert_eq!( - serde_json::Value::try_from(Value::Array(vec![ + ValueExt::try_from(Value::Array(vec![ Value::Int(1), Value::Int(2), Value::Int(3) - ]))?, + ])).await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2273,7 +2283,7 @@ Field with name '"b"' is not a member of the map items"#, ]) ); assert_eq!( - serde_json::Value::try_from(Value::Map( + ValueExt::try_from(Value::Map( vec![ ("v1".to_string(), Value::Int(1)), ("v2".to_string(), Value::Int(2)), @@ -2281,7 +2291,7 @@ Field with name '"b"' is not a member of the map items"#, ] .into_iter() .collect() - ))?, + )).await?, serde_json::Value::Object( vec![ ("v1".to_string(), serde_json::Value::Number(1.into())), @@ -2293,11 +2303,11 @@ Field with name '"b"' is not a member of the map items"#, ) ); assert_eq!( - serde_json::Value::try_from(Value::Record(vec![ + ValueExt::try_from(Value::Record(vec![ ("v1".to_string(), Value::Int(1)), ("v2".to_string(), Value::Int(2)), ("v3".to_string(), Value::Int(3)) - ]))?, + ])).await?, serde_json::Value::Object( vec![ ("v1".to_string(), serde_json::Value::Number(1.into())), @@ -2309,11 +2319,11 @@ Field with name '"b"' is not a member of the map items"#, ) ); assert_eq!( - serde_json::Value::try_from(Value::Date(1))?, + ValueExt::try_from(Value::Date(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::Decimal(vec![1, 2, 3].into()))?, + ValueExt::try_from(Value::Decimal(vec![1, 2, 3].into())).await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2321,44 +2331,44 @@ Field with name '"b"' is not a member of the map items"#, ]) ); assert_eq!( - serde_json::Value::try_from(Value::TimeMillis(1))?, + ValueExt::try_from(Value::TimeMillis(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::TimeMicros(1))?, + ValueExt::try_from(Value::TimeMicros(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::TimestampMillis(1))?, + ValueExt::try_from(Value::TimestampMillis(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::TimestampMicros(1))?, + ValueExt::try_from(Value::TimestampMicros(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::TimestampNanos(1))?, + ValueExt::try_from(Value::TimestampNanos(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::LocalTimestampMillis(1))?, + ValueExt::try_from(Value::LocalTimestampMillis(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::LocalTimestampMicros(1))?, + ValueExt::try_from(Value::LocalTimestampMicros(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::LocalTimestampNanos(1))?, + ValueExt::try_from(Value::LocalTimestampNanos(1)).await?, serde_json::Value::Number(1.into()) ); assert_eq!( - serde_json::Value::try_from(Value::Duration( + ValueExt::try_from(Value::Duration( [ 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8, 10u8, 11u8, 12u8 ] .into() - ))?, + )).await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2375,9 +2385,9 @@ Field with name '"b"' is not a member of the map items"#, ]) ); assert_eq!( - serde_json::Value::try_from(Value::Uuid(Uuid::parse_str( + ValueExt::try_from(Value::Uuid(Uuid::parse_str( "936DA01F-9ABD-4D9D-80C7-02AF85C822A8" - )?))?, + )?)).await?, serde_json::Value::String("936da01f-9abd-4d9d-80c7-02af85c822a8".into()) ); From 3b05483ce55e580d00fb744a645b7a684a71dd32 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 10:25:16 +0300 Subject: [PATCH 31/47] Simplify the Stream impl of Reader Now only two tests still fail Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/reader.rs | 58 ++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/avro/src/reader.rs b/avro/src/reader.rs index d6f66774..7442661b 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -223,13 +223,18 @@ mod reader { } async fn read_next(&mut self, read_schema: Option<&Schema>) -> AvroResult> { + dbg!("read_next called", &read_schema); if self.is_empty() { + dbg!("read_next called - empty"); self.read_block_next().await?; + dbg!("read_block_next passed"); if self.is_empty() { + dbg!("read_next called - empty 2"); return Ok(None); } } + dbg!("read_next called 2", self.buf_idx); let mut block_bytes = &self.buf[self.buf_idx..]; let b_original = block_bytes.len(); @@ -240,6 +245,7 @@ mod reader { &mut block_bytes, ) .await?; + dbg!("read_next called 3", &item); let item = match read_schema { Some(schema) => ValueExt::resolve(item, schema).await?, None => item, @@ -377,8 +383,17 @@ mod reader { reader_schema: Option<&'a Schema>, errored: bool, should_resolve_schema: bool, - #[synca::cfg(tokio)] - pending_future: Option> + Send>>>, + } + + impl std::fmt::Debug for Reader<'_, R> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Reader") + // .field("block", &self.block) + .field("reader_schema", &self.reader_schema) + .field("errored", &self.errored) + .field("should_resolve_schema", &self.should_resolve_schema) + .finish() + } } impl<'a, R> Reader<'a, R> @@ -396,8 +411,6 @@ mod reader { reader_schema: None, errored: false, should_resolve_schema: false, - #[synca::cfg(tokio)] - pending_future: None, }; Ok(reader) } @@ -413,8 +426,6 @@ mod reader { reader_schema: Some(schema), errored: false, should_resolve_schema: false, - #[synca::cfg(tokio)] - pending_future: None, }; // Check if the reader and writer schemas disagree. reader.should_resolve_schema = reader.writer_schema() != schema; @@ -436,8 +447,6 @@ mod reader { reader_schema: Some(schema), errored: false, should_resolve_schema: false, - #[synca::cfg(tokio)] - pending_future: None, }; // Check if the reader and writer schemas disagree. reader.should_resolve_schema = reader.writer_schema() != schema; @@ -469,6 +478,7 @@ mod reader { } else { None }; + dbg!("Reader::read_next called 1", &self); self.block.read_next(read_schema).await } @@ -503,30 +513,28 @@ mod reader { type Item = AvroResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + dbg!("poll_next called", &self); // to prevent keep on reading after the first error occurs if self.errored { return Poll::Ready(None); }; - if let Some(mut future) = self.pending_future.take() { - match future.as_mut().poll(cx) { - Poll::Ready(result) => { - // Future completed, return the result - if result.is_err() { - self.errored = true; + let mut future = Box::pin(self.read_next()); + match future.as_mut().poll(cx) { + Poll::Ready(result) => { + dbg!("Ready", &result); + match result { + Ok(opt) => Poll::Ready(opt.map(Ok)), + Err(e) => { + dbg!("Ready 1 - errored"); + // self.errored = true; + Poll::Ready(Some(Err(e))) } - Poll::Ready(Some(result)) - } - Poll::Pending => { - // Restore the pending future - self.pending_future = Some(future); - Poll::Pending } } - } else { - // TODO: mgrigorov: This breaks rustc !!! - // let future = self.read_next(); - // self.pending_future = Some(future); - Poll::Pending + Poll::Pending => { + dbg!("Pending"); + Poll::Pending + } } } } From 4b0b03f8925e4215497a0e06619a314cfd08ea62 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 11:05:20 +0300 Subject: [PATCH 32/47] Fix some Rustdoc tests Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/reader.rs | 9 +++++---- avro/src/types.rs | 37 +++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 7442661b..39e1ec92 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -368,15 +368,14 @@ mod reader { /// ```no_run /// use apache_avro::Reader; /// use std::io::Cursor; - /// async { + /// /// let input = Cursor::new(Vec::::new()); - /// for value in Reader::new(input).await.unwrap() { + /// for value in Reader::new(input).unwrap() { /// match value { /// Ok(v) => println!("{:?}", v), /// Err(e) => println!("Error: {}", e), /// }; /// } - /// } /// ``` pub struct Reader<'a, R> { block: Block<'a, R>, @@ -526,7 +525,8 @@ mod reader { Ok(opt) => Poll::Ready(opt.map(Ok)), Err(e) => { dbg!("Ready 1 - errored"); - // self.errored = true; + drop(future); + self.errored = true; Poll::Ready(Some(Err(e))) } } @@ -908,6 +908,7 @@ mod reader { #[allow(clippy::while_let_on_iterator)] while let Some(value) = reader.next().await { + dbg!(&value); assert!(value.is_err()); } diff --git a/avro/src/types.rs b/avro/src/types.rs index edf70336..c77cf991 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -328,7 +328,6 @@ mod types { pub struct ValueExt; impl ValueExt { - /// Convert Avro values to Json values async fn try_from(value: Value) -> AvroResult { match value { @@ -358,7 +357,7 @@ mod types { result.push(json); } Ok(serde_json::Value::Array(result)) - }, + } Value::Map(items) => { let mut result = serde_json::map::Map::with_capacity(items.len()); for (key, value) in items { @@ -366,7 +365,7 @@ mod types { result.insert(key, v); } Ok(serde_json::Value::Object(result)) - }, + } Value::Record(items) => { let mut result = serde_json::map::Map::with_capacity(items.len()); for (key, value) in items { @@ -374,15 +373,16 @@ mod types { result.insert(key, v); } Ok(serde_json::Value::Object(result)) - }, + } // Value::Record(items) => items // .into_iter() // .map(|(key, value)| Self::try_from(value).await.map(|v| (key, v))).await // .collect::, _>>() // .map(|v| serde_json::Value::Object(v.into_iter().collect())), Value::Date(d) => Ok(serde_json::Value::Number(d.into())), - Value::Decimal(ref d) => >::try_from(d) - .map(|vec| serde_json::Value::Array(vec.into_iter().map(|v| v.into()).collect())), + Value::Decimal(ref d) => >::try_from(d).map(|vec| { + serde_json::Value::Array(vec.into_iter().map(|v| v.into()).collect()) + }), Value::BigDecimal(ref bg) => { let vec1: Vec = serialize_big_decimal(bg).await?; Ok(serde_json::Value::Array( @@ -400,7 +400,9 @@ mod types { Value::Duration(d) => Ok(serde_json::Value::Array( <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), )), - Value::Uuid(uuid) => Ok(serde_json::Value::String(uuid.as_hyphenated().to_string())), + Value::Uuid(uuid) => { + Ok(serde_json::Value::String(uuid.as_hyphenated().to_string())) + } } } /// Validate the value against the given [Schema](../schema/enum.Schema.html). @@ -2264,10 +2266,8 @@ Field with name '"b"' is not a member of the map items"#, serde_json::Value::String("test_enum".into()) ); assert_eq!( - ValueExt::try_from(Value::Union( - 1, - Box::new(Value::String("test_enum".into())) - )).await?, + ValueExt::try_from(Value::Union(1, Box::new(Value::String("test_enum".into())))) + .await?, serde_json::Value::String("test_enum".into()) ); assert_eq!( @@ -2275,7 +2275,8 @@ Field with name '"b"' is not a member of the map items"#, Value::Int(1), Value::Int(2), Value::Int(3) - ])).await?, + ])) + .await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2291,7 +2292,8 @@ Field with name '"b"' is not a member of the map items"#, ] .into_iter() .collect() - )).await?, + )) + .await?, serde_json::Value::Object( vec![ ("v1".to_string(), serde_json::Value::Number(1.into())), @@ -2307,7 +2309,8 @@ Field with name '"b"' is not a member of the map items"#, ("v1".to_string(), Value::Int(1)), ("v2".to_string(), Value::Int(2)), ("v3".to_string(), Value::Int(3)) - ])).await?, + ])) + .await?, serde_json::Value::Object( vec![ ("v1".to_string(), serde_json::Value::Number(1.into())), @@ -2368,7 +2371,8 @@ Field with name '"b"' is not a member of the map items"#, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8, 10u8, 11u8, 12u8 ] .into() - )).await?, + )) + .await?, serde_json::Value::Array(vec![ serde_json::Value::Number(1.into()), serde_json::Value::Number(2.into()), @@ -2387,7 +2391,8 @@ Field with name '"b"' is not a member of the map items"#, assert_eq!( ValueExt::try_from(Value::Uuid(Uuid::parse_str( "936DA01F-9ABD-4D9D-80C7-02AF85C822A8" - )?)).await?, + )?)) + .await?, serde_json::Value::String("936da01f-9abd-4d9d-80c7-02af85c822a8".into()) ); From 45475e7b3bc5069001454e63e43a9247e30c3a2c Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 11:11:13 +0300 Subject: [PATCH 33/47] Use my fork of SyncA from git Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 1 + avro/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2f6750b..16876fe4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1313,6 +1313,7 @@ dependencies = [ [[package]] name = "synca" version = "0.5.3" +source = "git+https://github.com/martin-g/rs_synca/?branch=add-support-for-use-path-replacing#1a98bbafcebc82e4bc111036842f2287cbeec3ab" dependencies = [ "proc-macro2", "quote", diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 0c33604e..3953d421 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -77,8 +77,8 @@ tokio = { version = "1.47.0", features = [ "full" ], optional = true } uuid = { default-features = false, version = "1.18.0", features = ["serde", "std"] } xz2 = { default-features = false, version = "0.1.7", optional = true } zstd = { default-features = false, version = "0.13.3", optional = true } -#synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing" } -synca = { path = "/home/martin/git/rust/rs_synca/synca" } +synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing" } +#synca = { path = "/home/martin/git/rust/rs_synca/synca" } futures = { version = "0.3.31", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] From 26b2e303d40ef342e82c87d460b8db09dc2bf803 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 11:12:41 +0300 Subject: [PATCH 34/47] Update README.md Signed-off-by: Martin Tzvetanov Grigorov --- avro/README.md | 137 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 125 insertions(+), 12 deletions(-) diff --git a/avro/README.md b/avro/README.md index 3dd2c914..1284273f 100644 --- a/avro/README.md +++ b/avro/README.md @@ -127,6 +127,7 @@ Avro schemas are defined in **JSON** format and can just be parsed out of a raw ```rust use apache_avro::Schema; +use apache_avro::schema::sync::SchemaExt; let raw_schema = r#" { @@ -140,7 +141,7 @@ let raw_schema = r#" "#; // if the schema is not valid, this function will return an error -let schema = Schema::parse_str(raw_schema).unwrap(); +let schema = SchemaExt::parse_str(raw_schema).unwrap(); // schemas can be printed for debugging println!("{:?}", schema); @@ -151,6 +152,7 @@ them will be parsed into the corresponding schemas. ```rust use apache_avro::Schema; +use apache_avro::schema::sync::SchemaExt; let raw_schema_1 = r#"{ "name": "A", @@ -170,7 +172,7 @@ let raw_schema_2 = r#"{ }"#; // if the schemas are not valid, this function will return an error -let schemas = Schema::parse_list(&[raw_schema_1, raw_schema_2]).unwrap(); +let schemas = SchemaExt::parse_list(&[raw_schema_1, raw_schema_2]).unwrap(); // schemas can be printed for debugging println!("{:?}", schemas); @@ -203,8 +205,22 @@ Given that the schema we defined above is that of an Avro *Record*, we are going associated type provided by the library to specify the data we want to serialize: ```rust +use apache_avro::Schema; use apache_avro::types::Record; use apache_avro::Writer; +use apache_avro::schema::sync::SchemaExt; + +let raw_schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } +"#; +let schema = SchemaExt::parse_str(raw_schema).unwrap(); // a writer needs a schema and something to write to let mut writer = Writer::new(&schema, Vec::new()); @@ -239,7 +255,10 @@ Given that the schema we defined above is an Avro *Record*, we can directly use deriving `Serialize` to model our data: ```rust +use apache_avro::Schema; +use serde::Serialize; use apache_avro::Writer; +use apache_avro::schema::sync::SchemaExt; #[derive(Debug, Serialize)] struct Test { @@ -247,6 +266,17 @@ struct Test { b: String, } +let raw_schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } +"#; +let schema = SchemaExt::parse_str(raw_schema).unwrap(); // a writer needs a schema and something to write to let mut writer = Writer::new(&schema, Vec::new()); @@ -300,6 +330,19 @@ Avro supports three different compression codecs when encoding data: To specify a codec to use to compress data, just specify it while creating a `Writer`: ```rust use apache_avro::{Codec, DeflateSettings, Schema, Writer}; +use apache_avro::schema::sync::SchemaExt; + +let raw_schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } +"#; +let schema = SchemaExt::parse_str(raw_schema).unwrap(); let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate(DeflateSettings::default())); ``` @@ -311,6 +354,28 @@ codec: ```rust use apache_avro::Reader; +use apache_avro::Schema; +use apache_avro::schema::sync::SchemaExt; +use apache_avro::types::Record; +use apache_avro::Writer; + +let raw_schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } +"#; +let schema = SchemaExt::parse_str(raw_schema).unwrap(); +let mut writer = Writer::new(&schema, Vec::new()); +let mut record = Record::new(writer.schema()).unwrap(); +record.put("a", 27i64); +record.put("b", "foo"); +writer.append(record).unwrap(); +let input = writer.into_inner().unwrap(); // reader creation can fail in case the input to read from is not Avro-compatible or malformed let reader = Reader::new(&input[..]).unwrap(); ``` @@ -320,6 +385,27 @@ the data has been written with, we can just do as the following: ```rust use apache_avro::Schema; use apache_avro::Reader; +use apache_avro::types::Record; +use apache_avro::Writer; +use apache_avro::schema::sync::SchemaExt; + +let writer_raw_schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } +"#; +let writer_schema = SchemaExt::parse_str(writer_raw_schema).unwrap(); +let mut writer = Writer::new(&writer_schema, Vec::new()); +let mut record = Record::new(writer.schema()).unwrap(); +record.put("a", 27i64); +record.put("b", "foo"); +writer.append(record).unwrap(); +let input = writer.into_inner().unwrap(); let reader_raw_schema = r#" { @@ -333,7 +419,7 @@ let reader_raw_schema = r#" } "#; -let reader_schema = Schema::parse_str(reader_raw_schema).unwrap(); +let reader_schema = SchemaExt::parse_str(reader_raw_schema).unwrap(); // reader creation can fail in case the input to read from is not Avro-compatible or malformed let reader = Reader::with_schema(&reader_schema, &input[..]).unwrap(); @@ -357,7 +443,30 @@ interested. We can just read directly instances of `Value` out of the `Reader` iterator: ```rust +use apache_avro::Schema; +use apache_avro::types::Record; +use apache_avro::Writer; use apache_avro::Reader; +use apache_avro::schema::sync::SchemaExt; + +let raw_schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "a", "type": "long", "default": 42}, + {"name": "b", "type": "string"} + ] + } +"#; +let schema = SchemaExt::parse_str(raw_schema).unwrap(); +let schema = SchemaExt::parse_str(raw_schema).unwrap(); +let mut writer = Writer::new(&schema, Vec::new()); +let mut record = Record::new(writer.schema()).unwrap(); +record.put("a", 27i64); +record.put("b", "foo"); +writer.append(record).unwrap(); +let input = writer.into_inner().unwrap(); let reader = Reader::new(&input[..]).unwrap(); // value is a Result of an Avro Value in case the read operation fails @@ -374,6 +483,7 @@ read the data into: ```rust use apache_avro::Reader; +use apache_avro::schema::sync::SchemaExt; use apache_avro::from_value; #[derive(Debug, Deserialize)] @@ -397,6 +507,7 @@ quick reference of the library interface: ```rust use apache_avro::{Codec, DeflateSettings, Reader, Schema, Writer, from_value, types::Record, Error}; +use apache_avro::schema::sync::SchemaExt; use serde::{Deserialize, Serialize}; #[derive(Debug, Deserialize, Serialize)] @@ -417,7 +528,7 @@ fn main() -> Result<(), Error> { } "#; - let schema = Schema::parse_str(raw_schema)?; + let schema = SchemaExt::parse_str(raw_schema)?; println!("{:?}", schema); @@ -464,6 +575,7 @@ use apache_avro::{ types::Record, types::Value, Codec, Days, Decimal, DeflateSettings, Duration, Millis, Months, Reader, Schema, Writer, Error, }; +use apache_avro::schema::sync::SchemaExt; use num_bigint::ToBigInt; fn main() -> Result<(), Error> { @@ -543,7 +655,7 @@ fn main() -> Result<(), Error> { } "#; - let schema = Schema::parse_str(raw_schema)?; + let schema = SchemaExt::parse_str(raw_schema)?; println!("{:?}", schema); @@ -589,6 +701,7 @@ An example of fingerprinting for the supported fingerprints: ```rust use apache_avro::rabin::Rabin; use apache_avro::{Schema, Error}; +use apache_avro::schema::sync::SchemaExt; use md5::Md5; use sha2::Sha256; @@ -603,7 +716,7 @@ fn main() -> Result<(), Error> { ] } "#; - let schema = Schema::parse_str(raw_schema)?; + let schema = SchemaExt::parse_str(raw_schema)?; println!("{}", schema.fingerprint::()); println!("{}", schema.fingerprint::()); println!("{}", schema.fingerprint::()); @@ -652,10 +765,10 @@ Explanation: an int array schema can be read by a long array schema- an int (32bit signed integer) fits into a long (64bit signed integer) ```rust -use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; +use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; -let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); -let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); +let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); +let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_ok()); ``` @@ -665,10 +778,10 @@ Explanation: a long array schema cannot be read by an int array schema- a long (64bit signed integer) does not fit into an int (32bit signed integer) ```rust -use apache_avro::{Schema, schema_compatibility::SchemaCompatibility}; +use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; -let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); -let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); +let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); +let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); assert!(SchemaCompatibility::can_read(&writers_schema, &readers_schema).is_err()); ``` ### Custom names validators From 24406cac6ab3a97d1f6efcf335cb185de72cab03 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 11:18:38 +0300 Subject: [PATCH 35/47] Mark SpecificSingleObjectWriter as sync only ValueExt::try_from(types::Value) -> serde_json::Value seems unused Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/lib.rs | 2 +- avro/src/types.rs | 1 + avro/src/writer.rs | 12 ++++++++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 01ef0d57..3b3e9b48 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -947,7 +947,7 @@ pub use writer::sync::{ #[cfg(feature = "tokio")] pub use writer::tokio::{ GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, - SpecificSingleObjectWriter as AsyncSpecificSingleObjectWriter, Writer as AsyncWriter, + Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, to_avro_datum as async_to_avro_datum, to_avro_datum_schemata as async_to_avro_datum_schemata, }; diff --git a/avro/src/types.rs b/avro/src/types.rs index c77cf991..b851f7d6 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -329,6 +329,7 @@ mod types { impl ValueExt { /// Convert Avro values to Json values + #[allow(dead_code)] // TODO: mgrigorv: Seems unused! Delete ?! async fn try_from(value: Value) -> AvroResult { match value { Value::Null => Ok(serde_json::Value::Null), diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 8e4fa3d8..c5e11839 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -61,12 +61,17 @@ mod writer { error::Details, error::Error, headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, - schema::{AvroSchema, ResolvedOwnedSchema, ResolvedSchema, Schema}, + schema::{ResolvedOwnedSchema, ResolvedSchema, Schema}, types::Value, }; #[synca::cfg(sync)] use serde::Serialize; - use std::{collections::HashMap, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive}; + use std::{collections::HashMap, mem::ManuallyDrop, ops::RangeInclusive}; + + #[synca::cfg(sync)] + use std::marker::PhantomData; + #[synca::cfg(sync)] + use crate::AvroSchema; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; @@ -647,6 +652,7 @@ mod writer { } /// Writer that encodes messages according to the single object encoding v1 spec + #[synca::cfg(sync)] pub struct SpecificSingleObjectWriter where T: AvroSchema, @@ -657,6 +663,7 @@ mod writer { _model: PhantomData, } + #[synca::cfg(sync)] impl SpecificSingleObjectWriter where T: AvroSchema, @@ -672,6 +679,7 @@ mod writer { } } + #[synca::cfg(sync)] impl SpecificSingleObjectWriter where T: AvroSchema + Into, From 59674f42a15f0598621ad67a7d20ec42a7e0ffd8 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 11:21:11 +0300 Subject: [PATCH 36/47] Formatting Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/lib.rs | 3 +-- avro/src/writer.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 3b3e9b48..30af513f 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -946,8 +946,7 @@ pub use writer::sync::{ }; #[cfg(feature = "tokio")] pub use writer::tokio::{ - GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, - Writer as AsyncWriter, + GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, to_avro_datum as async_to_avro_datum, to_avro_datum_schemata as async_to_avro_datum_schemata, }; diff --git a/avro/src/writer.rs b/avro/src/writer.rs index c5e11839..d64f2fc6 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -68,10 +68,10 @@ mod writer { use serde::Serialize; use std::{collections::HashMap, mem::ManuallyDrop, ops::RangeInclusive}; - #[synca::cfg(sync)] - use std::marker::PhantomData; #[synca::cfg(sync)] use crate::AvroSchema; + #[synca::cfg(sync)] + use std::marker::PhantomData; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; From c9e31c5546c634deccb17fb5350a3f8e79fe11b7 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 14 Aug 2025 11:26:27 +0300 Subject: [PATCH 37/47] Mark some more Serde related items as sync only Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/writer.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/avro/src/writer.rs b/avro/src/writer.rs index d64f2fc6..bb59381e 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -884,6 +884,7 @@ mod writer { #[synca::cfg(tokio)] use futures::StreamExt; use pretty_assertions::assert_eq; + #[synca::cfg(sync)] use serde::{Deserialize, Serialize}; #[synca::cfg(sync)] use uuid::Uuid; @@ -1225,6 +1226,7 @@ mod writer { Ok(()) } + #[synca::cfg(sync)] #[derive(Debug, Clone, Deserialize, Serialize)] struct TestSerdeSerialize { a: i64, @@ -1549,6 +1551,7 @@ mod writer { Ok(()) } + #[synca::cfg(sync)] #[derive(Serialize, Clone)] struct TestSingleObjectWriter { a: i64, @@ -1586,6 +1589,7 @@ mod writer { } } + #[synca::cfg(sync)] impl From for Value { fn from(obj: TestSingleObjectWriter) -> Value { Value::Record(vec![ From 036e012eae139bf45173c9f88c4de1e6ff54848e Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 10:16:59 +0300 Subject: [PATCH 38/47] Explicitly list the tokio features we need in Avro Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 23 ----------------------- avro/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16876fe4..9cfeaa81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1235,15 +1235,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signal-hook-registry" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" -dependencies = [ - "libc", -] - [[package]] name = "slab" version = "0.4.9" @@ -1265,16 +1256,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - [[package]] name = "strsim" version = "0.11.1" @@ -1361,13 +1342,9 @@ dependencies = [ "io-uring", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "slab", - "socket2", "tokio-macros", - "windows-sys 0.59.0", ] [[package]] diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 3953d421..456bd2da 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -73,7 +73,7 @@ snap = { default-features = false, version = "1.1.0", optional = true } strum = { default-features = false, version = "0.27.2" } strum_macros = { default-features = false, version = "0.27.2" } thiserror = { default-features = false, version = "2.0.14" } -tokio = { version = "1.47.0", features = [ "full" ], optional = true } +tokio = { version = "1.47.0", default-features = false, features = [ "fs", "io-util", "macros", "rt-multi-thread" ], optional = true } uuid = { default-features = false, version = "1.18.0", features = ["serde", "std"] } xz2 = { default-features = false, version = "0.1.7", optional = true } zstd = { default-features = false, version = "0.13.3", optional = true } From 5cb5527759c0d36f60a00ff8cc580c6a219026b5 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 10:17:19 +0300 Subject: [PATCH 39/47] Use sync Avro APIs for the wasm demo crate Signed-off-by: Martin Tzvetanov Grigorov --- wasm-demo/Cargo.toml | 2 +- wasm-demo/tests/demos.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wasm-demo/Cargo.toml b/wasm-demo/Cargo.toml index 48f83d6a..c79a204e 100644 --- a/wasm-demo/Cargo.toml +++ b/wasm-demo/Cargo.toml @@ -34,7 +34,7 @@ publish = false crate-type = ["cdylib", "rlib"] [dependencies] -apache-avro = { path = "../avro" } +apache-avro = { path = "../avro", default-features = false, features = ["sync"] } serde = { workspace = true } wasm-bindgen = "0.2.97" diff --git a/wasm-demo/tests/demos.rs b/wasm-demo/tests/demos.rs index 050bf8be..c1fc45c1 100644 --- a/wasm-demo/tests/demos.rs +++ b/wasm-demo/tests/demos.rs @@ -20,7 +20,7 @@ use std::io::BufWriter; use wasm_bindgen_test::*; -use apache_avro::{Codec, Reader, Schema, Writer, from_value, to_value, types::Record}; +use apache_avro::{Codec, Reader, SchemaExt, Writer, from_value, to_value, types::Record}; use serde::{Deserialize, Serialize}; wasm_bindgen_test_configure!(run_in_browser); From 88e8d71947b77e6661c44befe031097840546ef9 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 10:47:15 +0300 Subject: [PATCH 40/47] Remove dbg leftovers Signed-off-by: Martin Tzvetanov Grigorov --- avro/src/reader.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 39e1ec92..7e41a920 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -223,18 +223,13 @@ mod reader { } async fn read_next(&mut self, read_schema: Option<&Schema>) -> AvroResult> { - dbg!("read_next called", &read_schema); if self.is_empty() { - dbg!("read_next called - empty"); self.read_block_next().await?; - dbg!("read_block_next passed"); if self.is_empty() { - dbg!("read_next called - empty 2"); return Ok(None); } } - dbg!("read_next called 2", self.buf_idx); let mut block_bytes = &self.buf[self.buf_idx..]; let b_original = block_bytes.len(); @@ -245,7 +240,7 @@ mod reader { &mut block_bytes, ) .await?; - dbg!("read_next called 3", &item); + let item = match read_schema { Some(schema) => ValueExt::resolve(item, schema).await?, None => item, @@ -477,8 +472,6 @@ mod reader { } else { None }; - dbg!("Reader::read_next called 1", &self); - self.block.read_next(read_schema).await } } @@ -512,7 +505,6 @@ mod reader { type Item = AvroResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - dbg!("poll_next called", &self); // to prevent keep on reading after the first error occurs if self.errored { return Poll::Ready(None); @@ -520,11 +512,9 @@ mod reader { let mut future = Box::pin(self.read_next()); match future.as_mut().poll(cx) { Poll::Ready(result) => { - dbg!("Ready", &result); match result { Ok(opt) => Poll::Ready(opt.map(Ok)), Err(e) => { - dbg!("Ready 1 - errored"); drop(future); self.errored = true; Poll::Ready(Some(Err(e))) @@ -532,7 +522,6 @@ mod reader { } } Poll::Pending => { - dbg!("Pending"); Poll::Pending } } @@ -908,7 +897,6 @@ mod reader { #[allow(clippy::while_let_on_iterator)] while let Some(value) = reader.next().await { - dbg!(&value); assert!(value.is_err()); } From 86c220dc9db8f8b268dc65c9f85939fcc1dc3bbb Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 10:47:32 +0300 Subject: [PATCH 41/47] Add an async version of the benchmark example Signed-off-by: Martin Tzvetanov Grigorov --- avro/Cargo.toml | 5 + avro/examples/async_benchmark.rs | 153 +++++++++++++++++++++++++++++++ avro/examples/benchmark.rs | 5 +- 3 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 avro/examples/async_benchmark.rs diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 456bd2da..6ee8f02e 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -108,6 +108,11 @@ name = "benchmark" path = "examples/benchmark.rs" required-features = ["sync"] +[[example]] +name = "async_benchmark" +path = "examples/async_benchmark.rs" +required-features = ["tokio"] + [[example]] name = "generate_interop_data" path = "examples/generate_interop_data.rs" diff --git a/avro/examples/async_benchmark.rs b/avro/examples/async_benchmark.rs new file mode 100644 index 00000000..bc06ff02 --- /dev/null +++ b/avro/examples/async_benchmark.rs @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use apache_avro::{ + AsyncReader as Reader, Schema, schema::tokio::SchemaExt, AsyncWriter as Writer, + types::{Record, Value}, +}; +use std::time::{Duration, Instant}; +use tokio::io::{BufReader, BufWriter}; +use futures::StreamExt; + +fn nanos(duration: Duration) -> u64 { + duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64 +} + +fn seconds(nanos: u64) -> f64 { + (nanos as f64) / 1_000_000_000f64 +} + +/* +fn duration(nanos: u64) -> Duration { + Duration::new(nanos / 1_000_000_000, (nanos % 1_000_000_000) as u32) +} +*/ + +async fn benchmark( + schema: &Schema, + record: &Value, + big_or_small: &str, + count: usize, + runs: usize, +) -> Result<(), Box> { + let mut records = Vec::new(); + for __ in 0..count { + records.push(record.clone()); + } + + let mut durations = Vec::with_capacity(runs); + + let mut bytes = None; + for _ in 0..runs { + let records = records.clone(); + + let start = Instant::now(); + let mut writer = Writer::new(schema, BufWriter::new(Vec::new())); + writer.extend(records).await?; + + let duration = Instant::now().duration_since(start); + durations.push(duration); + + bytes = Some(writer.into_inner().await?.into_inner()); + } + + let total_duration_write = durations.into_iter().fold(0u64, |a, b| a + nanos(b)); + + // println!("Write: {} {} {:?}", count, runs, seconds(total_duration)); + + let bytes = bytes.unwrap(); + + let mut durations = Vec::with_capacity(runs); + + for _ in 0..runs { + let start = Instant::now(); + let mut reader = Reader::with_schema(schema, BufReader::new(&bytes[..])).await?; + + let mut read_records = Vec::with_capacity(count); + while let Some(record) =reader.next().await { + read_records.push(record); + } + + let duration = Instant::now().duration_since(start); + durations.push(duration); + + assert_eq!(count, read_records.len()); + } + + let total_duration_read = durations.into_iter().fold(0u64, |a, b| a + nanos(b)); + + // println!("Read: {} {} {:?}", count, runs, seconds(total_duration)); + let (total_write_secs, total_read_secs) = + (seconds(total_duration_write), seconds(total_duration_read)); + + println!("{count}\t\t{runs}\t\t{big_or_small}\t\t{total_write_secs}\t\t{total_read_secs}"); + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let raw_small_schema = r#" + {"namespace": "test", "type": "record", "name": "Test", "fields": [{"type": {"type": "string"}, "name": "field"}]} + "#; + + let raw_big_schema = r#" + {"namespace": "my.example", "type": "record", "name": "userInfo", "fields": [{"default": "NONE", "type": "string", "name": "username"}, {"default": -1, "type": "int", "name": "age"}, {"default": "NONE", "type": "string", "name": "phone"}, {"default": "NONE", "type": "string", "name": "housenum"}, {"default": {}, "type": {"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}, "name": "address"}]} + "#; + + let small_schema = SchemaExt::parse_str(raw_small_schema).await?; + let big_schema = SchemaExt::parse_str(raw_big_schema).await?; + + println!("{small_schema:?}"); + println!("{big_schema:?}"); + + let mut small_record = Record::new(&small_schema).unwrap(); + small_record.put("field", "foo"); + let small_record = small_record.into(); + + let raw_address_schema = r#"{"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}"#; + let address_schema = SchemaExt::parse_str(raw_address_schema).await?; + let mut address = Record::new(&address_schema).unwrap(); + address.put("street", "street"); + address.put("city", "city"); + address.put("state_prov", "state_prov"); + address.put("country", "country"); + address.put("zip", "zip"); + + let mut big_record = Record::new(&big_schema).unwrap(); + big_record.put("username", "username"); + big_record.put("age", 10i32); + big_record.put("phone", "000000000"); + big_record.put("housenum", "0000"); + big_record.put("address", address); + let big_record = big_record.into(); + + println!(); + println!("Count\t\tRuns\t\tBig/Small\tTotal write secs\tTotal read secs"); + + benchmark(&small_schema, &small_record, "Small", 10_000, 1).await?; + benchmark(&big_schema, &big_record, "Big", 10_000, 1).await?; + + benchmark(&small_schema, &small_record, "Small", 1, 100_000).await?; + benchmark(&small_schema, &small_record, "Small", 100, 1000).await?; + benchmark(&small_schema, &small_record, "Small", 10_000, 10).await?; + + benchmark(&big_schema, &big_record, "Big", 1, 100_000).await?; + benchmark(&big_schema, &big_record, "Big", 100, 1000).await?; + benchmark(&big_schema, &big_record, "Big", 10_000, 10).await?; + + Ok(()) +} diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index b79cbccd..d4961b89 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -19,7 +19,6 @@ use apache_avro::{ Reader, Schema, SchemaExt, Writer, types::{Record, Value}, }; -use apache_avro_test_helper::TestResult; use std::{ io::{BufReader, BufWriter}, time::{Duration, Instant}, @@ -45,7 +44,7 @@ fn benchmark( big_or_small: &str, count: usize, runs: usize, -) -> TestResult { +) -> Result<(), Box> { let mut records = Vec::new(); for __ in 0..count { records.push(record.clone()); @@ -100,7 +99,7 @@ fn benchmark( Ok(()) } -fn main() -> TestResult { +fn main() -> Result<(), Box> { let raw_small_schema = r#" {"namespace": "test", "type": "record", "name": "Test", "fields": [{"type": {"type": "string"}, "name": "field"}]} "#; From 22e3c8b69fc64b9d707437137bb288ae58669589 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 13:33:26 +0300 Subject: [PATCH 42/47] Format async_benchmark.rs Signed-off-by: Martin Tzvetanov Grigorov --- avro/examples/async_benchmark.rs | 7 ++++--- avro/src/reader.rs | 22 +++++++++------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/avro/examples/async_benchmark.rs b/avro/examples/async_benchmark.rs index bc06ff02..a674e3a3 100644 --- a/avro/examples/async_benchmark.rs +++ b/avro/examples/async_benchmark.rs @@ -16,12 +16,13 @@ // under the License. use apache_avro::{ - AsyncReader as Reader, Schema, schema::tokio::SchemaExt, AsyncWriter as Writer, + AsyncReader as Reader, AsyncWriter as Writer, Schema, + schema::tokio::SchemaExt, types::{Record, Value}, }; +use futures::StreamExt; use std::time::{Duration, Instant}; use tokio::io::{BufReader, BufWriter}; -use futures::StreamExt; fn nanos(duration: Duration) -> u64 { duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64 @@ -78,7 +79,7 @@ async fn benchmark( let mut reader = Reader::with_schema(schema, BufReader::new(&bytes[..])).await?; let mut read_records = Vec::with_capacity(count); - while let Some(record) =reader.next().await { + while let Some(record) = reader.next().await { read_records.push(record); } diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 7e41a920..8731a393 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -240,7 +240,7 @@ mod reader { &mut block_bytes, ) .await?; - + let item = match read_schema { Some(schema) => ValueExt::resolve(item, schema).await?, None => item, @@ -511,19 +511,15 @@ mod reader { }; let mut future = Box::pin(self.read_next()); match future.as_mut().poll(cx) { - Poll::Ready(result) => { - match result { - Ok(opt) => Poll::Ready(opt.map(Ok)), - Err(e) => { - drop(future); - self.errored = true; - Poll::Ready(Some(Err(e))) - } + Poll::Ready(result) => match result { + Ok(opt) => Poll::Ready(opt.map(Ok)), + Err(e) => { + drop(future); + self.errored = true; + Poll::Ready(Some(Err(e))) } - } - Poll::Pending => { - Poll::Pending - } + }, + Poll::Pending => Poll::Pending, } } } From b10603b8b3666a68817e4631b21667c4390cd6e3 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 15:02:09 +0300 Subject: [PATCH 43/47] Add async version of tests/schema.rs IT test Signed-off-by: Martin Tzvetanov Grigorov --- avro/Cargo.toml | 5 + avro/tests/async_schema.rs | 2503 ++++++++++++++++++++++++++++++++++++ 2 files changed, 2508 insertions(+) create mode 100644 avro/tests/async_schema.rs diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 6ee8f02e..37fe2cd1 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -183,6 +183,11 @@ name = "schema" path = "tests/schema.rs" required-features = ["sync"] +[[test]] +name = "async_schema" +path = "tests/async_schema.rs" +required-features = ["tokio"] + [[test]] name = "shared" path = "tests/shared.rs" diff --git a/avro/tests/async_schema.rs b/avro/tests/async_schema.rs new file mode 100644 index 00000000..0b5c9f88 --- /dev/null +++ b/avro/tests/async_schema.rs @@ -0,0 +1,2503 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use apache_avro::types::tokio::ValueExt; +use apache_avro::{ + AsyncReader as Reader, AsyncWriter as Writer, Codec, async_from_avro_datum as from_avro_datum, + async_from_value as from_value, async_to_avro_datum as to_avro_datum, + async_to_value as to_value, + error::{Details, Error}, + schema::tokio::SchemaExt, + schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, + types::{Record, Value}, +}; +use apache_avro_test_helper::{ + TestResult, + data::{DOC_EXAMPLES, examples, valid_examples}, + init, +}; +use futures::StreamExt; +use std::{collections::HashMap, io::Cursor}; +use tokio::io::AsyncRead; + +#[tokio::test] +async fn test_correct_recursive_extraction() -> TestResult { + init(); + let raw_outer_schema = r#"{ + "type": "record", + "name": "X", + "fields": [ + { + "name": "y", + "type": { + "type": "record", + "name": "Y", + "fields": [ + { + "name": "Z", + "type": "X" + } + ] + } + } + ] + }"#; + let outer_schema = SchemaExt::parse_str(raw_outer_schema).await?; + if let Schema::Record(RecordSchema { + fields: outer_fields, + .. + }) = outer_schema + { + let inner_schema = &outer_fields[0].schema; + if let Schema::Record(RecordSchema { + fields: inner_fields, + .. + }) = inner_schema + { + if let Schema::Record(RecordSchema { + name: recursive_type, + .. + }) = &inner_fields[0].schema + { + assert_eq!("X", recursive_type.name.as_str()); + } + } else { + panic!("inner schema {inner_schema:?} should have been a record") + } + } else { + panic!("outer schema {outer_schema:?} should have been a record") + } + + Ok(()) +} + +#[tokio::test] +async fn test_parse() -> TestResult { + init(); + for (raw_schema, valid) in examples().iter() { + let schema = SchemaExt::parse_str(raw_schema).await; + if *valid { + assert!( + schema.is_ok(), + "schema {raw_schema} was supposed to be valid; error: {schema:?}", + ) + } else { + assert!( + schema.is_err(), + "schema {raw_schema} was supposed to be invalid" + ) + } + } + Ok(()) +} + +#[tokio::test] +async fn test_3799_parse_reader() -> TestResult { + init(); + for (raw_schema, valid) in examples().iter() { + let mut reader: &mut (dyn AsyncRead + Unpin) = &mut Cursor::new(raw_schema); + let schema = SchemaExt::parse_reader(&mut reader).await; + if *valid { + assert!( + schema.is_ok(), + "schema {raw_schema} was supposed to be valid; error: {schema:?}", + ) + } else { + assert!( + schema.is_err(), + "schema {raw_schema} was supposed to be invalid" + ) + } + } + + // Ensure it works for trait objects too. + for (raw_schema, valid) in examples().iter() { + let reader = &mut Cursor::new(raw_schema); + let schema = SchemaExt::parse_reader(reader).await; + if *valid { + assert!( + schema.is_ok(), + "schema {raw_schema} was supposed to be valid; error: {schema:?}", + ) + } else { + assert!( + schema.is_err(), + "schema {raw_schema} was supposed to be invalid" + ) + } + } + Ok(()) +} + +#[tokio::test] +async fn test_3799_raise_io_error_from_parse_read() -> Result<(), String> { + // 0xDF is invalid for UTF-8. + let mut invalid_data: &mut (dyn AsyncRead + Unpin) = &mut Cursor::new([0xDF]); + let error = SchemaExt::parse_reader(&mut invalid_data) + .await + .unwrap_err() + .into_details(); + + if let Details::ReadSchemaFromReader(e) = error { + assert!( + e.to_string().contains("stream did not contain valid UTF-8"), + "{e}" + ); + Ok(()) + } else { + Err(format!("Expected std::io::Error, got {error:?}")) + } +} + +#[tokio::test] +/// Test that the string generated by an Avro Schema object is, in fact, a valid Avro schema. +async fn test_valid_cast_to_string_after_parse() -> TestResult { + init(); + for (raw_schema, _) in valid_examples().iter() { + let schema = SchemaExt::parse_str(raw_schema).await?; + SchemaExt::parse_str(schema.canonical_form().as_str()).await?; + } + Ok(()) +} + +#[tokio::test] +/// Test that a list of schemas whose definitions do not depend on each other produces the same +/// result as parsing each element of the list individually +async fn test_parse_list_without_cross_deps() -> TestResult { + init(); + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "fixed", + "size": 16 + }"#; + let schema_strs = [schema_str_1, schema_str_2]; + let schemas = SchemaExt::parse_list(schema_strs).await?; + + for schema_str in &schema_strs { + let parsed = SchemaExt::parse_str(schema_str).await?; + assert!(schemas.contains(&parsed)); + } + Ok(()) +} + +#[tokio::test] +/// Test that the parsing of a list of schemas, whose definitions do depend on each other, can +/// perform the necessary schema composition. This should work regardless of the order in which +/// the schemas are input. +/// However, the output order is guaranteed to be the same as the input order. +async fn test_parse_list_with_cross_deps_basic() -> TestResult { + init(); + let schema_a_str = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_b_str = r#"{ + "name": "B", + "type": "record", + "fields": [ + {"name": "field_one", "type": "A"} + ] + }"#; + + let schema_strs_first = [schema_a_str, schema_b_str]; + let schema_strs_second = [schema_b_str, schema_a_str]; + let schemas_first = SchemaExt::parse_list(schema_strs_first).await?; + let schemas_second = SchemaExt::parse_list(schema_strs_second).await?; + + assert_eq!(schemas_first[0], schemas_second[1]); + assert_eq!(schemas_first[1], schemas_second[0]); + Ok(()) +} + +#[tokio::test] +async fn test_parse_list_recursive_type() -> TestResult { + init(); + let schema_str_1 = r#"{ + "name": "A", + "doc": "A's schema", + "type": "record", + "fields": [ + {"name": "a_field_one", "type": "B"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "doc": "B's schema", + "type": "record", + "fields": [ + {"name": "b_field_one", "type": "A"} + ] + }"#; + let schema_strs_first = [schema_str_1, schema_str_2]; + let schema_strs_second = [schema_str_2, schema_str_1]; + let _ = SchemaExt::parse_list(schema_strs_first).await?; + let _ = SchemaExt::parse_list(schema_strs_second).await?; + Ok(()) +} + +#[tokio::test] +/// Test that schema composition resolves namespaces. +async fn test_parse_list_with_cross_deps_and_namespaces() -> TestResult { + init(); + let schema_a_str = r#"{ + "name": "A", + "type": "record", + "namespace": "namespace", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_b_str = r#"{ + "name": "B", + "type": "record", + "fields": [ + {"name": "field_one", "type": "namespace.A"} + ] + }"#; + + let schemas_first = SchemaExt::parse_list([schema_a_str, schema_b_str]).await?; + let schemas_second = SchemaExt::parse_list([schema_b_str, schema_a_str]).await?; + + assert_eq!(schemas_first[0], schemas_second[1]); + assert_eq!(schemas_first[1], schemas_second[0]); + + Ok(()) +} + +#[tokio::test] +/// Test that schema composition fails on namespace errors. +async fn test_parse_list_with_cross_deps_and_namespaces_error() -> TestResult { + init(); + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "namespace": "namespace", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "record", + "fields": [ + {"name": "field_one", "type": "A"} + ] + }"#; + + let schema_strs_first = [schema_str_1, schema_str_2]; + let schema_strs_second = [schema_str_2, schema_str_1]; + let _ = SchemaExt::parse_list(schema_strs_first) + .await + .expect_err("Test failed"); + let _ = SchemaExt::parse_list(schema_strs_second) + .await + .expect_err("Test failed"); + + Ok(()) +} + +#[tokio::test] +// +// test that field's RecordSchema could be referenced by a following field by full name +async fn test_parse_reused_record_schema_by_fullname() -> TestResult { + init(); + let schema_str = r#" + { + "type" : "record", + "name" : "Weather", + "namespace" : "test", + "doc" : "A weather reading.", + "fields" : [ + { + "name" : "station", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + }, + { + "name" : "max_temp", + "type" : { + "type" : "record", + "name" : "Temp", + "namespace": "prefix", + "doc" : "A temperature reading.", + "fields" : [ { + "name" : "temp", + "type" : "long" + } ] + } + }, { + "name" : "min_temp", + "type" : "prefix.Temp" + } + ] + } + "#; + + let schema = SchemaExt::parse_str(schema_str).await; + assert!(schema.is_ok()); + match schema? { + Schema::Record(RecordSchema { + ref name, + aliases: _, + doc: _, + ref fields, + lookup: _, + attributes: _, + }) => { + assert_eq!(name.fullname(None), "test.Weather", "Name does not match!"); + + assert_eq!(fields.len(), 3, "The number of the fields is not correct!"); + + let RecordField { + name, + doc: _, + default: _, + aliases: _, + schema, + order: _, + position: _, + custom_attributes: _, + } = fields.get(2).unwrap(); + + assert_eq!(name, "min_temp"); + + match schema { + Schema::Ref { name } => { + assert_eq!(name.fullname(None), "prefix.Temp", "Name does not match!"); + } + unexpected => unreachable!("Unexpected schema type: {:?}", unexpected), + } + } + unexpected => unreachable!("Unexpected schema type: {:?}", unexpected), + } + + Ok(()) +} + +/// Return all permutations of an input slice +fn permutations(list: &[T]) -> Vec> { + let size = list.len(); + let indices = permutation_indices((0..size).collect()); + let mut perms = Vec::new(); + for perm_map in &indices { + let mut perm = Vec::new(); + for ix in perm_map { + perm.push(&list[*ix]); + } + perms.push(perm) + } + perms +} + +/// Return all permutations of the indices of a vector +fn permutation_indices(indices: Vec) -> Vec> { + let size = indices.len(); + let mut perms: Vec> = Vec::new(); + if size == 1 { + perms.push(indices); + return perms; + } + for index in 0..size { + let (head, tail) = indices.split_at(index); + let (first, rest) = tail.split_at(1); + let mut head = head.to_vec(); + head.extend_from_slice(rest); + for mut sub_index in permutation_indices(head) { + sub_index.insert(0, first[0]); + perms.push(sub_index); + } + } + + perms +} + +#[tokio::test] +/// Test that a type that depends on more than one other type is parsed correctly when all +/// definitions are passed in as a list. This should work regardless of the ordering of the list. +async fn test_parse_list_multiple_dependencies() -> TestResult { + init(); + let schema_a_str = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": ["null", "B", "C"]} + ] + }"#; + let schema_b_str = r#"{ + "name": "B", + "type": "fixed", + "size": 16 + }"#; + let schema_c_str = r#"{ + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + }"#; + + let parsed = SchemaExt::parse_list([schema_a_str, schema_b_str, schema_c_str]).await?; + let schema_strs = vec![schema_a_str, schema_b_str, schema_c_str]; + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemas = SchemaExt::parse_list(&schema_str_perm).await?; + assert_eq!(schemas.len(), 3); + for parsed_schema in &parsed { + assert!(schemas.contains(parsed_schema)); + } + } + Ok(()) +} + +#[tokio::test] +/// Test that a type that is depended on by more than one other type is parsed correctly when all +/// definitions are passed in as a list. This should work regardless of the ordering of the list. +async fn test_parse_list_shared_dependency() -> TestResult { + init(); + let schema_a_str = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "array", "items": "C"}} + ] + }"#; + let schema_b_str = r#"{ + "name": "B", + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "map", "values": "C"}} + ] + }"#; + let schema_c_str = r#"{ + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + }"#; + + let parsed = SchemaExt::parse_list([schema_a_str, schema_b_str, schema_c_str]).await?; + let schema_strs = vec![schema_a_str, schema_b_str, schema_c_str]; + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemas = SchemaExt::parse_list(&schema_str_perm).await?; + assert_eq!(schemas.len(), 3); + for parsed_schema in &parsed { + assert!(schemas.contains(parsed_schema)); + } + } + Ok(()) +} + +#[tokio::test] +/// Test that trying to parse two schemas with the same fullname returns an Error +async fn test_name_collision_error() -> TestResult { + init(); + let schema_str_1 = r#"{ + "name": "foo.A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "double"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "A", + "type": "record", + "namespace": "foo", + "fields": [ + {"name": "field_two", "type": "string"} + ] + }"#; + + let _ = SchemaExt::parse_list([schema_str_1, schema_str_2]) + .await + .expect_err("Test failed"); + Ok(()) +} + +#[tokio::test] +/// Test that having the same name but different fullnames does not return an error +async fn test_namespace_prevents_collisions() -> TestResult { + init(); + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "double"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "A", + "type": "record", + "namespace": "foo", + "fields": [ + {"name": "field_two", "type": "string"} + ] + }"#; + + let parsed = SchemaExt::parse_list([schema_str_1, schema_str_2]).await?; + let parsed_1 = SchemaExt::parse_str(schema_str_1).await?; + let parsed_2 = SchemaExt::parse_str(schema_str_2).await?; + assert_eq!(parsed, vec!(parsed_1, parsed_2)); + Ok(()) +} + +// The fullname is determined in one of the following ways: +// * A name and namespace are both specified. For example, +// one might use "name": "X", "namespace": "org.foo" +// to indicate the fullname "org.foo.X". +// * A fullname is specified. If the name specified contains +// a dot, then it is assumed to be a fullname, and any +// namespace also specified is ignored. For example, +// use "name": "org.foo.X" to indicate the +// fullname "org.foo.X". +// * A name only is specified, i.e., a name that contains no +// dots. In this case the namespace is taken from the most +// tightly enclosing schema or protocol. For example, +// if "name": "X" is specified, and this occurs +// within a field of the record definition /// of "org.foo.Y", then the fullname is "org.foo.X". + +// References to previously defined names are as in the latter +// two cases above: if they contain a dot they are a fullname, if +// they do not contain a dot, the namespace is the namespace of +// the enclosing definition. + +// Primitive type names have no namespace and their names may +// not be defined in any namespace. A schema may only contain +// multiple definitions of a fullname if the definitions are +// equivalent. + +#[tokio::test] +async fn test_fullname_name_and_namespace_specified() -> TestResult { + init(); + let name: Name = + serde_json::from_str(r#"{"name": "a", "namespace": "o.a.h", "aliases": null}"#)?; + let fullname = name.fullname(None); + assert_eq!("o.a.h.a", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_fullname_fullname_and_namespace_specified() -> TestResult { + init(); + let name: Name = serde_json::from_str(r#"{"name": "a.b.c.d", "namespace": "o.a.h"}"#)?; + assert_eq!(&name.name, "d"); + assert_eq!(name.namespace, Some("a.b.c".to_owned())); + let fullname = name.fullname(None); + assert_eq!("a.b.c.d", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_fullname_name_and_default_namespace_specified() -> TestResult { + init(); + let name: Name = serde_json::from_str(r#"{"name": "a", "namespace": null}"#)?; + assert_eq!(&name.name, "a"); + assert_eq!(name.namespace, None); + let fullname = name.fullname(Some("b.c.d".into())); + assert_eq!("b.c.d.a", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_fullname_fullname_and_default_namespace_specified() -> TestResult { + init(); + let name: Name = serde_json::from_str(r#"{"name": "a.b.c.d", "namespace": null}"#)?; + assert_eq!(&name.name, "d"); + assert_eq!(name.namespace, Some("a.b.c".to_owned())); + let fullname = name.fullname(Some("o.a.h".into())); + assert_eq!("a.b.c.d", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_avro_3452_parsing_name_without_namespace() -> TestResult { + init(); + let name: Name = serde_json::from_str(r#"{"name": "a.b.c.d"}"#)?; + assert_eq!(&name.name, "d"); + assert_eq!(name.namespace, Some("a.b.c".to_owned())); + let fullname = name.fullname(None); + assert_eq!("a.b.c.d", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_avro_3452_parsing_name_with_leading_dot_without_namespace() -> TestResult { + init(); + let name: Name = serde_json::from_str(r#"{"name": ".a"}"#)?; + assert_eq!(&name.name, "a"); + assert_eq!(name.namespace, None); + assert_eq!("a", name.fullname(None)); + Ok(()) +} + +#[tokio::test] +async fn test_avro_3452_parse_json_without_name_field() -> TestResult { + init(); + let result: serde_json::error::Result = serde_json::from_str(r#"{"unknown": "a"}"#); + assert!(&result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "No `name` field"); + Ok(()) +} + +#[tokio::test] +async fn test_fullname_fullname_namespace_and_default_namespace_specified() -> TestResult { + init(); + let name: Name = + serde_json::from_str(r#"{"name": "a.b.c.d", "namespace": "o.a.a", "aliases": null}"#)?; + assert_eq!(&name.name, "d"); + assert_eq!(name.namespace, Some("a.b.c".to_owned())); + let fullname = name.fullname(Some("o.a.h".into())); + assert_eq!("a.b.c.d", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_fullname_name_namespace_and_default_namespace_specified() -> TestResult { + init(); + let name: Name = + serde_json::from_str(r#"{"name": "a", "namespace": "o.a.a", "aliases": null}"#)?; + assert_eq!(&name.name, "a"); + assert_eq!(name.namespace, Some("o.a.a".to_owned())); + let fullname = name.fullname(Some("o.a.h".into())); + assert_eq!("o.a.a.a", fullname); + Ok(()) +} + +#[tokio::test] +async fn test_doc_attributes() -> TestResult { + init(); + fn assert_doc(schema: &Schema) { + match schema { + Schema::Enum(EnumSchema { doc, .. }) => assert!(doc.is_some()), + Schema::Record(RecordSchema { doc, .. }) => assert!(doc.is_some()), + Schema::Fixed(FixedSchema { doc, .. }) => assert!(doc.is_some()), + Schema::String => (), + _ => unreachable!("Unexpected schema type: {:?}", schema), + } + } + + for (raw_schema, _) in DOC_EXAMPLES.iter() { + let original_schema = SchemaExt::parse_str(raw_schema).await?; + assert_doc(&original_schema); + if let Schema::Record(RecordSchema { fields, .. }) = original_schema { + for f in fields { + assert_doc(&f.schema) + } + } + } + Ok(()) +} + +/* +TODO: (#94) add support for user-defined attributes and uncomment (may need some tweaks to compile) +#[tokio::test] +async fn test_other_attributes() { + fn assert_attribute_type(attribute: (String, serde_json::Value)) { + match attribute.1.as_ref() { + "cp_boolean" => assert!(attribute.2.is_bool()), + "cp_int" => assert!(attribute.2.is_i64()), + "cp_object" => assert!(attribute.2.is_object()), + "cp_float" => assert!(attribute.2.is_f64()), + "cp_array" => assert!(attribute.2.is_array()), + } + } + + for (raw_schema, _) in OTHER_ATTRIBUTES_EXAMPLES.iter() { + let schema = SchemaExt::parse_str(raw_schema).await?; + // all inputs have at least some user-defined attributes + assert!(schema.other_attributes.is_some()); + for prop in schema.other_attributes?.iter() { + assert_attribute_type(prop); + } + if let Schema::Record { fields, .. } = schema { + for f in fields { + // all fields in the record have at least some user-defined attributes + assert!(f.schema.other_attributes.is_some()); + for prop in f.schema.other_attributes?.iter() { + assert_attribute_type(prop); + } + } + } + } +} +*/ + +#[tokio::test] +async fn test_root_error_is_not_swallowed_on_parse_error() -> Result<(), String> { + init(); + let raw_schema = r#"/not/a/real/file"#; + let error = SchemaExt::parse_str(raw_schema) + .await + .unwrap_err() + .into_details(); + + if let Details::ParseSchemaJson(e) = error { + assert!( + e.to_string().contains("expected value at line 1 column 1"), + "{}", + e + ); + Ok(()) + } else { + Err(format!("Expected serde_json::error::Error, got {error:?}")) + } +} + +// AVRO-3302 +#[tokio::test] +async fn test_record_schema_with_cyclic_references() -> TestResult { + init(); + let schema = SchemaExt::parse_str( + r#" + { + "type": "record", + "name": "test", + "fields": [{ + "name": "recordField", + "type": { + "type": "record", + "name": "Node", + "fields": [ + {"name": "label", "type": "string"}, + {"name": "children", "type": {"type": "array", "items": "Node"}} + ] + } + }] + } + "#, + ) + .await?; + + let mut datum = Record::new(&schema).unwrap(); + datum.put( + "recordField", + Value::Record(vec![ + ("label".into(), Value::String("level_1".into())), + ( + "children".into(), + Value::Array(vec![Value::Record(vec![ + ("label".into(), Value::String("level_2".into())), + ( + "children".into(), + Value::Array(vec![Value::Record(vec![ + ("label".into(), Value::String("level_3".into())), + ( + "children".into(), + Value::Array(vec![Value::Record(vec![ + ("label".into(), Value::String("level_4".into())), + ("children".into(), Value::Array(vec![])), + ])]), + ), + ])]), + ), + ])]), + ), + ]), + ); + + let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Null); + if let Err(err) = writer.append(datum).await { + panic!("An error occurred while writing datum: {err:?}") + } + let bytes = writer.into_inner().await?; + assert_eq!(316, bytes.len()); + + let data = &mut bytes.as_slice(); + let reader = Reader::new(data).await; + match reader { + Ok(mut reader) => match reader.next().await { + Some(value) => log::debug!("{:?}", value?), + None => panic!("No value was read!"), + }, + Err(err) => panic!("An error occurred while reading datum: {err:?}"), + } + Ok(()) +} + +/* +// TODO: (#93) add support for logical type and attributes and uncomment (may need some tweaks to compile) +#[tokio::test] +async fn test_decimal_valid_type_attributes() { + init(); + let fixed_decimal = SchemaExt::parse_str(DECIMAL_LOGICAL_TYPE_ATTRIBUTES[0]).await?; + assert_eq!(4, fixed_decimal.get_attribute("precision")); + assert_eq!(2, fixed_decimal.get_attribute("scale")); + assert_eq!(2, fixed_decimal.get_attribute("size")); + + let bytes_decimal = SchemaExt::parse_str(DECIMAL_LOGICAL_TYPE_ATTRIBUTES[1]).await?; + assert_eq!(4, bytes_decimal.get_attribute("precision")); + assert_eq!(0, bytes_decimal.get_attribute("scale")); +} +*/ + +// https://github.com/flavray/avro-rs/issues/47 +#[tokio::test] +async fn avro_old_issue_47() -> TestResult { + init(); + let schema_str = r#" + { + "type": "record", + "name": "my_record", + "fields": [ + {"name": "a", "type": "long"}, + {"name": "b", "type": "string"} + ] + }"#; + let schema = SchemaExt::parse_str(schema_str).await?; + + use serde::{Deserialize, Serialize}; + + #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] + pub struct MyRecord { + b: String, + a: i64, + } + + let record = MyRecord { + b: "hello".to_string(), + a: 1, + }; + + let ser_value = to_value(record.clone())?; + let serialized_bytes = to_avro_datum(&schema, ser_value).await?; + + let de_value = &from_avro_datum(&schema, &mut &*serialized_bytes, None).await?; + let deserialized_record = from_value::(de_value)?; + + assert_eq!(record, deserialized_record); + Ok(()) +} + +#[tokio::test] +async fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_type() +-> TestResult { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] + pub struct BarUseParent { + #[serde(rename = "barUse")] + pub bar_use: Bar, + } + + #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Deserialize, Serialize)] + pub enum Bar { + #[serde(rename = "bar0")] + Bar0, + #[serde(rename = "bar1")] + Bar1, + #[serde(rename = "bar2")] + Bar2, + } + + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] + pub struct Foo { + #[serde(rename = "barInit")] + pub bar_init: Bar, + #[serde(rename = "barUseParent")] + pub bar_use_parent: Option, + } + + let writer_schema = r#"{ + "type": "record", + "name": "Foo", + "namespace": "name.space", + "fields": + [ + { + "name": "barInit", + "type": + { + "type": "enum", + "name": "Bar", + "symbols": + [ + "bar0", + "bar1", + "bar2" + ] + } + }, + { + "name": "barUseParent", + "type": [ + "null", + { + "type": "record", + "name": "BarUseParent", + "fields": [ + { + "name": "barUse", + "type": "Bar" + } + ] + } + ] + } + ] + }"#; + + let reader_schema = r#"{ + "type": "record", + "name": "Foo", + "namespace": "name.space", + "fields": + [ + { + "name": "barInit", + "type": + { + "type": "enum", + "name": "Bar", + "symbols": + [ + "bar0", + "bar1" + ] + } + }, + { + "name": "barUseParent", + "type": [ + "null", + { + "type": "record", + "name": "BarUseParent", + "fields": [ + { + "name": "barUse", + "type": "Bar" + } + ] + } + ] + } + ] + }"#; + + let writer_schema = SchemaExt::parse_str(writer_schema).await?; + let foo1 = Foo { + bar_init: Bar::Bar0, + bar_use_parent: Some(BarUseParent { bar_use: Bar::Bar1 }), + }; + let avro_value = crate::to_value(foo1)?; + assert!( + ValueExt::validate(&avro_value, &writer_schema).await, + "value is valid for schema", + ); + let datum = to_avro_datum(&writer_schema, avro_value).await?; + let mut x = &datum[..]; + let reader_schema = SchemaExt::parse_str(reader_schema).await?; + let deser_value = from_avro_datum(&writer_schema, &mut x, Some(&reader_schema)).await?; + match deser_value { + Value::Record(fields) => { + assert_eq!(fields.len(), 2); + } + _ => panic!("Expected Value::Record"), + } + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3847_union_field_with_default_value_of_ref() -> TestResult { + // Test for reference to Record + let writer_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + }, { + "name": "f2", + "type": ["record2", "int"], + "default": { + "f1_1": 100 + } + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ( + "f1".to_string(), + Value::Record(vec![("f1_1".to_string(), 10.into())]), + ), + ( + "f2".to_string(), + Value::Union( + 0, + Box::new(Value::Record(vec![("f1_1".to_string(), 100.into())])), + ), + ), + ]); + + assert_eq!(expected, result[0]); + + // Test for reference to Enum + let writer_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "enum1", + "type": "enum", + "symbols": ["a", "b"] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Enum(1, "b".to_string())); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "enum1", + "type": "enum", + "symbols": ["a", "b"] + } + }, { + "name": "f2", + "type": ["enum1", "int"], + "default": "a" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Enum(1, "b".to_string())), + ( + "f2".to_string(), + Value::Union(0, Box::new(Value::Enum(0, "a".to_string()))), + ), + ]); + + assert_eq!(expected, result[0]); + + // Test for reference to Fixed + let writer_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "fixed1", + "type": "fixed", + "size": 3 + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Fixed(3, vec![0, 1, 2])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "fixed1", + "type": "fixed", + "size": 3 + } + }, { + "name": "f2", + "type": ["fixed1", "int"], + "default": "abc" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Fixed(3, vec![0, 1, 2])), + ( + "f2".to_string(), + Value::Union(0, Box::new(Value::Fixed(3, vec![b'a', b'b', b'c']))), + ), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3847_union_field_with_default_value_of_ref_with_namespace() -> TestResult { + // Test for reference to Record + let writer_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + }, { + "name": "f2", + "type": ["ns.record2", "int"], + "default": { + "f1_1": 100 + } + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ( + "f1".to_string(), + Value::Record(vec![("f1_1".to_string(), 10.into())]), + ), + ( + "f2".to_string(), + Value::Union( + 0, + Box::new(Value::Record(vec![("f1_1".to_string(), 100.into())])), + ), + ), + ]); + + assert_eq!(expected, result[0]); + + // Test for reference to Enum + let writer_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "enum1", + "namespace": "ns", + "type": "enum", + "symbols": ["a", "b"] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Enum(1, "b".to_string())); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "enum1", + "namespace": "ns", + "type": "enum", + "symbols": ["a", "b"] + } + }, { + "name": "f2", + "type": ["ns.enum1", "int"], + "default": "a" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Enum(1, "b".to_string())), + ( + "f2".to_string(), + Value::Union(0, Box::new(Value::Enum(0, "a".to_string()))), + ), + ]); + + assert_eq!(expected, result[0]); + + // Test for reference to Fixed + let writer_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "fixed1", + "namespace": "ns", + "type": "fixed", + "size": 3 + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Fixed(3, vec![0, 1, 2])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "fixed1", + "namespace": "ns", + "type": "fixed", + "size": 3 + } + }, { + "name": "f2", + "type": ["ns.fixed1", "int"], + "default": "abc" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Fixed(3, vec![0, 1, 2])), + ( + "f2".to_string(), + Value::Union(0, Box::new(Value::Fixed(3, vec![b'a', b'b', b'c']))), + ), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3847_union_field_with_default_value_of_ref_with_enclosing_namespace() +-> TestResult { + // Test for reference to Record + let writer_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + }, { + "name": "f2", + "type": ["ns.record2", "int"], + "default": { + "f1_1": 100 + } + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ( + "f1".to_string(), + Value::Record(vec![("f1_1".to_string(), 10.into())]), + ), + ( + "f2".to_string(), + Value::Union( + 0, + Box::new(Value::Record(vec![("f1_1".to_string(), 100.into())])), + ), + ), + ]); + + assert_eq!(expected, result[0]); + + // Test for reference to Enum + let writer_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "enum1", + "type": "enum", + "symbols": ["a", "b"] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Enum(1, "b".to_string())); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "enum1", + "type": "enum", + "symbols": ["a", "b"] + } + }, { + "name": "f2", + "type": ["ns.enum1", "int"], + "default": "a" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Enum(1, "b".to_string())), + ( + "f2".to_string(), + Value::Union(0, Box::new(Value::Enum(0, "a".to_string()))), + ), + ]); + + assert_eq!(expected, result[0]); + + // Test for reference to Fixed + let writer_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "fixed1", + "type": "fixed", + "size": 3 + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Fixed(3, vec![0, 1, 2])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "fixed1", + "type": "fixed", + "size": 3 + } + }, { + "name": "f2", + "type": ["ns.fixed1", "int"], + "default": "abc" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Fixed(3, vec![0, 1, 2])), + ( + "f2".to_string(), + Value::Union(0, Box::new(Value::Fixed(3, vec![b'a', b'b', b'c']))), + ), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +async fn write_schema_for_default_value_test() -> apache_avro::AvroResult> { + let writer_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()) + .ok_or("Expected Some(Record), but got None") + .unwrap(); + record.put("f1", 10); + writer.append(record).await?; + + writer.into_inner().await +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_simple_record_field() -> TestResult { + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + }, { + "name": "f2", + "type": "int", + "default": 20 + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = write_schema_for_default_value_test().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Int(10)), + ("f2".to_string(), Value::Int(20)), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_nested_record_field() -> TestResult { + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + }, { + "name": "f2", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + }, + "default": { + "f1_1": 100 + } + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = write_schema_for_default_value_test().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Int(10)), + ( + "f2".to_string(), + Value::Record(vec![("f1_1".to_string(), 100.into())]), + ), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_enum_record_field() -> TestResult { + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + }, { + "name": "f2", + "type": { + "name": "enum1", + "type": "enum", + "symbols": ["a", "b", "c"] + }, + "default": "a" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = write_schema_for_default_value_test().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Int(10)), + ("f2".to_string(), Value::Enum(0, "a".to_string())), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_fixed_record_field() -> TestResult { + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + }, { + "name": "f2", + "type": { + "name": "fixed1", + "type": "fixed", + "size": 3 + }, + "default": "abc" + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = write_schema_for_default_value_test().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Int(10)), + ("f2".to_string(), Value::Fixed(3, vec![b'a', b'b', b'c'])), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_array_record_field() -> TestResult { + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + }, { + "name": "f2", + "type": "array", + "items": "int", + "default": [1, 2, 3] + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = write_schema_for_default_value_test().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Int(10)), + ( + "f2".to_string(), + Value::Array(vec![1.into(), 2.into(), 3.into()]), + ), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_map_record_field() -> TestResult { + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": "int" + }, { + "name": "f2", + "type": "map", + "values": "string", + "default": { "a": "A", "b": "B", "c": "C" } + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = write_schema_for_default_value_test().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let map = HashMap::from_iter([ + ("a".to_string(), "A".into()), + ("b".to_string(), "B".into()), + ("c".to_string(), "C".into()), + ]); + let expected = Value::Record(vec![ + ("f1".to_string(), Value::Int(10)), + ("f2".to_string(), Value::Map(map)), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_ref_record_field() -> TestResult { + let writer_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + } + ] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + let mut record = Record::new(writer.schema()).ok_or("Expected Some(Record), but got None")?; + record.put("f1", Value::Record(vec![("f1_1".to_string(), 10.into())])); + writer.append(record).await?; + + let reader_schema_str = r#" + { + "name": "record1", + "namespace": "ns", + "type": "record", + "fields": [ + { + "name": "f1", + "type": { + "name": "record2", + "type": "record", + "fields": [ + { + "name": "f1_1", + "type": "int" + } + ] + } + }, { + "name": "f2", + "type": "ns.record2", + "default": { "f1_1": 100 } + } + ] + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Record(vec![ + ( + "f1".to_string(), + Value::Record(vec![("f1_1".to_string(), 10.into())]), + ), + ( + "f2".to_string(), + Value::Record(vec![("f1_1".to_string(), 100.into())]), + ), + ]); + + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn test_avro_3851_read_default_value_for_enum() -> TestResult { + let writer_schema_str = r#" + { + "name": "enum1", + "namespace": "ns", + "type": "enum", + "symbols": ["a", "b", "c"] + } + "#; + let writer_schema = SchemaExt::parse_str(writer_schema_str).await?; + let mut writer = Writer::new(&writer_schema, Vec::new()); + writer.append("c").await?; + + let reader_schema_str = r#" + { + "name": "enum1", + "namespace": "ns", + "type": "enum", + "symbols": ["a", "b"], + "default": "a" + } + "#; + let reader_schema = SchemaExt::parse_str(reader_schema_str).await?; + let input = writer.into_inner().await?; + let mut reader = Reader::with_schema(&reader_schema, &input[..]).await?; + let mut result = Vec::new(); + while let Some(value) = reader.next().await { + match value { + Ok(val) => result.push(val), + Err(e) => panic!("Error reading value: {:?}", e), + } + } + + assert_eq!(1, result.len()); + + let expected = Value::Enum(0, "a".to_string()); + assert_eq!(expected, result[0]); + + Ok(()) +} + +#[tokio::test] +async fn avro_rs_66_test_independent_canonical_form_primitives() -> TestResult { + init(); + let record_primitive = r#"{ + "name": "Rec", + "namespace": "ns", + "type": "record", + "fields": [ + {"name": "v", "type": "int"} + ] + }"#; + + let enum_primitive = r#"{ + "name": "En", + "type": "enum", + "symbols": [ "bar0", "bar1" ] + }"#; + + let fixed_primitive = r#"{ + "name": "Fix", + "type": "fixed", + "size": 4 + }"#; + + let record_with_dependencies = r#"{ + "name": "RecWithDeps", + "type": "record", + "fields": [ + {"name": "v1", "type": "ns.Rec"}, + {"name": "v2", "type": "En"}, + {"name": "v3", "type": "Fix"}, + {"name": "v4", "type": "ns.Rec"}, + {"name": "v5", "type": "En"}, + {"name": "v6", "type": "Fix"} + ] + }"#; + + let record_with_no_dependencies = r#"{ + "name": "RecWithDeps", + "type": "record", + "fields": [ + { + "name": "v1", "type": { + "name": "Rec", + "namespace": "ns", + "type": "record", + "fields": [ + {"name": "v", "type": "int"} + ] + } + }, + { + "name": "v2", "type": { + "name": "En", + "type": "enum", + "symbols": [ "bar0", "bar1" ] + } + }, + {"name": "v3", "type": + { + "name": "Fix", + "type": "fixed", + "size": 4 + } + }, + {"name": "v4", "type": "ns.Rec"}, + {"name": "v5", "type": "En"}, + {"name": "v6", "type": "Fix"} + ] + }"#; + + let independent_schema = SchemaExt::parse_str(record_with_no_dependencies).await?; + let schema_strs = [ + fixed_primitive, + enum_primitive, + record_primitive, + record_with_dependencies, + ]; + + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemata = SchemaExt::parse_list(&schema_str_perm).await?; + assert_eq!(schemata.len(), schema_strs.len()); + let test_schema = schemata + .iter() + .find(|a| a.name().unwrap().to_string() == *"RecWithDeps") + .unwrap(); + + assert_eq!( + independent_schema.independent_canonical_form(&schemata)?, + independent_schema.canonical_form() + ); + + assert_eq!( + independent_schema.canonical_form(), + test_schema.independent_canonical_form(&schemata)? + ); + } + Ok(()) +} + +#[tokio::test] +async fn avro_rs_66_test_independent_canonical_form_usages() -> TestResult { + init(); + let record_primitive = r#"{ + "name": "Rec", + "namespace": "ns", + "type": "record", + "fields": [ + {"name": "v", "type": "int"} + ] + }"#; + + let record_usage = r#"{ + "name": "RecUsage", + "type": "record", + "fields": [ + {"name": "v1", "type": "ns.Rec"}, + {"name": "v2", "type": "ns.Rec"} + ] + }"#; + let record_usage_independent = r#"{ + "name": "RecUsage", + "type": "record", + "fields": [ + {"name": "v1", "type": { + "name": "ns.Rec", "type": "record","fields": [{"name": "v", "type": "int"}]} + }, + {"name": "v2", "type": "ns.Rec"} + ] + }"#; + + let array_usage = r#"{ + "name": "ArrayUsage", + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "array", "items": "ns.Rec"}}, + {"name": "field_two", "type": {"type": "array", "items": "ns.Rec"}} + ] + }"#; + let array_usage_independent = r#"{ + "name": "ArrayUsage", + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "array", "items": { + "name": "ns.Rec", "type": "record","fields": [{"name": "v", "type": "int"}]} + }}, + {"name": "field_two", "type": {"type": "array", "items": "ns.Rec"}} + ] + }"#; + + let union_usage = r#"{ + "name": "UnionUsage", + "type": "record", + "fields": [ + {"name": "field_one", "type": ["null", "ns.Rec"]}, + {"name": "field_two", "type": ["null", "ns.Rec"]} + ] + }"#; + let union_usage_independent = r#"{ + "name": "UnionUsage", + "type": "record", + "fields": [ + {"name": "field_one", "type": ["null", { + "name": "ns.Rec", "type": "record","fields": [{"name": "v", "type": "int"}]} + ]}, + {"name": "field_two", "type": ["null", "ns.Rec"]} + ] + }"#; + + let map_usage = r#"{ + "name": "MapUsage", + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "map", "values": "ns.Rec"}}, + {"name": "field_two", "type": {"type": "map", "values": "ns.Rec"}} + ] + }"#; + let map_usage_independent = r#"{ + "name": "MapUsage", + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "map", "values": { + "name": "ns.Rec", "type": "record","fields": [{"name": "v", "type": "int"}]} + }}, + {"name": "field_two", "type": {"type": "map", "values": "ns.Rec"}} + ] + }"#; + + let schema_strs = [ + record_primitive, + record_usage, + array_usage, + map_usage, + union_usage, + ]; + + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemata = SchemaExt::parse_list(&schema_str_perm).await?; + for schema in &schemata { + match schema.name().unwrap().to_string().as_str() { + "RecUsage" => { + assert_eq!( + schema.independent_canonical_form(&schemata)?, + SchemaExt::parse_str(record_usage_independent) + .await? + .canonical_form() + ); + } + "ArrayUsage" => { + assert_eq!( + schema.independent_canonical_form(&schemata)?, + SchemaExt::parse_str(array_usage_independent) + .await? + .canonical_form() + ); + } + "UnionUsage" => { + assert_eq!( + schema.independent_canonical_form(&schemata)?, + SchemaExt::parse_str(union_usage_independent) + .await? + .canonical_form() + ); + } + "MapUsage" => { + assert_eq!( + schema.independent_canonical_form(&schemata)?, + SchemaExt::parse_str(map_usage_independent) + .await? + .canonical_form() + ); + } + "ns.Rec" => { + assert_eq!( + schema.independent_canonical_form(&schemata)?, + schema.canonical_form() + ); + } + other => unreachable!("Unknown schema name: {}", other), + } + } + } + Ok(()) +} + +#[tokio::test] +async fn avro_rs_66_test_independent_canonical_form_deep_recursion() -> TestResult { + init(); + let record_primitive = r#"{ + "name": "Rec", + "namespace": "ns", + "type": "record", + "fields": [ + {"name": "v", "type": "int"} + ] + }"#; + + let record_usage = r#"{ + "name": "RecUsage", + "type": "record", + "fields": [ + {"name": "v1", "type": "ns.Rec"}, + {"name": "v2", "type": "ns.Rec"} + ] + }"#; + + let record_usage_usage = r#"{ + "name": "RecUsageUsage", + "type": "record", + "fields": [ + {"name": "r1", "type": "RecUsage"}, + {"name": "r2", "type": "RecUsage"} + ] + }"#; + + let record_usage_usage_independent = r#"{ + "name": "RecUsageUsage", + "type": "record", + "fields": [ + {"name": "r1", "type": { + "name": "RecUsage", + "type": "record", + "fields": [ + { + "name": "v1", "type": { + "name": "ns.Rec", "type": "record","fields": [{"name": "v", "type": "int"}] + } + }, + {"name": "v2", "type": "ns.Rec"} + ] + }}, + {"name": "r2", "type": "RecUsage"} + ] + + }"#; + + let schema_strs = [record_primitive, record_usage, record_usage_usage]; + + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemata = SchemaExt::parse_list(&schema_str_perm).await?; + let ruu = schemata + .iter() + .find(|s| s.name().unwrap().to_string().as_str() == "RecUsageUsage") + .unwrap(); + assert_eq!( + ruu.independent_canonical_form(&schemata)?, + SchemaExt::parse_str(record_usage_usage_independent) + .await? + .canonical_form() + ); + } + Ok(()) +} + +#[tokio::test] +async fn avro_rs_66_test_independent_canonical_form_missing_ref() -> TestResult { + init(); + let record_primitive = r#"{ + "name": "Rec", + "namespace": "ns", + "type": "record", + "fields": [ + {"name": "v", "type": "int"} + ] + }"#; + + let record_usage = r#"{ + "name": "RecUsage", + "type": "record", + "fields": [ + {"name": "v1", "type": "ns.Rec"} + ] + }"#; + + let schema_strs = [record_primitive, record_usage]; + let schemata = SchemaExt::parse_list(schema_strs).await?; + assert!(matches!( + schemata[1] + .independent_canonical_form(&Vec::with_capacity(0)) + .map_err(Error::into_details), //NOTE - we're passing in an empty schemata + Err(Details::SchemaResolutionError(..)) + )); + Ok(()) +} + +#[tokio::test] +async fn avro_rs_181_single_null_record() -> TestResult { + let mut buff = Cursor::new(Vec::new()); + let schema = SchemaExt::parse_str(r#""null""#).await?; + let mut writer = Writer::new(&schema, &mut buff); + writer.append(serde_json::Value::Null).await?; + writer.into_inner().await?; + buff.set_position(0); + + let mut reader = Reader::new(buff).await?; + while let Some(value) = reader.next().await { + assert_eq!(Value::Null, value?); + } + + Ok(()) +} From 0b923769b8f5f51b8e2a75807e47488652a42bb6 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 15 Aug 2025 15:49:17 +0300 Subject: [PATCH 44/47] Explicitly list the used features for synca dependency Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 2 +- avro/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cfeaa81..e57f71f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1294,7 +1294,7 @@ dependencies = [ [[package]] name = "synca" version = "0.5.3" -source = "git+https://github.com/martin-g/rs_synca/?branch=add-support-for-use-path-replacing#1a98bbafcebc82e4bc111036842f2287cbeec3ab" +source = "git+https://github.com/martin-g/rs_synca/?branch=add-support-for-use-path-replacing#99486dde498a1065733639638fb6342fd922d28b" dependencies = [ "proc-macro2", "quote", diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 37fe2cd1..a2fe3488 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -77,8 +77,8 @@ tokio = { version = "1.47.0", default-features = false, features = [ "fs", "io-u uuid = { default-features = false, version = "1.18.0", features = ["serde", "std"] } xz2 = { default-features = false, version = "0.1.7", optional = true } zstd = { default-features = false, version = "0.13.3", optional = true } -synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing" } -#synca = { path = "/home/martin/git/rust/rs_synca/synca" } +synca = { git = "https://github.com/martin-g/rs_synca/", branch = "add-support-for-use-path-replacing", default-features = false, features = ["boxpin", "usepath"] } +#synca = { path = "/home/martin/git/rust/rs_synca/synca", default-features = false, features = ["boxpin", "usepath"] } futures = { version = "0.3.31", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] From aa1c26387e65f2f267f8d9910767a18873deef30 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 25 Aug 2025 15:12:26 +0300 Subject: [PATCH 45/47] Use futures::io::AsyncRead/Write instead of Tokio ones This way Tokio could be just a dev-dependency and users could use another async runtime if they want! Signed-off-by: Martin Tzvetanov Grigorov --- Cargo.lock | 15 +++++++++++++++ avro/Cargo.toml | 24 ++++++++++++++---------- avro/examples/async_benchmark.rs | 2 +- avro/examples/benchmark.rs | 6 ++---- avro/src/bigdecimal.rs | 13 +++++++++---- avro/src/bytes.rs | 2 +- avro/src/de.rs | 8 ++++++-- avro/src/decode.rs | 8 ++++---- avro/src/encode.rs | 8 ++++---- avro/src/error.rs | 2 +- avro/src/headers.rs | 2 +- avro/src/lib.rs | 12 ++++++------ avro/src/reader.rs | 13 ++++++++----- avro/src/schema.rs | 8 ++++---- avro/src/schema_compatibility.rs | 2 +- avro/src/schema_equality.rs | 2 +- avro/src/ser.rs | 2 +- avro/src/types.rs | 2 +- avro/src/util.rs | 16 ++++++++-------- avro/src/validator.rs | 2 +- avro/src/writer.rs | 8 ++++---- avro/tests/async_schema.rs | 19 +++++++++++-------- 22 files changed, 104 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1308cd42..bbe25bc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,7 @@ dependencies = [ "synca", "thiserror", "tokio", + "tokio-util", "uuid", "xz2", "zstd", @@ -1358,6 +1359,20 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-util" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 024fc8c5..f5c7f992 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -29,14 +29,14 @@ categories.workspace = true documentation.workspace = true [features] -default = ["tokio"] +default = ["async"] bzip = ["dep:bzip2"] derive = ["dep:apache-avro-derive"] snappy = ["dep:crc32fast", "dep:snap"] xz = ["dep:xz2"] zstandard = ["dep:zstd"] sync = [] -tokio = ["dep:tokio", "dep:futures"] +async = ["dep:futures"] [lib] # disable benchmarks to allow passing criterion arguments to `cargo bench` @@ -86,12 +86,6 @@ snap = { default-features = false, version = "1.1.0", optional = true } strum = { default-features = false, version = "0.27.2" } strum_macros = { default-features = false, version = "0.27.2" } thiserror = { default-features = false, version = "2.0.16" } -tokio = { version = "1.47.0", default-features = false, features = [ - "fs", - "io-util", - "macros", - "rt-multi-thread", -], optional = true } uuid = { default-features = false, version = "1.18.0", features = [ "serde", "std", @@ -122,6 +116,16 @@ serial_test = "3.2.0" sha2 = { default-features = false, version = "0.10.9" } paste = { default-features = false, version = "1.0.15" } rstest = { default-features = false, version = "0.26.1" } +tokio = { version = "1.47.0", default-features = false, features = [ + "fs", + "io-util", + "macros", + "rt-multi-thread" +]} + +tokio-util = { version = "0.7.16", default-features = false, features = [ + "compat" +]} [package.metadata.docs.rs] all-features = true @@ -135,7 +139,7 @@ required-features = ["sync"] [[example]] name = "async_benchmark" path = "examples/async_benchmark.rs" -required-features = ["tokio"] +required-features = ["async"] [[example]] name = "generate_interop_data" @@ -210,7 +214,7 @@ required-features = ["sync"] [[test]] name = "async_schema" path = "tests/async_schema.rs" -required-features = ["tokio"] +required-features = ["async"] [[test]] name = "shared" diff --git a/avro/examples/async_benchmark.rs b/avro/examples/async_benchmark.rs index a674e3a3..0d8f8520 100644 --- a/avro/examples/async_benchmark.rs +++ b/avro/examples/async_benchmark.rs @@ -22,7 +22,7 @@ use apache_avro::{ }; use futures::StreamExt; use std::time::{Duration, Instant}; -use tokio::io::{BufReader, BufWriter}; +use futures::io::{BufReader, BufWriter}; fn nanos(duration: Duration) -> u64 { duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64 diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index d4961b89..0da68f3f 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -19,10 +19,8 @@ use apache_avro::{ Reader, Schema, SchemaExt, Writer, types::{Record, Value}, }; -use std::{ - io::{BufReader, BufWriter}, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; +use std::io::{BufReader, BufWriter}; fn nanos(duration: Duration) -> u64 { duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64 diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index a0a1429c..ddfe72c7 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -18,7 +18,7 @@ pub use bigdecimal::BigDecimal; #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { @@ -79,7 +79,7 @@ mod bigdecimal { #[synca::cfg(sync)] use std::io::Read; #[synca::cfg(tokio)] - use tokio::io::AsyncReadExt; + use futures::AsyncReadExt; bytes .read_exact(&mut big_decimal_buffer[..]) @@ -119,7 +119,9 @@ mod bigdecimal { #[synca::cfg(tokio)] use tokio::fs::File; #[synca::cfg(tokio)] - use tokio::io::BufReader; + use futures::io::BufReader; + #[synca::cfg(tokio)] + use tokio_util::compat::TokioAsyncReadCompatExt; #[tokio::test] async fn test_avro_3779_bigdecimal_serial() -> TestResult { @@ -227,7 +229,10 @@ mod bigdecimal { // Open file generated with Java code to ensure compatibility // with Java big decimal logical type. let file = File::open("./tests/bigdec.avro").await?; - let mut reader = Reader::new(BufReader::new(file)).await?; + #[synca::cfg(tokio)] + let file = file.compat(); + let buf_reader = BufReader::new(file); + let mut reader = Reader::new(buf_reader).await?; let next_element = reader.next().await; assert!(next_element.is_some()); let value = next_element.unwrap()?; diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 5f5f1dbf..3382db59 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -286,7 +286,7 @@ pub mod serde_avro_slice_opt { } #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio_tests { }, #[cfg(feature = "sync")] pub mod sync_tests { diff --git a/avro/src/de.rs b/avro/src/de.rs index 11251453..74610198 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -18,7 +18,7 @@ //! Logic for serde-compatible deserialization. #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { @@ -811,6 +811,10 @@ mod de { use apache_avro_test_helper::TestResult; use crate::decimal::Decimal; + #[synca::cfg(sync)] + use std::io::Cursor; + #[synca::cfg(tokio)] + use futures::io::Cursor; use super::*; @@ -848,7 +852,7 @@ mod de { // encode into avro let value = to_value(&data)?; - let mut buf = std::io::Cursor::new(to_avro_datum(&schema, value).await?); + let mut buf = Cursor::new(to_avro_datum(&schema, value).await?); // decode from avro let value = from_avro_datum(&schema, &mut buf, None).await?; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index eed85b90..4a23e346 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -16,7 +16,7 @@ // under the License. #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio {}, #[cfg(feature = "sync")] pub mod sync { @@ -37,9 +37,9 @@ mod decode { #[synca::cfg(sync)] use std::io::Read as AvroRead; #[synca::cfg(tokio)] - use tokio::io::AsyncRead as AvroRead; - #[cfg(feature = "tokio")] - use tokio::io::AsyncReadExt; + use futures::AsyncRead as AvroRead; + #[cfg(feature = "async")] + use futures::AsyncReadExt; use crate::AvroResult; use crate::Uuid; diff --git a/avro/src/encode.rs b/avro/src/encode.rs index d4b13d5e..77505c43 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -16,7 +16,7 @@ // under the License. #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio {}, #[cfg(feature = "sync")] pub mod sync { @@ -51,9 +51,9 @@ mod encode { use std::io::Write as AvroWrite; use std::marker::Unpin; #[synca::cfg(tokio)] - use tokio::io::AsyncWrite as AvroWrite; - #[cfg(feature = "tokio")] - use tokio::io::AsyncWriteExt; + use futures::AsyncWrite as AvroWrite; + #[cfg(feature = "async")] + use futures::AsyncWriteExt; use log::error; use std::{borrow::Borrow, collections::HashMap}; diff --git a/avro/src/error.rs b/avro/src/error.rs index 8999ec35..c44b391c 100644 --- a/avro/src/error.rs +++ b/avro/src/error.rs @@ -16,7 +16,7 @@ // under the License. // #[synca::synca( -// #[cfg(feature = "tokio")] +// #[cfg(feature = "async")] // pub mod tokio { }, // #[cfg(feature = "sync")] // pub mod sync { diff --git a/avro/src/headers.rs b/avro/src/headers.rs index 4cd532e1..bb473213 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -18,7 +18,7 @@ //! Handling of Avro magic headers #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 30af513f..5c19c914 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -909,7 +909,7 @@ pub use codec::zstandard::ZstandardSettings; pub use codec::{Codec, DeflateSettings}; #[cfg(feature = "sync")] pub use de::sync::from_value; -#[cfg(feature = "tokio")] +#[cfg(feature = "async")] pub use de::tokio::from_value as async_from_value; pub use decimal::Decimal; pub use duration::{Days, Duration, Millis, Months}; @@ -919,7 +919,7 @@ pub use reader::sync::{ GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, }; -#[cfg(feature = "tokio")] +#[cfg(feature = "async")] pub use reader::tokio::{ GenericSingleObjectReader as AsyncGenericSingleObjectReader, Reader as AsyncReader, SpecificSingleObjectReader as AsyncSpecificSingleObjectReader, @@ -931,11 +931,11 @@ pub use schema::AvroSchema; pub use schema::Schema; #[cfg(feature = "sync")] pub use schema::sync::SchemaExt; -#[cfg(feature = "tokio")] +#[cfg(feature = "async")] pub use schema::tokio::SchemaExt as AsyncSchemaExt; #[cfg(feature = "sync")] pub use ser::sync::to_value; -#[cfg(feature = "tokio")] +#[cfg(feature = "async")] pub use ser::tokio::to_value as async_to_value; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; @@ -944,7 +944,7 @@ pub use writer::sync::{ GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, to_avro_datum_schemata, write_avro_datum_ref, }; -#[cfg(feature = "tokio")] +#[cfg(feature = "async")] pub use writer::tokio::{ GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, to_avro_datum as async_to_avro_datum, @@ -957,7 +957,7 @@ pub use apache_avro_derive::*; pub type AvroResult = Result; #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 8731a393..76786dfc 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -16,7 +16,7 @@ // under the License. #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio {}, #[cfg(feature = "sync")] pub mod sync { @@ -46,9 +46,9 @@ mod reader { #[synca::cfg(sync)] use std::io::Read as AvroRead; #[synca::cfg(tokio)] - use tokio::io::AsyncRead as AvroRead; - #[cfg(feature = "tokio")] - use tokio::io::AsyncReadExt; + use futures::AsyncRead as AvroRead; + #[cfg(feature = "async")] + use futures::AsyncReadExt; use crate::util::tokio::safe_len; use crate::{ @@ -704,7 +704,10 @@ mod reader { use futures::StreamExt; use pretty_assertions::assert_eq; use serde::Deserialize; + #[synca::cfg(sync)] use std::io::Cursor; + #[synca::cfg(tokio)] + use futures::io::Cursor; #[synca::cfg(sync)] use uuid::Uuid; @@ -1097,7 +1100,7 @@ mod reader { let mut to_read = std::io::Read::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); #[synca::cfg(tokio)] - let mut to_read = tokio::io::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]) + let mut to_read = futures::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]) .chain(&to_read_3[..]); let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) diff --git a/avro/src/schema.rs b/avro/src/schema.rs index c1adcef3..c01e7f47 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1531,7 +1531,7 @@ pub mod derive { } #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio {}, #[cfg(feature = "sync")] pub mod sync { @@ -1576,9 +1576,9 @@ mod schema { str::FromStr, }; #[synca::cfg(tokio)] - use tokio::io::AsyncRead as AvroRead; - #[cfg(feature = "tokio")] - use tokio::io::AsyncReadExt; + use futures::AsyncRead as AvroRead; + #[cfg(feature = "async")] + use futures::AsyncReadExt; pub struct RecordFieldExt; diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index f7b42b95..c41549b7 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -18,7 +18,7 @@ //! Logic for checking schema compatibility #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { diff --git a/avro/src/schema_equality.rs b/avro/src/schema_equality.rs index efa7bfe9..ccaf82c7 100644 --- a/avro/src/schema_equality.rs +++ b/avro/src/schema_equality.rs @@ -16,7 +16,7 @@ // under the License. // #[synca::synca( -// #[cfg(feature = "tokio")] +// #[cfg(feature = "async")] // pub mod tokio {}, // #[cfg(feature = "sync")] // pub mod sync { diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 28968ebc..4ea2c933 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -18,7 +18,7 @@ //! Logic for serde-compatible serialization. #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { diff --git a/avro/src/types.rs b/avro/src/types.rs index b851f7d6..7d08ee74 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -279,7 +279,7 @@ to_value!(BigDecimal, Value::BigDecimal); to_value!(Duration, Value::Duration); #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { diff --git a/avro/src/util.rs b/avro/src/util.rs index e9c303b5..92e98b02 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -106,7 +106,7 @@ impl MapHelper for Map { } #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { @@ -125,20 +125,20 @@ impl MapHelper for Map { } )] mod util { - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] use futures::future::TryFutureExt; #[synca::cfg(sync)] use std::io::Read as AvroRead; #[synca::cfg(sync)] use std::io::Write as AvroWrite; #[synca::cfg(tokio)] - use tokio::io::AsyncRead as AvroRead; - #[cfg(feature = "tokio")] - use tokio::io::AsyncReadExt; + use futures::AsyncRead as AvroRead; + #[cfg(feature = "async")] + use futures::AsyncReadExt; #[synca::cfg(tokio)] - use tokio::io::AsyncWrite as AvroWrite; - #[cfg(feature = "tokio")] - use tokio::io::AsyncWriteExt; + use futures::AsyncWrite as AvroWrite; + #[cfg(feature = "async")] + use futures::AsyncWriteExt; use crate::AvroResult; use crate::error::Details; diff --git a/avro/src/validator.rs b/avro/src/validator.rs index cc22a8ca..723a1dcf 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -16,7 +16,7 @@ // under the License. // #[synca::synca( -// #[cfg(feature = "tokio")] +// #[cfg(feature = "async")] // pub mod tokio { }, // #[cfg(feature = "sync")] // pub mod sync { diff --git a/avro/src/writer.rs b/avro/src/writer.rs index bb59381e..10c099a0 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -18,7 +18,7 @@ //! Logic handling writing in Avro format at user level. #[synca::synca( - #[cfg(feature = "tokio")] + #[cfg(feature = "async")] pub mod tokio { }, #[cfg(feature = "sync")] pub mod sync { @@ -45,9 +45,9 @@ mod writer { use std::io::Write as AvroWrite; use std::marker::Unpin; #[synca::cfg(tokio)] - use tokio::io::AsyncWrite as AvroWrite; - #[cfg(feature = "tokio")] - use tokio::io::AsyncWriteExt; + use futures::AsyncWrite as AvroWrite; + #[cfg(feature = "async")] + use futures::AsyncWriteExt; use crate::AvroResult; #[synca::cfg(sync)] diff --git a/avro/tests/async_schema.rs b/avro/tests/async_schema.rs index 0b5c9f88..299ae3a5 100644 --- a/avro/tests/async_schema.rs +++ b/avro/tests/async_schema.rs @@ -31,8 +31,9 @@ use apache_avro_test_helper::{ init, }; use futures::StreamExt; -use std::{collections::HashMap, io::Cursor}; -use tokio::io::AsyncRead; +use std::collections::HashMap; +use futures::AsyncRead; +use futures::io::BufWriter; #[tokio::test] async fn test_correct_recursive_extraction() -> TestResult { @@ -109,7 +110,7 @@ async fn test_parse() -> TestResult { async fn test_3799_parse_reader() -> TestResult { init(); for (raw_schema, valid) in examples().iter() { - let mut reader: &mut (dyn AsyncRead + Unpin) = &mut Cursor::new(raw_schema); + let mut reader: &mut (dyn AsyncRead + Unpin) = &mut raw_schema.as_bytes(); let schema = SchemaExt::parse_reader(&mut reader).await; if *valid { assert!( @@ -126,7 +127,7 @@ async fn test_3799_parse_reader() -> TestResult { // Ensure it works for trait objects too. for (raw_schema, valid) in examples().iter() { - let reader = &mut Cursor::new(raw_schema); + let reader = &mut raw_schema.as_bytes(); let schema = SchemaExt::parse_reader(reader).await; if *valid { assert!( @@ -146,7 +147,9 @@ async fn test_3799_parse_reader() -> TestResult { #[tokio::test] async fn test_3799_raise_io_error_from_parse_read() -> Result<(), String> { // 0xDF is invalid for UTF-8. - let mut invalid_data: &mut (dyn AsyncRead + Unpin) = &mut Cursor::new([0xDF]); + let mut buf = Vec::with_capacity(1); + buf.push(0xDF_u8); + let mut invalid_data: &mut (dyn AsyncRead + Unpin) = &mut buf.as_slice(); let error = SchemaExt::parse_reader(&mut invalid_data) .await .unwrap_err() @@ -2487,14 +2490,14 @@ async fn avro_rs_66_test_independent_canonical_form_missing_ref() -> TestResult #[tokio::test] async fn avro_rs_181_single_null_record() -> TestResult { - let mut buff = Cursor::new(Vec::new()); + let mut buff = BufWriter::new(Vec::new()); let schema = SchemaExt::parse_str(r#""null""#).await?; let mut writer = Writer::new(&schema, &mut buff); writer.append(serde_json::Value::Null).await?; writer.into_inner().await?; - buff.set_position(0); + let reader = buff.into_inner(); - let mut reader = Reader::new(buff).await?; + let mut reader = Reader::new(&reader[..]).await?; while let Some(value) = reader.next().await { assert_eq!(Value::Null, value?); } From f3f3c7297b4a446cecf2797154297f2eac080400 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 25 Aug 2025 15:17:53 +0300 Subject: [PATCH 46/47] fmt && clippy Signed-off-by: Martin Tzvetanov Grigorov --- avro/examples/async_benchmark.rs | 2 +- avro/examples/benchmark.rs | 2 +- avro/src/bigdecimal.rs | 8 ++++---- avro/src/de.rs | 4 ++-- avro/src/decode.rs | 4 ++-- avro/src/encode.rs | 6 +++--- avro/src/reader.rs | 12 ++++++------ avro/src/schema.rs | 8 ++++---- avro/src/util.rs | 12 ++++++------ avro/src/writer.rs | 6 +++--- avro/tests/async_schema.rs | 7 +++---- 11 files changed, 35 insertions(+), 36 deletions(-) diff --git a/avro/examples/async_benchmark.rs b/avro/examples/async_benchmark.rs index 0d8f8520..b71cc66e 100644 --- a/avro/examples/async_benchmark.rs +++ b/avro/examples/async_benchmark.rs @@ -21,8 +21,8 @@ use apache_avro::{ types::{Record, Value}, }; use futures::StreamExt; -use std::time::{Duration, Instant}; use futures::io::{BufReader, BufWriter}; +use std::time::{Duration, Instant}; fn nanos(duration: Duration) -> u64 { duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64 diff --git a/avro/examples/benchmark.rs b/avro/examples/benchmark.rs index 0da68f3f..e05908ad 100644 --- a/avro/examples/benchmark.rs +++ b/avro/examples/benchmark.rs @@ -19,8 +19,8 @@ use apache_avro::{ Reader, Schema, SchemaExt, Writer, types::{Record, Value}, }; -use std::time::{Duration, Instant}; use std::io::{BufReader, BufWriter}; +use std::time::{Duration, Instant}; fn nanos(duration: Duration) -> u64 { duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64 diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index ddfe72c7..a331c6f8 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -76,10 +76,10 @@ mod bigdecimal { Err(err) => return Err(Details::BigDecimalLen(Box::new(err)).into()), }; - #[synca::cfg(sync)] - use std::io::Read; #[synca::cfg(tokio)] use futures::AsyncReadExt; + #[synca::cfg(sync)] + use std::io::Read; bytes .read_exact(&mut big_decimal_buffer[..]) @@ -107,6 +107,8 @@ mod bigdecimal { use bigdecimal::{One, Zero}; #[synca::cfg(tokio)] use futures::StreamExt; + #[synca::cfg(tokio)] + use futures::io::BufReader; use pretty_assertions::assert_eq; #[synca::cfg(sync)] use std::fs::File; @@ -119,8 +121,6 @@ mod bigdecimal { #[synca::cfg(tokio)] use tokio::fs::File; #[synca::cfg(tokio)] - use futures::io::BufReader; - #[synca::cfg(tokio)] use tokio_util::compat::TokioAsyncReadCompatExt; #[tokio::test] diff --git a/avro/src/de.rs b/avro/src/de.rs index 74610198..e720ba20 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -811,10 +811,10 @@ mod de { use apache_avro_test_helper::TestResult; use crate::decimal::Decimal; - #[synca::cfg(sync)] - use std::io::Cursor; #[synca::cfg(tokio)] use futures::io::Cursor; + #[synca::cfg(sync)] + use std::io::Cursor; use super::*; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index 4a23e346..a8c5c76c 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -34,12 +34,12 @@ } )] mod decode { - #[synca::cfg(sync)] - use std::io::Read as AvroRead; #[synca::cfg(tokio)] use futures::AsyncRead as AvroRead; #[cfg(feature = "async")] use futures::AsyncReadExt; + #[synca::cfg(sync)] + use std::io::Read as AvroRead; use crate::AvroResult; use crate::Uuid; diff --git a/avro/src/encode.rs b/avro/src/encode.rs index 77505c43..ba28ec1d 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -47,13 +47,13 @@ mod encode { }, types::{Value, ValueKind}, }; - #[synca::cfg(sync)] - use std::io::Write as AvroWrite; - use std::marker::Unpin; #[synca::cfg(tokio)] use futures::AsyncWrite as AvroWrite; #[cfg(feature = "async")] use futures::AsyncWriteExt; + #[synca::cfg(sync)] + use std::io::Write as AvroWrite; + use std::marker::Unpin; use log::error; use std::{borrow::Borrow, collections::HashMap}; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 76786dfc..96a242cd 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -43,12 +43,12 @@ mod reader { use crate::decode::tokio::{decode, decode_internal}; #[synca::cfg(sync)] use crate::from_value; - #[synca::cfg(sync)] - use std::io::Read as AvroRead; #[synca::cfg(tokio)] use futures::AsyncRead as AvroRead; #[cfg(feature = "async")] use futures::AsyncReadExt; + #[synca::cfg(sync)] + use std::io::Read as AvroRead; use crate::util::tokio::safe_len; use crate::{ @@ -702,12 +702,12 @@ mod reader { use apache_avro_test_helper::TestResult; #[synca::cfg(tokio)] use futures::StreamExt; + #[synca::cfg(tokio)] + use futures::io::Cursor; use pretty_assertions::assert_eq; use serde::Deserialize; #[synca::cfg(sync)] use std::io::Cursor; - #[synca::cfg(tokio)] - use futures::io::Cursor; #[synca::cfg(sync)] use uuid::Uuid; @@ -1100,8 +1100,8 @@ mod reader { let mut to_read = std::io::Read::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); #[synca::cfg(tokio)] - let mut to_read = futures::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]) - .chain(&to_read_3[..]); + let mut to_read = + futures::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); let generic_reader = GenericSingleObjectReader::new(TestSingleObjectReader::get_schema()) .expect("Schema should resolve"); diff --git a/avro/src/schema.rs b/avro/src/schema.rs index c01e7f47..12cec75d 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1567,6 +1567,10 @@ mod schema { types::{Value, ValueKind}, validator::{validate_enum_symbol_name, validate_record_field_name}, }; + #[synca::cfg(tokio)] + use futures::AsyncRead as AvroRead; + #[cfg(feature = "async")] + use futures::AsyncReadExt; use log::{debug, error, warn}; #[synca::cfg(sync)] use std::io::Read as AvroRead; @@ -1575,10 +1579,6 @@ mod schema { fmt::Debug, str::FromStr, }; - #[synca::cfg(tokio)] - use futures::AsyncRead as AvroRead; - #[cfg(feature = "async")] - use futures::AsyncReadExt; pub struct RecordFieldExt; diff --git a/avro/src/util.rs b/avro/src/util.rs index 92e98b02..faf008c6 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -125,12 +125,6 @@ impl MapHelper for Map { } )] mod util { - #[cfg(feature = "async")] - use futures::future::TryFutureExt; - #[synca::cfg(sync)] - use std::io::Read as AvroRead; - #[synca::cfg(sync)] - use std::io::Write as AvroWrite; #[synca::cfg(tokio)] use futures::AsyncRead as AvroRead; #[cfg(feature = "async")] @@ -139,6 +133,12 @@ mod util { use futures::AsyncWrite as AvroWrite; #[cfg(feature = "async")] use futures::AsyncWriteExt; + #[cfg(feature = "async")] + use futures::future::TryFutureExt; + #[synca::cfg(sync)] + use std::io::Read as AvroRead; + #[synca::cfg(sync)] + use std::io::Write as AvroWrite; use crate::AvroResult; use crate::error::Details; diff --git a/avro/src/writer.rs b/avro/src/writer.rs index 10c099a0..bcb3874f 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -41,13 +41,13 @@ )] mod writer { - #[synca::cfg(sync)] - use std::io::Write as AvroWrite; - use std::marker::Unpin; #[synca::cfg(tokio)] use futures::AsyncWrite as AvroWrite; #[cfg(feature = "async")] use futures::AsyncWriteExt; + #[synca::cfg(sync)] + use std::io::Write as AvroWrite; + use std::marker::Unpin; use crate::AvroResult; #[synca::cfg(sync)] diff --git a/avro/tests/async_schema.rs b/avro/tests/async_schema.rs index 299ae3a5..1b17a300 100644 --- a/avro/tests/async_schema.rs +++ b/avro/tests/async_schema.rs @@ -30,10 +30,10 @@ use apache_avro_test_helper::{ data::{DOC_EXAMPLES, examples, valid_examples}, init, }; -use futures::StreamExt; -use std::collections::HashMap; use futures::AsyncRead; +use futures::StreamExt; use futures::io::BufWriter; +use std::collections::HashMap; #[tokio::test] async fn test_correct_recursive_extraction() -> TestResult { @@ -147,8 +147,7 @@ async fn test_3799_parse_reader() -> TestResult { #[tokio::test] async fn test_3799_raise_io_error_from_parse_read() -> Result<(), String> { // 0xDF is invalid for UTF-8. - let mut buf = Vec::with_capacity(1); - buf.push(0xDF_u8); + let buf = vec![0xDF_u8]; let mut invalid_data: &mut (dyn AsyncRead + Unpin) = &mut buf.as_slice(); let error = SchemaExt::parse_reader(&mut invalid_data) .await From 42396999ae91dd948c2ee08db1cc9230fb251219 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 25 Aug 2025 16:55:47 +0300 Subject: [PATCH 47/47] Rename the name of the generated modules to asynch and synch This way "tokio" is not advertized. Signed-off-by: Martin Tzvetanov Grigorov --- avro/Cargo.toml | 50 ++++++------- avro/README.md | 28 ++++---- avro/examples/async_benchmark.rs | 2 +- avro/src/bigdecimal.rs | 52 +++++++------- avro/src/bytes.rs | 20 +++--- avro/src/de.rs | 40 +++++------ avro/src/decode.rs | 38 +++++----- avro/src/encode.rs | 34 ++++----- avro/src/error.rs | 24 +++---- avro/src/headers.rs | 24 +++---- avro/src/lib.rs | 106 ++++++++++++++-------------- avro/src/reader.rs | 90 ++++++++++++------------ avro/src/schema.rs | 56 +++++++-------- avro/src/schema_compatibility.rs | 40 +++++------ avro/src/schema_equality.rs | 26 +++---- avro/src/ser.rs | 24 +++---- avro/src/ser_schema.rs | 6 +- avro/src/types.rs | 40 +++++------ avro/src/util.rs | 40 +++++------ avro/src/validator.rs | 28 ++++---- avro/src/writer.rs | 116 +++++++++++++++---------------- avro/tests/async_schema.rs | 4 +- avro/tests/avro-3786.rs | 2 +- avro/tests/avro-3787.rs | 2 +- avro/tests/io.rs | 2 +- avro/tests/schema.rs | 2 +- avro_derive/Cargo.toml | 2 +- avro_derive/tests/derive.rs | 2 +- wasm-demo/Cargo.toml | 2 +- 29 files changed, 451 insertions(+), 451 deletions(-) diff --git a/avro/Cargo.toml b/avro/Cargo.toml index f5c7f992..5e2a1ce4 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -29,14 +29,14 @@ categories.workspace = true documentation.workspace = true [features] -default = ["async"] +default = ["asynch"] bzip = ["dep:bzip2"] derive = ["dep:apache-avro-derive"] snappy = ["dep:crc32fast", "dep:snap"] xz = ["dep:xz2"] zstandard = ["dep:zstd"] -sync = [] -async = ["dep:futures"] +synch = [] +asynch = ["dep:futures"] [lib] # disable benchmarks to allow passing criterion arguments to `cargo bench` @@ -134,109 +134,109 @@ rustdoc-args = ["--cfg", "docsrs"] [[example]] name = "benchmark" path = "examples/benchmark.rs" -required-features = ["sync"] +required-features = ["synch"] [[example]] name = "async_benchmark" path = "examples/async_benchmark.rs" -required-features = ["async"] +required-features = ["asynch"] [[example]] name = "generate_interop_data" path = "examples/generate_interop_data.rs" -required-features = ["sync"] +required-features = ["synch"] [[example]] name = "specific_single_object" path = "examples/specific_single_object.rs" -required-features = ["sync", "derive"] +required-features = ["synch", "derive"] [[example]] name = "test_interop_data" path = "examples/test_interop_data.rs" -required-features = ["sync"] +required-features = ["synch"] [[example]] name = "test_interop_single_object_encoding" path = "examples/test_interop_single_object_encoding.rs" -required-features = ["sync"] +required-features = ["synch"] [[example]] name = "to_value" path = "examples/to_value.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "append_to_existing" path = "tests/append_to_existing.rs" -required-features = ["sync", "derive"] +required-features = ["synch", "derive"] [[test]] name = "avro-3786" path = "tests/avro-3786.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "avro-3787" path = "tests/avro-3787.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "avro-rs-219" path = "tests/avro-rs-219.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "avro-rs-226" path = "tests/avro-rs-226.rs" -required-features = ["sync", "derive"] +required-features = ["synch", "derive"] [[test]] name = "big_decimal" path = "tests/big_decimal.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "codecs" path = "tests/codecs.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "io" path = "tests/io.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "schema" path = "tests/schema.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "async_schema" path = "tests/async_schema.rs" -required-features = ["async"] +required-features = ["asynch"] [[test]] name = "shared" path = "tests/shared.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "to_from_avro_datum_schemata" path = "tests/to_from_avro_datum_schemata.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "union_schema" path = "tests/union_schema.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "uuids" path = "tests/uuids.rs" -required-features = ["sync"] +required-features = ["synch"] [[test]] name = "validators" path = "tests/validators.rs" -required-features = ["sync"] +required-features = ["synch"] diff --git a/avro/README.md b/avro/README.md index 1284273f..983df9c0 100644 --- a/avro/README.md +++ b/avro/README.md @@ -127,7 +127,7 @@ Avro schemas are defined in **JSON** format and can just be parsed out of a raw ```rust use apache_avro::Schema; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; let raw_schema = r#" { @@ -152,7 +152,7 @@ them will be parsed into the corresponding schemas. ```rust use apache_avro::Schema; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; let raw_schema_1 = r#"{ "name": "A", @@ -208,7 +208,7 @@ associated type provided by the library to specify the data we want to serialize use apache_avro::Schema; use apache_avro::types::Record; use apache_avro::Writer; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; let raw_schema = r#" { @@ -258,7 +258,7 @@ deriving `Serialize` to model our data: use apache_avro::Schema; use serde::Serialize; use apache_avro::Writer; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; #[derive(Debug, Serialize)] struct Test { @@ -330,7 +330,7 @@ Avro supports three different compression codecs when encoding data: To specify a codec to use to compress data, just specify it while creating a `Writer`: ```rust use apache_avro::{Codec, DeflateSettings, Schema, Writer}; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; let raw_schema = r#" { @@ -355,7 +355,7 @@ codec: ```rust use apache_avro::Reader; use apache_avro::Schema; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; use apache_avro::types::Record; use apache_avro::Writer; @@ -387,7 +387,7 @@ use apache_avro::Schema; use apache_avro::Reader; use apache_avro::types::Record; use apache_avro::Writer; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; let writer_raw_schema = r#" { @@ -447,7 +447,7 @@ use apache_avro::Schema; use apache_avro::types::Record; use apache_avro::Writer; use apache_avro::Reader; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; let raw_schema = r#" { @@ -483,7 +483,7 @@ read the data into: ```rust use apache_avro::Reader; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; use apache_avro::from_value; #[derive(Debug, Deserialize)] @@ -507,7 +507,7 @@ quick reference of the library interface: ```rust use apache_avro::{Codec, DeflateSettings, Reader, Schema, Writer, from_value, types::Record, Error}; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; use serde::{Deserialize, Serialize}; #[derive(Debug, Deserialize, Serialize)] @@ -575,7 +575,7 @@ use apache_avro::{ types::Record, types::Value, Codec, Days, Decimal, DeflateSettings, Duration, Millis, Months, Reader, Schema, Writer, Error, }; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; use num_bigint::ToBigInt; fn main() -> Result<(), Error> { @@ -701,7 +701,7 @@ An example of fingerprinting for the supported fingerprints: ```rust use apache_avro::rabin::Rabin; use apache_avro::{Schema, Error}; -use apache_avro::schema::sync::SchemaExt; +use apache_avro::schema::synch::SchemaExt; use md5::Md5; use sha2::Sha256; @@ -765,7 +765,7 @@ Explanation: an int array schema can be read by a long array schema- an int (32bit signed integer) fits into a long (64bit signed integer) ```rust -use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; +use apache_avro::{Schema, schema::synch::SchemaExt, schema_compatibility::synch::SchemaCompatibility}; let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); @@ -778,7 +778,7 @@ Explanation: a long array schema cannot be read by an int array schema- a long (64bit signed integer) does not fit into an int (32bit signed integer) ```rust -use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; +use apache_avro::{Schema, schema::synch::SchemaExt, schema_compatibility::synch::SchemaCompatibility}; let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); diff --git a/avro/examples/async_benchmark.rs b/avro/examples/async_benchmark.rs index b71cc66e..81a5fb32 100644 --- a/avro/examples/async_benchmark.rs +++ b/avro/examples/async_benchmark.rs @@ -17,7 +17,7 @@ use apache_avro::{ AsyncReader as Reader, AsyncWriter as Writer, Schema, - schema::tokio::SchemaExt, + schema::asynch::SchemaExt, types::{Record, Value}, }; use futures::StreamExt; diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs index a331c6f8..3d03b1a6 100644 --- a/avro/src/bigdecimal.rs +++ b/avro/src/bigdecimal.rs @@ -18,21 +18,21 @@ pub use bigdecimal::BigDecimal; #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::reader::tokio => crate::reader::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::writer::tokio => crate::writer::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::reader::asynch => crate::reader::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::writer::asynch => crate::writer::synch, #[tokio::test] => #[test] ); } @@ -40,8 +40,8 @@ pub use bigdecimal::BigDecimal; mod bigdecimal { use crate::AvroResult; use crate::{ - decode::tokio::{decode_len, decode_long}, - encode::tokio::{encode_bytes, encode_long}, + decode::asynch::{decode_len, decode_long}, + encode::asynch::{encode_bytes, encode_long}, error::Details, types::Value, }; @@ -76,9 +76,9 @@ mod bigdecimal { Err(err) => return Err(Details::BigDecimalLen(Box::new(err)).into()), }; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncReadExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Read; bytes @@ -100,27 +100,27 @@ mod bigdecimal { mod tests { use super::*; use crate::{ - codec::Codec, error::Error, reader::tokio::Reader, schema::tokio::SchemaExt, - types::Record, writer::tokio::Writer, + codec::Codec, error::Error, reader::asynch::Reader, schema::asynch::SchemaExt, + types::Record, writer::asynch::Writer, }; use apache_avro_test_helper::TestResult; use bigdecimal::{One, Zero}; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::StreamExt; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::io::BufReader; use pretty_assertions::assert_eq; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::fs::File; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::BufReader; use std::{ ops::{Div, Mul}, str::FromStr, }; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use tokio::fs::File; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use tokio_util::compat::TokioAsyncReadCompatExt; #[tokio::test] @@ -229,7 +229,7 @@ mod bigdecimal { // Open file generated with Java code to ensure compatibility // with Java big decimal logical type. let file = File::open("./tests/bigdec.avro").await?; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] let file = file.compat(); let buf_reader = BufReader::new(file); let mut reader = Reader::new(buf_reader).await?; diff --git a/avro/src/bytes.rs b/avro/src/bytes.rs index 3382db59..65c5597b 100644 --- a/avro/src/bytes.rs +++ b/avro/src/bytes.rs @@ -286,16 +286,16 @@ pub mod serde_avro_slice_opt { } #[synca::synca( - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] pub mod tokio_tests { }, - #[cfg(feature = "sync")] + #[cfg(feature = "synch")] pub mod sync_tests { sync!(); replace!( - crate::de::tokio => crate::de::sync, - crate::schema::tokio => crate::schema::sync, - crate::ser::tokio => crate::ser::sync, - crate::types::tokio => crate::types::sync, + crate::de::asynch => crate::de::synch, + crate::schema::asynch => crate::schema::synch, + crate::ser::asynch => crate::ser::synch, + crate::types::asynch => crate::types::synch, #[tokio::test] => #[test] ); } @@ -303,10 +303,10 @@ pub mod serde_avro_slice_opt { #[cfg(test)] mod tests { use super::*; - use crate::de::tokio::from_value; - use crate::schema::tokio::SchemaExt; - use crate::ser::tokio::to_value; - use crate::types::{Value, tokio::ValueExt}; + use crate::de::asynch::from_value; + use crate::schema::asynch::SchemaExt; + use crate::ser::asynch::to_value; + use crate::types::{Value, asynch::ValueExt}; use apache_avro_test_helper::TestResult; use serde::{Deserialize, Serialize}; diff --git a/avro/src/de.rs b/avro/src/de.rs index e720ba20..d7007e0c 100644 --- a/avro/src/de.rs +++ b/avro/src/de.rs @@ -18,22 +18,22 @@ //! Logic for serde-compatible deserialization. #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::ser::tokio => crate::ser::sync, - crate::schema::tokio => crate::schema::sync, - crate::reader::tokio => crate::reader::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::writer::tokio => crate::writer::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::ser::asynch => crate::ser::synch, + crate::schema::asynch => crate::schema::synch, + crate::reader::asynch => crate::reader::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::writer::asynch => crate::writer::synch, #[tokio::test] => #[test] ); } @@ -797,10 +797,10 @@ mod de { #[cfg(test)] mod tests { - use crate::reader::tokio::from_avro_datum; - use crate::schema::tokio::SchemaExt; - use crate::ser::tokio::to_value; - use crate::writer::tokio::to_avro_datum; + use crate::reader::asynch::from_avro_datum; + use crate::schema::asynch::SchemaExt; + use crate::ser::asynch::to_value; + use crate::writer::asynch::to_avro_datum; use num_bigint::BigInt; use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; @@ -811,9 +811,9 @@ mod de { use apache_avro_test_helper::TestResult; use crate::decimal::Decimal; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::io::Cursor; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Cursor; use super::*; diff --git a/avro/src/decode.rs b/avro/src/decode.rs index a8c5c76c..be209870 100644 --- a/avro/src/decode.rs +++ b/avro/src/decode.rs @@ -16,36 +16,36 @@ // under the License. #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch {}, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, #[tokio::test] => #[test] ); } )] mod decode { - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncRead as AvroRead; - #[cfg(feature = "async")] + #[synca::cfg(asynch)] use futures::AsyncReadExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Read as AvroRead; use crate::AvroResult; use crate::Uuid; - use crate::util::tokio::{safe_len, zag_i32, zag_i64}; + use crate::util::asynch::{safe_len, zag_i32, zag_i64}; use crate::{ - bigdecimal::tokio::deserialize_big_decimal, + bigdecimal::asynch::deserialize_big_decimal, decimal::Decimal, duration::Duration, error::Details, @@ -396,11 +396,11 @@ mod decode { #[cfg(test)] #[allow(clippy::expect_fun_call)] mod tests { - use crate::schema::tokio::SchemaExt; + use crate::schema::asynch::SchemaExt; use crate::{ decimal::Decimal, - decode::tokio::decode, - encode::tokio::{encode, tests::success}, + decode::asynch::decode, + encode::asynch::{encode, tests::success}, schema::{DecimalSchema, FixedSchema, Name, Schema}, types::Value, }; diff --git a/avro/src/encode.rs b/avro/src/encode.rs index ba28ec1d..dd95e11a 100644 --- a/avro/src/encode.rs +++ b/avro/src/encode.rs @@ -16,30 +16,30 @@ // under the License. #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch {}, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, #[tokio::test] => #[test] ); } )] mod encode { - use crate::util::tokio::{zig_i32, zig_i64}; + use crate::util::asynch::{zig_i32, zig_i64}; use crate::AvroResult; use crate::{ - bigdecimal::tokio::serialize_big_decimal, + bigdecimal::asynch::serialize_big_decimal, error::Details, schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, RecordSchema, ResolvedSchema, @@ -47,11 +47,11 @@ mod encode { }, types::{Value, ValueKind}, }; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncWrite as AvroWrite; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::AsyncWriteExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Write as AvroWrite; use std::marker::Unpin; @@ -423,7 +423,7 @@ mod encode { pub(crate) mod tests { use super::*; use crate::error::{Details, Error}; - use crate::schema::tokio::SchemaExt; + use crate::schema::asynch::SchemaExt; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; use uuid::Uuid; diff --git a/avro/src/error.rs b/avro/src/error.rs index c44b391c..21f18919 100644 --- a/avro/src/error.rs +++ b/avro/src/error.rs @@ -16,20 +16,20 @@ // under the License. // #[synca::synca( -// #[cfg(feature = "async")] -// pub mod tokio { }, -// #[cfg(feature = "sync")] -// pub mod sync { +// #[cfg(feature = "asynch")] +// pub mod asynch { }, +// #[cfg(feature = "synch")] +// pub mod synch { // sync!(); // replace!( -// crate::bigdecimal::tokio => crate::bigdecimal::sync, -// crate::decimal::tokio => crate::decimal::sync, -// crate::decode::tokio => crate::decode::sync, -// crate::encode::tokio => crate::encode::sync, -// crate::error::tokio => crate::error::sync, -// crate::schema::tokio => crate::schema::sync, -// crate::util::tokio => crate::util::sync, -// crate::types::tokio => crate::types::sync, +// crate::bigdecimal::asynch => crate::bigdecimal::synch, +// crate::decimal::asynch => crate::decimal::synch, +// crate::decode::asynch => crate::decode::synch, +// crate::encode::asynch => crate::encode::synch, +// crate::error::asynch => crate::error::synch, +// crate::schema::asynch => crate::schema::synch, +// crate::util::asynch => crate::util::synch, +// crate::types::asynch => crate::types::synch, // #[tokio::test] => #[test] // ); // } diff --git a/avro/src/headers.rs b/avro/src/headers.rs index bb473213..1158e99e 100644 --- a/avro/src/headers.rs +++ b/avro/src/headers.rs @@ -18,19 +18,19 @@ //! Handling of Avro magic headers #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, #[tokio::test] => #[test] ); } @@ -135,7 +135,7 @@ mod headers { mod tests { use super::*; use crate::error::{Details, Error}; - use crate::schema::tokio::SchemaExt; + use crate::schema::asynch::SchemaExt; use apache_avro_test_helper::TestResult; #[tokio::test] diff --git a/avro/src/lib.rs b/avro/src/lib.rs index 5c19c914..a86fab53 100644 --- a/avro/src/lib.rs +++ b/avro/src/lib.rs @@ -116,7 +116,7 @@ //! //! ``` //! use apache_avro::Schema; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! let raw_schema = r#" //! { @@ -141,7 +141,7 @@ //! //! ``` //! use apache_avro::Schema; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! let raw_schema_1 = r#"{ //! "name": "A", @@ -197,7 +197,7 @@ //! use apache_avro::Schema; //! use apache_avro::types::Record; //! use apache_avro::Writer; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! let raw_schema = r#" //! { @@ -247,7 +247,7 @@ //! use apache_avro::Schema; //! use serde::Serialize; //! use apache_avro::Writer; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! #[derive(Debug, Serialize)] //! struct Test { @@ -319,7 +319,7 @@ //! To specify a codec to use to compress data, just specify it while creating a `Writer`: //! ``` //! use apache_avro::{Codec, DeflateSettings, Schema, Writer}; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! let raw_schema = r#" //! { @@ -344,7 +344,7 @@ //! ``` //! use apache_avro::Reader; //! use apache_avro::Schema; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! use apache_avro::types::Record; //! use apache_avro::Writer; //! @@ -376,7 +376,7 @@ //! use apache_avro::Reader; //! use apache_avro::types::Record; //! use apache_avro::Writer; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! let writer_raw_schema = r#" //! { @@ -436,7 +436,7 @@ //! use apache_avro::types::Record; //! use apache_avro::Writer; //! use apache_avro::Reader; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! //! let raw_schema = r#" //! { @@ -475,7 +475,7 @@ //! # use apache_avro::Writer; //! # use serde::{Deserialize, Serialize}; //! use apache_avro::Reader; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! use apache_avro::from_value; //! //! # #[derive(Serialize)] @@ -518,7 +518,7 @@ //! //! ``` //! use apache_avro::{Codec, DeflateSettings, Reader, Schema, Writer, from_value, types::Record, Error}; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! use serde::{Deserialize, Serialize}; //! //! #[derive(Debug, Deserialize, Serialize)] @@ -586,7 +586,7 @@ //! types::Record, types::Value, Codec, Days, Decimal, DeflateSettings, Duration, Millis, Months, Reader, Schema, //! Writer, Error, //! }; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! use num_bigint::ToBigInt; //! //! fn main() -> Result<(), Error> { @@ -712,7 +712,7 @@ //! ```rust //! use apache_avro::rabin::Rabin; //! use apache_avro::{Schema, Error}; -//! use apache_avro::schema::sync::SchemaExt; +//! use apache_avro::schema::synch::SchemaExt; //! use md5::Md5; //! use sha2::Sha256; //! @@ -776,7 +776,7 @@ //! (32bit signed integer) fits into a long (64bit signed integer) //! //! ```rust -//! use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; +//! use apache_avro::{Schema, schema::synch::SchemaExt, schema_compatibility::synch::SchemaCompatibility}; //! //! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); //! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); @@ -789,7 +789,7 @@ //! long (64bit signed integer) does not fit into an int (32bit signed integer) //! //! ```rust -//! use apache_avro::{Schema, schema::sync::SchemaExt, schema_compatibility::sync::SchemaCompatibility}; +//! use apache_avro::{Schema, schema::synch::SchemaExt, schema_compatibility::synch::SchemaCompatibility}; //! //! let writers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap(); //! let readers_schema = SchemaExt::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap(); @@ -881,7 +881,7 @@ mod duration; mod encode; mod reader; mod ser; -#[cfg(feature = "sync")] +#[cfg(feature = "synch")] mod ser_schema; mod util; mod writer; @@ -907,49 +907,49 @@ pub use codec::xz::XzSettings; #[cfg(feature = "zstandard")] pub use codec::zstandard::ZstandardSettings; pub use codec::{Codec, DeflateSettings}; -#[cfg(feature = "sync")] -pub use de::sync::from_value; -#[cfg(feature = "async")] -pub use de::tokio::from_value as async_from_value; +#[cfg(feature = "asynch")] +pub use de::asynch::from_value as async_from_value; +#[cfg(feature = "synch")] +pub use de::synch::from_value; pub use decimal::Decimal; pub use duration::{Days, Duration, Millis, Months}; pub use error::Error; -#[cfg(feature = "sync")] -pub use reader::sync::{ - GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, - from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, -}; -#[cfg(feature = "async")] -pub use reader::tokio::{ +#[cfg(feature = "asynch")] +pub use reader::asynch::{ GenericSingleObjectReader as AsyncGenericSingleObjectReader, Reader as AsyncReader, SpecificSingleObjectReader as AsyncSpecificSingleObjectReader, from_avro_datum as async_from_avro_datum, from_avro_datum_reader_schemata as async_from_avro_datum_reader_schemata, from_avro_datum_schemata as async_from_avro_datum_schemata, read_marker as async_read_marker, }; +#[cfg(feature = "synch")] +pub use reader::synch::{ + GenericSingleObjectReader, Reader, SpecificSingleObjectReader, from_avro_datum, + from_avro_datum_reader_schemata, from_avro_datum_schemata, read_marker, +}; pub use schema::AvroSchema; pub use schema::Schema; -#[cfg(feature = "sync")] -pub use schema::sync::SchemaExt; -#[cfg(feature = "async")] -pub use schema::tokio::SchemaExt as AsyncSchemaExt; -#[cfg(feature = "sync")] -pub use ser::sync::to_value; -#[cfg(feature = "async")] -pub use ser::tokio::to_value as async_to_value; +#[cfg(feature = "asynch")] +pub use schema::asynch::SchemaExt as AsyncSchemaExt; +#[cfg(feature = "synch")] +pub use schema::synch::SchemaExt; +#[cfg(feature = "asynch")] +pub use ser::asynch::to_value as async_to_value; +#[cfg(feature = "synch")] +pub use ser::synch::to_value; pub use util::{max_allocation_bytes, set_serde_human_readable}; pub use uuid::Uuid; -#[cfg(feature = "sync")] -pub use writer::sync::{ - GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, - to_avro_datum_schemata, write_avro_datum_ref, -}; -#[cfg(feature = "async")] -pub use writer::tokio::{ +#[cfg(feature = "asynch")] +pub use writer::asynch::{ GenericSingleObjectWriter as AsyncGenericSingleObjectWriter, Writer as AsyncWriter, WriterBuilder as AsyncWriterBuilder, to_avro_datum as async_to_avro_datum, to_avro_datum_schemata as async_to_avro_datum_schemata, }; +#[cfg(feature = "synch")] +pub use writer::synch::{ + GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder, to_avro_datum, + to_avro_datum_schemata, write_avro_datum_ref, +}; #[cfg(feature = "derive")] pub use apache_avro_derive::*; @@ -957,16 +957,16 @@ pub use apache_avro_derive::*; pub type AvroResult = Result; #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::reader::tokio => crate::reader::sync, - crate::schema::tokio => crate::schema::sync, - crate::types::tokio => crate::types::sync, - crate::writer::tokio => crate::writer::sync, + crate::reader::asynch => crate::reader::synch, + crate::schema::asynch => crate::schema::synch, + crate::types::asynch => crate::types::synch, + crate::writer::asynch => crate::writer::synch, #[tokio::test] => #[test] ); } @@ -975,12 +975,12 @@ pub type AvroResult = Result; mod tests { use crate::{ codec::Codec, - reader::tokio::{Reader, from_avro_datum}, - schema::tokio::SchemaExt, + reader::asynch::{Reader, from_avro_datum}, + schema::asynch::SchemaExt, types::{Record, Value}, - writer::tokio::Writer, + writer::asynch::Writer, }; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::StreamExt; use pretty_assertions::assert_eq; diff --git a/avro/src/reader.rs b/avro/src/reader.rs index 96a242cd..32debb4f 100644 --- a/avro/src/reader.rs +++ b/avro/src/reader.rs @@ -16,21 +16,21 @@ // under the License. #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch {}, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::headers::tokio => crate::headers::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::writer::tokio => crate::writer::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::headers::asynch => crate::headers::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::writer::asynch => crate::writer::synch, #[tokio::test] => #[test] ); } @@ -38,32 +38,32 @@ mod reader { use crate::AvroResult; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use crate::async_from_value as from_value; - use crate::decode::tokio::{decode, decode_internal}; - #[synca::cfg(sync)] + use crate::decode::asynch::{decode, decode_internal}; + #[synca::cfg(synch)] use crate::from_value; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncRead as AvroRead; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::AsyncReadExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Read as AvroRead; - use crate::util::tokio::safe_len; + use crate::util::asynch::safe_len; use crate::{ codec::Codec, error::Details, error::Error, - headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, + headers::asynch::{HeaderBuilder, RabinFingerprintHeader}, schema::AvroSchema, - schema::tokio::SchemaExt, + schema::asynch::SchemaExt, schema::{ Names, ResolvedOwnedSchema, ResolvedSchema, Schema, resolve_names, resolve_names_with_schemata, }, - types::{Value, tokio::ValueExt}, - util::tokio::read_long, + types::{Value, asynch::ValueExt}, + util::asynch::read_long, }; use log::warn; use serde::de::DeserializeOwned; @@ -476,7 +476,7 @@ mod reader { } } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl Iterator for Reader<'_, R> { type Item = AvroResult; @@ -495,12 +495,12 @@ mod reader { } } - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use std::pin::Pin; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use std::task::{Context, Poll}; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] impl futures::Stream for Reader<'_, R> { type Item = AvroResult; @@ -692,23 +692,23 @@ mod reader { #[cfg(test)] mod tests { use super::*; - #[synca::cfg(sync)] - use crate::encode::tokio::encode; - #[synca::cfg(sync)] - use crate::headers::sync::GlueSchemaUuidHeader; - #[synca::cfg(sync)] + #[synca::cfg(synch)] + use crate::encode::asynch::encode; + #[synca::cfg(synch)] + use crate::headers::synch::GlueSchemaUuidHeader; + #[synca::cfg(synch)] use crate::rabin::Rabin; use crate::types::Record; use apache_avro_test_helper::TestResult; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::StreamExt; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::io::Cursor; use pretty_assertions::assert_eq; use serde::Deserialize; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Cursor; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use uuid::Uuid; const SCHEMA: &str = r#" @@ -925,7 +925,7 @@ mod reader { #[tokio::test] async fn test_avro_3405_read_user_metadata_success() -> TestResult { - use crate::writer::tokio::Writer; + use crate::writer::asynch::Writer; let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); @@ -964,7 +964,7 @@ mod reader { c: Vec, } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl AvroSchema for TestSingleObjectReader { fn get_schema() -> Schema { let schema = r#" @@ -1039,7 +1039,7 @@ mod reader { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn test_avro_3507_single_object_reader() -> TestResult { let obj = TestSingleObjectReader { a: 42, @@ -1073,7 +1073,7 @@ mod reader { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn avro_3642_test_single_object_reader_incomplete_reads() -> TestResult { let obj = TestSingleObjectReader { a: 42, @@ -1096,10 +1096,10 @@ mod reader { ) .expect("Encode should succeed"); - #[synca::cfg(sync)] + #[synca::cfg(synch)] let mut to_read = std::io::Read::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] let mut to_read = futures::AsyncReadExt::chain(&to_read_1[..], &to_read_2[..]).chain(&to_read_3[..]); let generic_reader = @@ -1115,7 +1115,7 @@ mod reader { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn test_avro_3507_reader_parity() -> TestResult { let obj = TestSingleObjectReader { a: 42, @@ -1163,7 +1163,7 @@ mod reader { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn avro_rs_164_generic_reader_alternate_header() -> TestResult { let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?; let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid); diff --git a/avro/src/schema.rs b/avro/src/schema.rs index 12cec75d..352dcf27 100644 --- a/avro/src/schema.rs +++ b/avro/src/schema.rs @@ -1531,23 +1531,23 @@ pub mod derive { } #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio {}, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch {}, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::de::tokio => crate::de::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::reader::tokio => crate::reader::sync, - crate::writer::tokio => crate::writer::sync, - crate::ser::tokio => crate::ser::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::util::tokio => crate::util::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::de::asynch => crate::de::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::reader::asynch => crate::reader::synch, + crate::writer::asynch => crate::writer::synch, + crate::ser::asynch => crate::ser::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::util::asynch => crate::util::synch, #[tokio::test] => #[test] ); } @@ -1561,18 +1561,18 @@ mod schema { Namespace, Precision, RecordField, RecordFieldOrder, RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, }; - use crate::types::tokio::ValueExt; + use crate::types::asynch::ValueExt; use crate::util::MapHelper; use crate::{ types::{Value, ValueKind}, validator::{validate_enum_symbol_name, validate_record_field_name}, }; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncRead as AvroRead; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::AsyncReadExt; use log::{debug, error, warn}; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Read as AvroRead; use std::{ collections::{BTreeMap, HashMap, HashSet}, @@ -2756,15 +2756,15 @@ mod schema { #[cfg(test)] mod tests { use super::*; - #[synca::cfg(sync)] - use crate::writer::sync::SpecificSingleObjectWriter; + #[synca::cfg(synch)] + use crate::writer::synch::SpecificSingleObjectWriter; use crate::{ - de::tokio::from_value, + de::asynch::from_value, error::Details, rabin::Rabin, - reader::tokio::from_avro_datum, - ser::tokio::to_value, - writer::tokio::{Writer, to_avro_datum}, + reader::asynch::from_avro_datum, + ser::asynch::to_value, + writer::asynch::{Writer, to_avro_datum}, }; use apache_avro_test_helper::{ TestResult, @@ -2772,9 +2772,9 @@ mod schema { }; use serde::{Deserialize, Serialize}; use serde_json::json; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use serial_test::serial; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::sync::atomic::Ordering; #[tokio::test] @@ -6913,7 +6913,7 @@ mod schema { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] #[serial(serde_is_human_readable)] fn avro_rs_53_uuid_with_fixed() -> TestResult { #[derive(Debug, Serialize, Deserialize)] diff --git a/avro/src/schema_compatibility.rs b/avro/src/schema_compatibility.rs index c41549b7..d521eeb1 100644 --- a/avro/src/schema_compatibility.rs +++ b/avro/src/schema_compatibility.rs @@ -18,21 +18,21 @@ //! Logic for checking schema compatibility #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::reader::tokio => crate::reader::sync, - crate::types::tokio => crate::types::sync, - crate::util::tokio => crate::util::sync, - crate::writer::tokio => crate::writer::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::reader::asynch => crate::reader::synch, + crate::types::asynch => crate::types::synch, + crate::util::asynch => crate::util::synch, + crate::writer::asynch => crate::writer::synch, #[tokio::test] => #[test] ); } @@ -556,15 +556,15 @@ mod schema_compatibility { use super::*; use crate::{ codec::Codec, - reader::tokio::Reader, - schema::tokio::SchemaExt, + reader::asynch::Reader, + schema::asynch::SchemaExt, types::{Record, Value}, - writer::tokio::Writer, + writer::asynch::Writer, }; use apache_avro_test_helper::TestResult; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::StreamExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use rstest::*; async fn int_array_schema() -> Schema { @@ -817,7 +817,7 @@ mod schema_compatibility { ); } - #[synca::cfg(sync)] + #[synca::cfg(synch)] #[rstest] // Record type test #[case( @@ -896,7 +896,7 @@ mod schema_compatibility { assert!(SchemaCompatibility::match_schemas(&writer_schema, &reader_schema).is_ok()); } - #[synca::cfg(sync)] + #[synca::cfg(synch)] #[rstest] // Record type test #[case( diff --git a/avro/src/schema_equality.rs b/avro/src/schema_equality.rs index ccaf82c7..d6ee7f4e 100644 --- a/avro/src/schema_equality.rs +++ b/avro/src/schema_equality.rs @@ -16,21 +16,21 @@ // under the License. // #[synca::synca( -// #[cfg(feature = "async")] -// pub mod tokio {}, -// #[cfg(feature = "sync")] -// pub mod sync { +// #[cfg(feature = "asynch")] +// pub mod asynch {}, +// #[cfg(feature = "synch")] +// pub mod synch { // sync!(); // replace!( -// crate::bigdecimal::tokio => crate::bigdecimal::sync, -// crate::decode::tokio => crate::decode::sync, -// crate::encode::tokio => crate::encode::sync, -// crate::error::tokio => crate::error::sync, -// crate::schema::tokio => crate::schema::sync, -// crate::util::tokio => crate::util::sync, -// crate::types::tokio => crate::types::sync, -// crate::schema_equality::tokio => crate::schema_equality::sync, -// crate::util::tokio => crate::util::sync, +// crate::bigdecimal::asynch => crate::bigdecimal::synch, +// crate::decode::asynch => crate::decode::synch, +// crate::encode::asynch => crate::encode::synch, +// crate::error::asynch => crate::error::synch, +// crate::schema::asynch => crate::schema::synch, +// crate::util::asynch => crate::util::synch, +// crate::types::asynch => crate::types::synch, +// crate::schema_equality::asynch => crate::schema_equality::synch, +// crate::util::asynch => crate::util::synch, // #[tokio::test] => #[test] // ); // } diff --git a/avro/src/ser.rs b/avro/src/ser.rs index 4ea2c933..80c9d96a 100644 --- a/avro/src/ser.rs +++ b/avro/src/ser.rs @@ -18,20 +18,20 @@ //! Logic for serde-compatible serialization. #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::util::tokio => crate::util::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::util::asynch => crate::util::synch, #[tokio::test] => #[test] ); } diff --git a/avro/src/ser_schema.rs b/avro/src/ser_schema.rs index a095614f..8ad73502 100644 --- a/avro/src/ser_schema.rs +++ b/avro/src/ser_schema.rs @@ -19,8 +19,8 @@ //! which writes directly to a `Write` stream use crate::{ - bigdecimal::sync::big_decimal_as_bytes, - encode::sync::{encode_int, encode_long}, + bigdecimal::synch::big_decimal_as_bytes, + encode::synch::{encode_int, encode_long}, error::{Details, Error}, schema::{Name, Names, Namespace, RecordField, RecordSchema, Schema}, }; @@ -1765,7 +1765,7 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s #[cfg(test)] mod tests { use super::*; - use crate::schema::sync::SchemaExt; + use crate::schema::synch::SchemaExt; use crate::{ Days, Duration, Millis, Months, decimal::Decimal, error::Details, schema::ResolvedSchema, }; diff --git a/avro/src/types.rs b/avro/src/types.rs index 7d08ee74..07389d1e 100644 --- a/avro/src/types.rs +++ b/avro/src/types.rs @@ -279,21 +279,21 @@ to_value!(BigDecimal, Value::BigDecimal); to_value!(Duration, Value::Duration); #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::ser::tokio => crate::ser::sync, - crate::util::tokio => crate::util::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::ser::asynch => crate::ser::synch, + crate::util::asynch => crate::util::synch, #[tokio::test] => #[test] ); } @@ -303,12 +303,12 @@ mod types { use crate::AvroResult; use crate::{ Uuid, - bigdecimal::tokio::{deserialize_big_decimal, serialize_big_decimal}, + bigdecimal::asynch::{deserialize_big_decimal, serialize_big_decimal}, decimal::Decimal, duration::Duration, error::Details, error::Error, - schema::tokio::{RecordFieldExt, UnionSchemaExt}, + schema::asynch::{RecordFieldExt, UnionSchemaExt}, schema::{ DecimalSchema, EnumSchema, FixedSchema, Name, Namespace, Precision, RecordField, RecordSchema, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, @@ -1265,7 +1265,7 @@ mod types { // .map(|value| (key, value)) // }) // .collect::>(); - // #[synca::cfg(tokio)] + // #[synca::cfg(asynch)] // let resolved = futures::future::try_join_all(resolved).await?; Ok(Value::Map(resolved)) } @@ -1373,8 +1373,8 @@ mod types { duration::{Days, Millis, Months}, error::Details, schema::RecordFieldOrder, - schema::tokio::SchemaExt, - ser::tokio::Serializer, + schema::asynch::SchemaExt, + ser::asynch::Serializer, types::{Record, Value}, }; use apache_avro_test_helper::{ @@ -3001,7 +3001,7 @@ Field with name '"b"' is not a member of the map items"#, #[tokio::test] async fn test_avro_3460_validation_with_refs_real_struct() -> TestResult { - use crate::ser::tokio::Serializer; + use crate::ser::asynch::Serializer; use serde::Serialize; #[derive(Serialize, Clone)] @@ -3091,7 +3091,7 @@ Field with name '"b"' is not a member of the map items"#, } async fn avro_3674_with_or_without_namespace(with_namespace: bool) -> TestResult { - use crate::ser::tokio::Serializer; + use crate::ser::asynch::Serializer; use serde::Serialize; let schema_str = r#" diff --git a/avro/src/util.rs b/avro/src/util.rs index faf008c6..cb8675b7 100644 --- a/avro/src/util.rs +++ b/avro/src/util.rs @@ -106,38 +106,38 @@ impl MapHelper for Map { } #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::schema::tokio => crate::schema::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::util::tokio => crate::util::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::schema::asynch => crate::schema::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::util::asynch => crate::util::synch, #[tokio::test] => #[test] ); } )] mod util { - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncRead as AvroRead; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::AsyncReadExt; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncWrite as AvroWrite; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::AsyncWriteExt; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::future::TryFutureExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Read as AvroRead; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Write as AvroWrite; use crate::AvroResult; @@ -233,7 +233,7 @@ mod util { #[cfg(test)] mod tests { use super::*; - use crate::util::tokio::safe_len; + use crate::util::asynch::safe_len; use apache_avro_test_helper::TestResult; use pretty_assertions::assert_eq; diff --git a/avro/src/validator.rs b/avro/src/validator.rs index 723a1dcf..c8a962ba 100644 --- a/avro/src/validator.rs +++ b/avro/src/validator.rs @@ -16,22 +16,22 @@ // under the License. // #[synca::synca( -// #[cfg(feature = "async")] -// pub mod tokio { }, -// #[cfg(feature = "sync")] -// pub mod sync { +// #[cfg(feature = "asynch")] +// pub mod asynch { }, +// #[cfg(feature = "synch")] +// pub mod synch { // sync!(); // replace!( -// crate::bigdecimal::tokio => crate::bigdecimal::sync, -// crate::decode::tokio => crate::decode::sync, -// crate::encode::tokio => crate::encode::sync, -// crate::error::tokio => crate::error::sync, -// crate::schema::tokio => crate::schema::sync, -// crate::util::tokio => crate::util::sync, -// crate::types::tokio => crate::types::sync, -// crate::schema_equality::tokio => crate::schema_equality::sync, -// crate::util::tokio => crate::util::sync, -// crate::validator::tokio => crate::validator::sync, +// crate::bigdecimal::asynch => crate::bigdecimal::synch, +// crate::decode::asynch => crate::decode::synch, +// crate::encode::asynch => crate::encode::synch, +// crate::error::asynch => crate::error::synch, +// crate::schema::asynch => crate::schema::synch, +// crate::util::asynch => crate::util::synch, +// crate::types::asynch => crate::types::synch, +// crate::schema_equality::asynch => crate::schema_equality::synch, +// crate::util::asynch => crate::util::synch, +// crate::validator::asynch => crate::validator::synch, // #[tokio::test] => #[test] // ); // } diff --git a/avro/src/writer.rs b/avro/src/writer.rs index bcb3874f..803c53e2 100644 --- a/avro/src/writer.rs +++ b/avro/src/writer.rs @@ -18,59 +18,59 @@ //! Logic handling writing in Avro format at user level. #[synca::synca( - #[cfg(feature = "async")] - pub mod tokio { }, - #[cfg(feature = "sync")] - pub mod sync { + #[cfg(feature = "asynch")] + pub mod asynch { }, + #[cfg(feature = "synch")] + pub mod synch { sync!(); replace!( - crate::bigdecimal::tokio => crate::bigdecimal::sync, - crate::decode::tokio => crate::decode::sync, - crate::encode::tokio => crate::encode::sync, - crate::error::tokio => crate::error::sync, - crate::headers::tokio => crate::headers::sync, - crate::schema::tokio => crate::schema::sync, - crate::ser_schema::tokio => crate::ser_schema::sync, - crate::reader::tokio => crate::reader::sync, - crate::util::tokio => crate::util::sync, - crate::types::tokio => crate::types::sync, - crate::util::tokio => crate::util::sync, + crate::bigdecimal::asynch => crate::bigdecimal::synch, + crate::decode::asynch => crate::decode::synch, + crate::encode::asynch => crate::encode::synch, + crate::error::asynch => crate::error::synch, + crate::headers::asynch => crate::headers::synch, + crate::schema::asynch => crate::schema::synch, + crate::ser_schema::asynch => crate::ser_schema::synch, + crate::reader::asynch => crate::reader::synch, + crate::util::asynch => crate::util::synch, + crate::types::asynch => crate::types::synch, + crate::util::asynch => crate::util::synch, #[tokio::test] => #[test] ); } )] mod writer { - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::AsyncWrite as AvroWrite; - #[cfg(feature = "async")] + #[cfg(feature = "asynch")] use futures::AsyncWriteExt; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::io::Write as AvroWrite; use std::marker::Unpin; use crate::AvroResult; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use crate::schema::Name; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use crate::ser_schema::SchemaAwareWriteSerializer; - use crate::types::tokio::ValueExt; + use crate::types::asynch::ValueExt; use crate::{ codec::Codec, - encode::tokio::{encode, encode_internal, encode_to_vec}, + encode::asynch::{encode, encode_internal, encode_to_vec}, error::Details, error::Error, - headers::tokio::{HeaderBuilder, RabinFingerprintHeader}, + headers::asynch::{HeaderBuilder, RabinFingerprintHeader}, schema::{ResolvedOwnedSchema, ResolvedSchema, Schema}, types::Value, }; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use serde::Serialize; use std::{collections::HashMap, mem::ManuallyDrop, ops::RangeInclusive}; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use crate::AvroSchema; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use std::marker::PhantomData; const DEFAULT_BLOCK_SIZE: usize = 16000; @@ -250,7 +250,7 @@ mod writer { /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on /// internal buffering for performance reasons. If you want to be sure the value has been /// written, then call [`flush`](Writer::flush). - #[synca::cfg(sync)] + #[synca::cfg(synch)] pub fn append_ser(&mut self, value: S) -> AvroResult { let n = self.maybe_write_header()?; @@ -321,7 +321,7 @@ mod writer { /// /// **NOTE**: This function forces the written data to be flushed (an implicit /// call to [`flush`](Writer::flush) is performed). - #[synca::cfg(sync)] + #[synca::cfg(synch)] pub fn extend_ser(&mut self, values: I) -> AvroResult where I: IntoIterator, @@ -541,7 +541,7 @@ mod writer { } } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl Drop for Writer<'_, W> { /// Drop the writer, will try to flush ignoring any errors. fn drop(&mut self) { @@ -652,7 +652,7 @@ mod writer { } /// Writer that encodes messages according to the single object encoding v1 spec - #[synca::cfg(sync)] + #[synca::cfg(synch)] pub struct SpecificSingleObjectWriter where T: AvroSchema, @@ -663,7 +663,7 @@ mod writer { _model: PhantomData, } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl SpecificSingleObjectWriter where T: AvroSchema, @@ -679,7 +679,7 @@ mod writer { } } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl SpecificSingleObjectWriter where T: AvroSchema + Into, @@ -696,7 +696,7 @@ mod writer { } } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl SpecificSingleObjectWriter where T: AvroSchema + Serialize, @@ -816,7 +816,7 @@ mod writer { /// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync /// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible /// if you don't know what you are doing, instead. - #[synca::cfg(sync)] + #[synca::cfg(synch)] pub fn write_avro_datum_ref( schema: &Schema, data: &T, @@ -866,27 +866,27 @@ mod writer { #[cfg(test)] mod tests { use super::*; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use crate::schema::AvroSchema; use crate::{codec::DeflateSettings, error::Details}; use crate::{ decimal::Decimal, duration::{Days, Duration, Millis, Months}, - reader::tokio::{Reader, from_avro_datum}, - schema::tokio::SchemaExt, + reader::asynch::{Reader, from_avro_datum}, + schema::asynch::SchemaExt, schema::{DecimalSchema, FixedSchema, Name}, types::Record, - util::tokio::zig_i64, + util::asynch::zig_i64, }; - #[synca::cfg(sync)] - use crate::{headers::tokio::GlueSchemaUuidHeader, rabin::Rabin}; + #[synca::cfg(synch)] + use crate::{headers::asynch::GlueSchemaUuidHeader, rabin::Rabin}; use apache_avro_test_helper::TestResult; - #[synca::cfg(tokio)] + #[synca::cfg(asynch)] use futures::StreamExt; use pretty_assertions::assert_eq; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use serde::{Deserialize, Serialize}; - #[synca::cfg(sync)] + #[synca::cfg(synch)] use uuid::Uuid; const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len(); @@ -929,7 +929,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn avro_rs_193_write_avro_datum_ref() -> TestResult { #[derive(Serialize)] struct TestStruct { @@ -1226,7 +1226,7 @@ mod writer { Ok(()) } - #[synca::cfg(sync)] + #[synca::cfg(synch)] #[derive(Debug, Clone, Deserialize, Serialize)] struct TestSerdeSerialize { a: i64, @@ -1234,7 +1234,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn test_writer_append_ser() -> TestResult { let schema = SchemaExt::parse_str(SCHEMA)?; let mut writer = Writer::new(&schema, Vec::new()); @@ -1268,7 +1268,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn test_writer_extend_ser() -> TestResult { let schema = SchemaExt::parse_str(SCHEMA).await?; let mut writer = Writer::new(&schema, Vec::new()); @@ -1551,7 +1551,7 @@ mod writer { Ok(()) } - #[synca::cfg(sync)] + #[synca::cfg(synch)] #[derive(Serialize, Clone)] struct TestSingleObjectWriter { a: i64, @@ -1559,7 +1559,7 @@ mod writer { c: Vec, } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl AvroSchema for TestSingleObjectWriter { fn get_schema() -> Schema { let schema = r#" @@ -1589,7 +1589,7 @@ mod writer { } } - #[synca::cfg(sync)] + #[synca::cfg(synch)] impl From for Value { fn from(obj: TestSingleObjectWriter) -> Value { Value::Record(vec![ @@ -1603,7 +1603,7 @@ mod writer { } } - #[synca::cfg(sync)] + #[synca::cfg(synch)] #[test] fn test_single_object_writer() -> TestResult { let mut buf: Vec = Vec::new(); @@ -1645,7 +1645,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] async fn test_single_object_writer_with_header_builder() -> TestResult { let mut buf: Vec = Vec::new(); let obj = TestSingleObjectWriter { @@ -1673,7 +1673,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn test_writer_parity() -> TestResult { let obj1 = TestSingleObjectWriter { a: 300, @@ -1710,7 +1710,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult { const SCHEMA: &str = r#" { @@ -1744,7 +1744,7 @@ mod writer { } #[test] - #[synca::cfg(sync)] + #[synca::cfg(synch)] fn avro_4014_validation_returns_a_detailed_error() -> TestResult { const SCHEMA: &str = r#" { @@ -1783,7 +1783,7 @@ mod writer { } } - // #[cfg(feature = "sync")] + // #[cfg(feature = "synch")] // #[test] // fn avro_4063_flush_applies_to_inner_writer() -> TestResult { // const SCHEMA: &str = r#" @@ -1796,8 +1796,8 @@ mod writer { // } // "#; // use std::{cell::RefCell, rc::Rc}; - // use crate::writer::sync::Writer; - // use crate::schema::sync::SchemaExt; + // use crate::writer::synch::Writer; + // use crate::schema::synch::SchemaExt; // // #[derive(Clone, Default)] // struct TestBuffer(Rc>>); diff --git a/avro/tests/async_schema.rs b/avro/tests/async_schema.rs index 1b17a300..0b04f11f 100644 --- a/avro/tests/async_schema.rs +++ b/avro/tests/async_schema.rs @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::types::tokio::ValueExt; +use apache_avro::types::asynch::ValueExt; use apache_avro::{ AsyncReader as Reader, AsyncWriter as Writer, Codec, async_from_avro_datum as from_avro_datum, async_from_value as from_value, async_to_avro_datum as to_avro_datum, async_to_value as to_value, error::{Details, Error}, - schema::tokio::SchemaExt, + schema::asynch::SchemaExt, schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema, Schema}, types::{Record, Value}, }; diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs index 91ef5b07..e8d1dff4 100644 --- a/avro/tests/avro-3786.rs +++ b/avro/tests/avro-3786.rs @@ -16,7 +16,7 @@ // under the License. use apache_avro::types::Value; -use apache_avro::types::sync::ValueExt; +use apache_avro::types::synch::ValueExt; use apache_avro::{SchemaExt, from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs index 85a4e690..f4aea1e9 100644 --- a/avro/tests/avro-3787.rs +++ b/avro/tests/avro-3787.rs @@ -17,7 +17,7 @@ use apache_avro::SchemaExt; use apache_avro::types::Value; -use apache_avro::types::sync::ValueExt; +use apache_avro::types::synch::ValueExt; use apache_avro::{from_avro_datum, to_avro_datum, to_value}; use apache_avro_test_helper::TestResult; diff --git a/avro/tests/io.rs b/avro/tests/io.rs index 281c7fbb..56aa2f17 100644 --- a/avro/tests/io.rs +++ b/avro/tests/io.rs @@ -16,7 +16,7 @@ // under the License. //! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py -use apache_avro::types::sync::ValueExt; +use apache_avro::types::synch::ValueExt; use apache_avro::{ Error, Schema, SchemaExt, error::Details, from_avro_datum, to_avro_datum, types::Value, }; diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs index 6ed97b1a..a3e3f32d 100644 --- a/avro/tests/schema.rs +++ b/avro/tests/schema.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use apache_avro::types::sync::ValueExt; +use apache_avro::types::synch::ValueExt; use apache_avro::{ Codec, Reader, SchemaExt, Writer, error::{Details, Error}, diff --git a/avro_derive/Cargo.toml b/avro_derive/Cargo.toml index 7ea66d29..f9a0f40b 100644 --- a/avro_derive/Cargo.toml +++ b/avro_derive/Cargo.toml @@ -40,7 +40,7 @@ syn = { default-features = false, version = "2.0.106", features = ["full", "fold [dev-dependencies] apache-avro = { default-features = false, path = "../avro", features = [ - "derive", "sync" + "derive", "synch" ] } proptest = { default-features = false, version = "1.7.0", features = ["std"] } serde = { workspace = true } diff --git a/avro_derive/tests/derive.rs b/avro_derive/tests/derive.rs index ccf0ef7b..f30b08ea 100644 --- a/avro_derive/tests/derive.rs +++ b/avro_derive/tests/derive.rs @@ -20,7 +20,7 @@ mod test_derive { use apache_avro::{ AvroSchema, Reader, Schema, Writer, from_value, schema::derive::AvroSchemaComponent, - schema::sync::SchemaExt, + schema::synch::SchemaExt, }; // use apache_avro_derive::*; use proptest::prelude::*; diff --git a/wasm-demo/Cargo.toml b/wasm-demo/Cargo.toml index c79a204e..7438ead6 100644 --- a/wasm-demo/Cargo.toml +++ b/wasm-demo/Cargo.toml @@ -34,7 +34,7 @@ publish = false crate-type = ["cdylib", "rlib"] [dependencies] -apache-avro = { path = "../avro", default-features = false, features = ["sync"] } +apache-avro = { path = "../avro", default-features = false, features = ["synch"] } serde = { workspace = true } wasm-bindgen = "0.2.97"