Merge pull request #884 from Mingun/eol-normalization

Proper EOL normalization

Author: Mingun

Changelog.md

@@ -22,11 +22,18 @@
   - `Deserializer::buffering_with_resolver`
 - [#878]: Add ability to serialize structs in `$value` fields. The struct name will
   be used as a tag name. Previously only enums was allowed there.
+- [#806]: Add `BytesText::xml_content`, `BytesCData::xml_content` and `BytesRef::xml_content`
+  methods which returns XML EOL normalized strings.
+- [#806]: Add `BytesText::html_content`, `BytesCData::html_content` and `BytesRef::html_content`
+  methods which returns HTML EOL normalized strings.
 
 ### Bug Fixes
 
+- [#806]: Properly normalize EOL characters in `Deserializer`.
+
 ### Misc Changes
 
+[#806]: https://github.com/tafia/quick-xml/issues/806
 [#878]: https://github.com/tafia/quick-xml/pull/878
 [#882]: https://github.com/tafia/quick-xml/pull/882
 

benches/macrobenches.rs

@@ -55,7 +55,7 @@ fn parse_document_from_str(doc: &str) -> XmlResult<()> {
                 }
             }
             Event::Text(e) => {
-                black_box(e.decode()?);
+                black_box(e.xml_content()?);
             }
             Event::CData(e) => {
                 black_box(e.into_inner());
@@ -80,7 +80,7 @@ fn parse_document_from_bytes(doc: &[u8]) -> XmlResult<()> {
                 }
             }
             Event::Text(e) => {
-                black_box(e.decode()?);
+                black_box(e.xml_content()?);
             }
             Event::CData(e) => {
                 black_box(e.into_inner());
@@ -106,7 +106,7 @@ fn parse_document_from_str_with_namespaces(doc: &str) -> XmlResult<()> {
                 }
             }
             (resolved_ns, Event::Text(e)) => {
-                black_box(e.decode()?);
+                black_box(e.xml_content()?);
                 black_box(resolved_ns);
             }
             (resolved_ns, Event::CData(e)) => {
@@ -134,7 +134,7 @@ fn parse_document_from_bytes_with_namespaces(doc: &[u8]) -> XmlResult<()> {
                 }
             }
             (resolved_ns, Event::Text(e)) => {
-                black_box(e.decode()?);
+                black_box(e.xml_content()?);
                 black_box(resolved_ns);
             }
             (resolved_ns, Event::CData(e)) => {

benches/microbenches.rs

@@ -146,7 +146,7 @@ fn one_event(c: &mut Criterion) {
             config.trim_text(true);
             config.check_end_names = false;
             match r.read_event() {
-                Ok(Event::Comment(e)) => nbtxt += e.decode().unwrap().len(),
+                Ok(Event::Comment(e)) => nbtxt += e.xml_content().unwrap().len(),
                 something_else => panic!("Did not expect {:?}", something_else),
             };
 

src/de/mod.rs

@@ -2439,8 +2439,8 @@ impl<'i, R: XmlRead<'i>, E: EntityResolver> XmlReader<'i, R, E> {
             }
 
             match self.next_impl()? {
-                PayloadEvent::Text(e) => result.to_mut().push_str(&e.decode()?),
-                PayloadEvent::CData(e) => result.to_mut().push_str(&e.decode()?),
+                PayloadEvent::Text(e) => result.to_mut().push_str(&e.xml_content()?),
+                PayloadEvent::CData(e) => result.to_mut().push_str(&e.xml_content()?),
                 PayloadEvent::GeneralRef(e) => self.resolve_reference(result.to_mut(), e)?,
 
                 // SAFETY: current_event_is_last_text checks that event is Text, CData or GeneralRef
@@ -2456,8 +2456,8 @@ impl<'i, R: XmlRead<'i>, E: EntityResolver> XmlReader<'i, R, E> {
             return match self.next_impl()? {
                 PayloadEvent::Start(e) => Ok(DeEvent::Start(e)),
                 PayloadEvent::End(e) => Ok(DeEvent::End(e)),
-                PayloadEvent::Text(e) => self.drain_text(e.decode()?),
-                PayloadEvent::CData(e) => self.drain_text(e.decode()?),
+                PayloadEvent::Text(e) => self.drain_text(e.xml_content()?),
+                PayloadEvent::CData(e) => self.drain_text(e.xml_content()?),
                 PayloadEvent::DocType(e) => {
                     self.entity_resolver
                         .capture(e)

src/encoding.rs

@@ -150,6 +150,30 @@ impl Decoder {
             Cow::Owned(bytes) => Ok(self.decode(bytes)?.into_owned().into()),
         }
     }
+
+    /// Decodes the `Cow` buffer, normalizes XML EOLs, preserves the lifetime
+    pub(crate) fn content<'b>(
+        &self,
+        bytes: &Cow<'b, [u8]>,
+        normalize_eol: impl Fn(&str) -> Cow<str>,
+    ) -> Result<Cow<'b, str>, EncodingError> {
+        match bytes {
+            Cow::Borrowed(bytes) => {
+                let text = self.decode(bytes)?;
+                match normalize_eol(&text) {
+                    // If text borrowed after normalization that means that it's not changed
+                    Cow::Borrowed(_) => Ok(text),
+                    Cow::Owned(s) => Ok(Cow::Owned(s)),
+                }
+            }
+            Cow::Owned(bytes) => {
+                let text = self.decode(bytes)?;
+                let text = normalize_eol(&text);
+                // Convert to owned, because otherwise Cow will be bound with wrong lifetime
+                Ok(text.into_owned().into())
+            }
+        }
+    }
 }
 
 /// Decodes the provided bytes using the specified encoding.