Add compression + API fixes

Added support for compression and decompression (feature gated by the "compression" feature which is
enabled by default).

- LZMA, LZ4 and Zlib compression are now all supported for base64 encoded appended data blobs.
- Compression level is currently ignored on LZ4 until either the `lz4_flex` crate implements
  support, or the `lz4` crate supports LZ4 block format.
- Binary appended data blobs are currently not supported until
  [#253](tafia/quick-xml#253) is merged into the `quick-xml` crate.
- Note that solutions to either of the above problems should only cause a minor version bumb.

Also The VTK file API was changed to include an optional `file_path`,
which encodes the original path to the VTK file. This allows relative
paths when reading in "parallel" XML files. This is how ParaView deals
with "parallel" XML files for instance. Note that the "parallel" files
refers to how they are defined in the VTK documentation; async file
loading is not yet supprted, but it is planned.

bytemuck is not made a mandatory dependency to fix non xml build.
The xml feature flag is also now used to disable all xml features in the
non-xml build.

Added a load_all_pieces function to simplify loading parallel xml VTK
files.

Renamed `load_piece_data` to `into_loaded_piece_data` and added a
`load_piece_data_in_place`.

Added a `try_into_xml_format` function to allow compression before
exporting.

CHANGELOG.md

@@ -1,11 +1,30 @@
+# CHANGELOG
+
+This document outlines changes and updates in major releases of `vtkio`.
+
 # Release 0.6
 
-Another small update to make API function names more consistent throughout the library (more
+Make API function names more consistent throughout the library (more
 specifically between parse and write functions).
 
 - `parse_vtk_{be|le}` is renamed to `parse_legacy_{be|le}`.
 - `parse_vtk_buf_{be|le}` is renamed to `parse_legacy_buf_{be|le}`.
 
+Add support for compression and decompression (feature gated by the "compression" feature which is
+enabled by default).
+
+- LZMA, LZ4 and Zlib compression are now all supported for base64 encoded appended data blobs.
+- Compression level is currently ignored on LZ4 until either the `lz4_flex` crate implements
+  support, or the `lz4` crate supports LZ4 block format.
+- Binary appended data blobs are currently not supported until
+  [#253](https://github.com/tafia/quick-xml/pull/253) is merged into the `quick-xml` crate.
+- Note that solutions to either of the above problems should only cause a minor version bumb.
+
+The VTK file API was changed to include an optional `file_path`, which encodes the original path to the
+VTK file. This allows relative paths when reading in "parallel" XML files. This is how
+ParaView deals with "parallel" XML files for instance. Note that the "parallel" files refers to how
+they are defined in the VTK documentation; async file loading is not yet supprted, but it is planned.
+
 There are also additional fixes and docs clarifications.
 
 - Fixed how leading bytes are used to specify attribute data arrays.

Cargo.toml

@@ -25,13 +25,17 @@ num-traits = "0.2"
 num-derive = "0.3"
 byteorder = "1.3"
 base64 = "0.12"
-bytemuck = { version = "1.4", features = ["extern_crate_alloc"], optional = true }
+bytemuck = { version = "1.5", features = ["extern_crate_alloc"] }
+lz4 = { package = "lz4_flex", version = "0.7", optional = true }
+flate2 = { version = "1.0.19", optional = true }
+xz2 = { version = "0.1.6", optional = true }
 quick-xml = { version = "0.20", features = ["serialize"], optional = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
 tokio = { version = "0.3", features = ["fs", "io-util"], optional = true }
 
 [features]
-default = ["xml"]
+default = ["xml", "compression"]
 async = ["tokio"]
-xml = ["quick-xml", "serde", "bytemuck"]
+compression = ["lz4", "xz2", "flate2"]
+xml = ["quick-xml", "serde"]
 unstable = []

assets/hexahedron_parallel.pvtu

@@ -0,0 +1,8 @@
+<VTKFile type="PUnstructuredGrid" version="1.0" byte_order="LittleEndian" header_type="UInt64">
+  <PUnstructuredGrid GhostLevel="1">
+    <PPoints>
+      <PDataArray type="Float32" Name="Points" NumberOfComponents="3"/>
+    </PPoints>
+    <Piece Source="hexahedron_parallel_0.vtu"/>
+  </PUnstructuredGrid>
+</VTKFile>

assets/cube_compressed.pvtu → assets/hexahedron_parallel_lzma.pvtu

@@ -1,9 +1,9 @@
 <?xml version="1.0"?>
-<VTKFile type="PUnstructuredGrid" version="1.0" byte_order="LittleEndian" header_type="UInt64" compressor="vtkZLibDataCompressor">
+<VTKFile type="PUnstructuredGrid" version="1.0" byte_order="LittleEndian" header_type="UInt32" compressor="vtkLZMADataCompressor">
   <PUnstructuredGrid GhostLevel="1">
     <PPoints>
       <PDataArray type="Float32" Name="Points" NumberOfComponents="3"/>
     </PPoints>
-    <Piece Source="cube_compressed/cube_compressed_0.vtu"/>
+    <Piece Source="hexahedron_parallel_lzma_0.vtu"/>
   </PUnstructuredGrid>
 </VTKFile>

assets/hexahedron_parallel_lzma_0.vtu

@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<VTKFile type="UnstructuredGrid" version="1.0" byte_order="LittleEndian" header_type="UInt32" compressor="vtkLZMADataCompressor">
+  <UnstructuredGrid>
+    <Piece NumberOfPoints="8"                    NumberOfCells="1"                   >
+      <PointData>
+      </PointData>
+      <CellData>
+      </CellData>
+      <Points>
+        <DataArray type="Float32" Name="Points" NumberOfComponents="3" format="appended" RangeMin="0"                    RangeMax="1.7320508076"         offset="0"                   >
+          <InformationKey name="L2_NORM_FINITE_RANGE" location="vtkDataArray" length="2">
+            <Value index="0">
+              0
+            </Value>
+            <Value index="1">
+              1.7320508076
+            </Value>
+          </InformationKey>
+          <InformationKey name="L2_NORM_RANGE" location="vtkDataArray" length="2">
+            <Value index="0">
+              0
+            </Value>
+            <Value index="1">
+              1.7320508076
+            </Value>
+          </InformationKey>
+        </DataArray>
+      </Points>
+      <Cells>
+        <DataArray type="Int64" Name="connectivity" format="appended" RangeMin=""                     RangeMax=""                     offset="132"                 />
+        <DataArray type="Int64" Name="offsets" format="appended" RangeMin=""                     RangeMax=""                     offset="260"                 />
+        <DataArray type="UInt8" Name="types" format="appended" RangeMin=""                     RangeMax=""                     offset="364"                 />
+      </Cells>
+    </Piece>
+  </UnstructuredGrid>
+  <AppendedData encoding="base64">
+   _AQAAAACAAABgAAAAUAAAAA==/Td6WFoAAAFpIt42AsAgYCEBHADVyr+K4ABfABhdAABuBMhFyK1W9uD9tSdenv6myo1MgjQtAAAt9fkxAAEwYIDicrKQQpkNAQAAAAABWVo=AQAAAACAAABAAAAATAAAAA==/Td6WFoAAAFpIt42AsAcQCEBHAAHIsY44AA/ABRdAABqf3sRcnVE9tuf42GTc5gyx1nDAFJBZZwAASxAFZ9rb5BCmQ0BAAAAAAFZWg==AQAAAACAAAAIAAAAPAAAAA==/Td6WFoAAAFpIt42AsAMCCEBHAAUM5NTAQAHCAAAAAAAAAAA3MTHtgABHAhEYCrIkEKZDQEAAAAAAVlaAQAAAACAAAABAAAAOAAAAA==/Td6WFoAAAFpIt42AsAFASEBHACtAIx5AQAADAAAAACmo7TbAAEVAaljNGCQQpkNAQAAAAABWVo=
+  </AppendedData>
+</VTKFile>

assets/cube_compressed/cube_compressed_0.vtu → assets/hexahedron_zlib.vtu

@@ -34,6 +34,6 @@
     </Piece>
   </UnstructuredGrid>
   <AppendedData encoding="base64">
-   _AQAAAAAAAAAAgAAAAAAAAGAAAAAAAAAAIAAAAAAAAAA=eJxjYMAGGvZDaXskMXuIOLoYTD1Y3h5JLVg9AHfjCvU=AQAAAAAAAAAAgAAAAAAAAEAAAAAAAAAAHgAAAAAAAAA=eJxjYIAAFijNCqUZoTQTlGaD0uxQmhlKAwADkAAdAQAAAAAAAAAAgAAAAAAAAAgAAAAAAAAACwAAAAAAAAA=eJzjYIAAAABIAAk=AQAAAAAAAAAAgAAAAAAAAAEAAAAAAAAACQAAAAAAAAA=eJzjAQAADQAN
+   _AQAAAAAAAAAAgAAAAAAAAGAAAAAAAAAAIAAAAAAAAAA=eNpjYMAGGvZDaXskMXuIOLoYTD1Y3h5JLVg9AHfjCvU=AQAAAAAAAAAAgAAAAAAAAEAAAAAAAAAAHgAAAAAAAAA=eNpjYIAAFijNCqUZoTQTlGaD0uxQmhlKAwADkAAdAQAAAAAAAAAAgAAAAAAAAAgAAAAAAAAACwAAAAAAAAA=eNrjYIAAAABIAAk=AQAAAAAAAAAAgAAAAAAAAAEAAAAAAAAACQAAAAAAAAA=eNrjAQAADQAN
   </AppendedData>
 </VTKFile>

src/lib.rs

@@ -59,11 +59,15 @@ pub mod basic;
 pub mod model;
 pub mod parser;
 pub mod writer;
+#[cfg(feature = "xml")]
 pub mod xml;
 
+#[cfg(feature = "xml")]
 use std::convert::{TryFrom, TryInto};
 use std::fs::File;
-use std::io::{self, BufRead, BufWriter, Read, Write};
+#[cfg(feature = "xml")]
+use std::io::BufRead;
+use std::io::{self, BufWriter, Read, Write};
 use std::path::Path;
 
 use crate::writer::{AsciiWriter, BinaryWriter, WriteVtk};
@@ -76,22 +80,26 @@ pub enum Error {
     IO(io::Error),
     Write(writer::Error),
     Parse(nom::ErrorKind<u32>),
+    #[cfg(feature = "xml")]
     XML(xml::Error),
     UnknownFileExtension(Option<String>),
+    Load(model::Error),
     Unknown,
 }
 
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
-            Error::IO(source) => write!(f, "IO error: {:?}", source),
-            Error::Write(source) => write!(f, "Write error: {:?}", source),
+            Error::IO(source) => write!(f, "IO error: {}", source),
+            Error::Write(source) => write!(f, "Write error: {}", source),
             Error::Parse(source) => write!(f, "Parse error: {:?}", source),
-            Error::XML(source) => write!(f, "XML error: {:?}", source),
+            #[cfg(feature = "xml")]
+            Error::XML(source) => write!(f, "XML error: {}", source),
             Error::UnknownFileExtension(Some(ext)) => {
                 write!(f, "Unknown file extension: {:?}", ext)
             }
             Error::UnknownFileExtension(None) => write!(f, "Missing file extension"),
+            Error::Load(source) => write!(f, "Load error: {}", source),
             Error::Unknown => write!(f, "Unknown error"),
         }
     }
@@ -103,8 +111,10 @@ impl std::error::Error for Error {
             Error::IO(source) => Some(source),
             Error::Write(_) => None, // TODO: implement std::error for writer::Error
             Error::Parse(_) => None,
+            #[cfg(feature = "xml")]
             Error::XML(source) => Some(source),
             Error::UnknownFileExtension(_) => None,
+            Error::Load(source) => Some(source),
             Error::Unknown => None,
         }
     }
@@ -120,6 +130,7 @@ impl From<io::Error> for Error {
 /// Convert [`xml::Error`] error into the top level `vtkio` error.
 ///
 /// [`xml::Error`]: xml.enum.Error.html
+#[cfg(feature = "xml")]
 impl From<xml::Error> for Error {
     fn from(e: xml::Error) -> Error {
         Error::XML(e)
@@ -197,6 +208,7 @@ where
 ///     version: Version::new((2,0)),
 ///     byte_order: ByteOrder::BigEndian, // This is default
 ///     title: String::from("Triangle example"),
+///     file_path: None,
 ///     data: DataSet::inline(PolyDataPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0].into(),
 ///         polys: Some(VertexNumbers::Legacy {
@@ -244,6 +256,7 @@ pub fn parse_legacy_be(reader: impl Read) -> Result<model::Vtk, Error> {
 ///     version: Version::new((2,0)),
 ///     byte_order: ByteOrder::BigEndian, // This is default
 ///     title: String::from("Triangle example"),
+///     file_path: None,
 ///     data: DataSet::inline(PolyDataPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0].into(),
 ///         polys: Some(VertexNumbers::Legacy {
@@ -314,6 +327,7 @@ pub fn parse_legacy_buf_le(reader: impl Read, buf: &mut Vec<u8>) -> Result<model
 ///     version: Version::new((2,0)),
 ///     byte_order: ByteOrder::BigEndian, // This is default
 ///     title: String::new(),
+///     file_path: None,
 ///     data: DataSet::inline(PolyDataPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0].into(),
 ///         polys: Some(VertexNumbers::XML {
@@ -325,6 +339,7 @@ pub fn parse_legacy_buf_le(reader: impl Read, buf: &mut Vec<u8>) -> Result<model
 ///     })
 /// });
 /// ```
+#[cfg(feature = "xml")]
 pub fn parse_xml(reader: impl BufRead) -> Result<model::Vtk, Error> {
     // There is no extension to check with the data is provided directly.
     // Luckily the xml file contains all the data necessary to determine which data is
@@ -395,6 +410,7 @@ fn import_impl(path: &Path) -> Result<model::Vtk, Error> {
         .ok_or(Error::UnknownFileExtension(None))?;
     match ext {
         "vtk" => import_vtk(path, parser::parse_be),
+        #[cfg(feature = "xml")]
         ext => {
             let ft = xml::FileType::try_from_ext(ext)
                 .ok_or(Error::UnknownFileExtension(Some(ext.to_string())))?;
@@ -403,9 +419,13 @@ fn import_impl(path: &Path) -> Result<model::Vtk, Error> {
             if ft != exp_ft {
                 Err(Error::XML(xml::Error::TypeExtensionMismatch))
             } else {
-                Ok(vtk_file.try_into()?)
+                let mut vtk: model::Vtk = vtk_file.try_into()?;
+                vtk.file_path = Some(path.into());
+                Ok(vtk)
             }
         }
+        #[cfg(not(feature = "xml"))]
+        _ => Err(Error::UnknownFileExtension(None)),
     }
 }
 
@@ -445,12 +465,10 @@ fn import_impl(path: &Path) -> Result<model::Vtk, Error> {
 #[cfg(feature = "async_blocked")]
 pub async fn import_async(file_path: impl AsRef<Path>) -> Result<model::Vtk, Error> {
     let path = file_path.as_ref();
-    let ext = path
-        .extension()
-        .and_then(|s| s.to_str())
-        .ok_or(Error::UnknownFileExtension(None))?;
+    let ext = path.extension().and_then(|s| s.to_str()).ok_or()?;
     match ext {
         "vtk" => import_vtk_async(path, parser::parse_be).await,
+        #[cfg(feature = "xml")]
         ext => {
             let ft = xml::FileType::try_from_ext(ext)
                 .ok_or(Error::UnknownFileExtension(Some(ext.to_string())))?;
@@ -462,6 +480,8 @@ pub async fn import_async(file_path: impl AsRef<Path>) -> Result<model::Vtk, Err
                 Ok(vtk_file.try_into()?)
             }
         }
+        #[cfg(not(feature = "xml"))]
+        _ => Err(Error::UnknownFileExtension(None)),
     }
 }
 
@@ -533,6 +553,7 @@ pub fn import_be(file_path: impl AsRef<Path>) -> Result<model::Vtk, Error> {
 ///     version: Version::new((4,1)),
 ///     byte_order: ByteOrder::BigEndian,
 ///     title: String::from("Tetrahedron"),
+///     file_path: Some(PathBuf::from("./test.vtk")),
 ///     data: DataSet::inline(UnstructuredGridPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 1.0, 0.0].into(),
 ///         cells: Cells {
@@ -566,6 +587,7 @@ fn export_impl(data: model::Vtk, path: &Path) -> Result<(), Error> {
             BinaryWriter(BufWriter::new(file)).write_vtk(data)?;
             Ok(())
         }
+        #[cfg(feature = "xml")]
         ext => {
             let ft = xml::FileType::try_from_ext(ext)
                 .ok_or(Error::UnknownFileExtension(Some(ext.to_string())))?;
@@ -578,6 +600,8 @@ fn export_impl(data: model::Vtk, path: &Path) -> Result<(), Error> {
                 Ok(())
             }
         }
+        #[cfg(not(feature = "xml"))]
+        _ => Err(Error::UnknownFileExtension(None)),
     }
 }
 
@@ -597,6 +621,7 @@ fn export_impl(data: model::Vtk, path: &Path) -> Result<(), Error> {
 ///     version: Version::new((2,0)),
 ///     byte_order: ByteOrder::BigEndian, // This is default
 ///     title: String::from("Triangle example"),
+///     file_path: None,
 ///     data: DataSet::inline(PolyDataPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0].into(),
 ///         polys: Some(VertexNumbers::Legacy {
@@ -630,6 +655,7 @@ pub fn write_legacy(vtk: model::Vtk, writer: impl std::io::Write) -> Result<(),
 ///     version: Version::new((2,0)),
 ///     byte_order: ByteOrder::BigEndian, // This is default
 ///     title: String::from("Triangle example"),
+///     file_path: None,
 ///     data: DataSet::inline(PolyDataPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0].into(),
 ///         polys: Some(VertexNumbers::Legacy {
@@ -680,6 +706,7 @@ pub fn write_legacy_ascii(vtk: model::Vtk, writer: impl std::fmt::Write) -> Resu
 ///     version: Version::new((2,0)),
 ///     byte_order: ByteOrder::BigEndian, // This is default
 ///     title: String::from("Triangle example"),
+///     file_path: None,
 ///     data: DataSet::inline(PolyDataPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0].into(),
 ///         polys: Some(VertexNumbers::Legacy {
@@ -714,6 +741,7 @@ pub fn write_legacy_ascii(vtk: model::Vtk, writer: impl std::fmt::Write) -> Resu
 ///   </PolyData>\
 /// </VTKFile>");
 /// ```
+#[cfg(feature = "xml")]
 pub fn write_xml(vtk: model::Vtk, writer: impl Write) -> Result<(), Error> {
     let vtk_file = xml::VTKFile::try_from(vtk)?;
     xml::write(&vtk_file, writer)?;
@@ -756,6 +784,7 @@ pub fn export_be(data: model::Vtk, file_path: impl AsRef<Path>) -> Result<(), Er
 ///     version: Version::new((4,1)),
 ///     title: String::from("Tetrahedron"),
 ///     byte_order: ByteOrder::BigEndian,
+///     file_path: Some(PathBuf::from("./test.vtk")),
 ///     data: DataSet::inline(UnstructuredGridPiece {
 ///         points: vec![0.0f32, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 1.0, 0.0].into(),
 ///         cells: Cells {