diff --git a/mithril-client/.gitignore b/mithril-client/.gitignore index 3e410c824b0..729f94331b4 100644 --- a/mithril-client/.gitignore +++ b/mithril-client/.gitignore @@ -1,5 +1,4 @@ data/ target/ mithril-client -.DS_Store -tests-work-folder/ \ No newline at end of file +.DS_Store \ No newline at end of file diff --git a/mithril-common/.gitignore b/mithril-common/.gitignore index 41e91754f8b..e08c79410a5 100644 --- a/mithril-common/.gitignore +++ b/mithril-common/.gitignore @@ -1,3 +1,2 @@ target/ -tests-work-folder/ .DS_Store diff --git a/mithril-common/src/digesters/digester.rs b/mithril-common/src/digesters/digester.rs index e639b1ff10f..08955ebab52 100644 --- a/mithril-common/src/digesters/digester.rs +++ b/mithril-common/src/digesters/digester.rs @@ -1,3 +1,4 @@ +use crate::digesters::ImmutableFileListingError; use crate::entities::ImmutableFileNumber; use std::io; use thiserror::Error; @@ -13,8 +14,8 @@ pub struct DigesterResult { #[derive(Error, Debug)] pub enum DigesterError { - #[error("Immutable files listing failed: `{0}`")] - ListImmutablesError(String), + #[error("Immutable files listing failed")] + ListImmutablesError(#[from] ImmutableFileListingError), #[error("At least two immutables chunk should exists")] NotEnoughImmutable(), diff --git a/mithril-common/src/digesters/immutable_digester.rs b/mithril-common/src/digesters/immutable_digester.rs index 1b95243f0e6..6d7212a2cb9 100644 --- a/mithril-common/src/digesters/immutable_digester.rs +++ b/mithril-common/src/digesters/immutable_digester.rs @@ -1,14 +1,11 @@ +use crate::digesters::immutable_file::ImmutableFile; use crate::digesters::{Digester, DigesterError, DigesterResult}; -use crate::entities::ImmutableFileNumber; use sha2::{Digest, Sha256}; use slog::{debug, info, Logger}; -use std::ffi::OsStr; use std::fs::File; use std::io; -use std::path::{Path, PathBuf}; - -use walkdir::WalkDir; +use std::path::PathBuf; /// A digester working directly on a Cardano DB immutables files pub struct ImmutableDigester { @@ -51,8 +48,7 @@ impl ImmutableDigester { impl Digester for ImmutableDigester { fn compute_digest(&self) -> Result { - let immutables = ImmutableFile::list_in_dir(&*self.db_directory) - .map_err(DigesterError::ListImmutablesError)?; + let immutables = ImmutableFile::list_completed_in_dir(&*self.db_directory)?; let last_immutable = immutables .last() .ok_or(DigesterError::NotEnoughImmutable())?; @@ -95,90 +91,9 @@ impl std::fmt::Display for Progress { } } -fn is_immutable(path: &Path) -> bool { - let immutable = OsStr::new("immutable"); - path.iter().any(|component| component == immutable) -} - -#[derive(Debug)] -struct ImmutableFile { - path: PathBuf, - number: ImmutableFileNumber, -} - -impl ImmutableFile { - fn new(path: PathBuf) -> Result { - let filename = path - .file_stem() - .ok_or(format!("Couldn't extract the file stem for '{:?}'", path))?; - let filename = filename.to_str().ok_or(format!( - "Couldn't extract the filename as string for '{:?}'", - path - ))?; - let immutable_file_number = filename - .parse::() - .map_err(|e| e.to_string())?; - - Ok(Self { - path, - number: immutable_file_number, - }) - } - - /// List all [`ImmutableFile`] in a given directory. - /// - /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet - /// complete. - fn list_in_dir(dir: &Path) -> Result, String> { - let mut files: Vec = vec![]; - - for path in WalkDir::new(dir) - .into_iter() - .filter_map(|file| file.ok()) - .map(|f| f.path().to_owned()) - { - let metadata = path.metadata().map_err(|e| e.to_string())?; - if metadata.is_file() && is_immutable(&path) { - let immutable_file = ImmutableFile::new(path)?; - files.push(immutable_file); - } - } - files.sort_by(|left, right| left.number.cmp(&right.number)); - - Ok(files.into_iter().rev().skip(3).rev().collect()) - } -} - #[cfg(test)] mod tests { - use super::*; - use std::fs; - use std::io::prelude::*; - - const ROOT_FOLDER: &str = "tests-work-folder/"; - - fn get_test_dir(subdir_name: &str) -> PathBuf { - let parent_dir = format!("{}{}", ROOT_FOLDER, subdir_name); - let parent_dir = Path::new(&parent_dir).to_path_buf(); - - if parent_dir.exists() { - fs::remove_dir_all(&parent_dir) - .expect(&*format!("Could not remove dir {:?}", parent_dir)); - } - fs::create_dir_all(&parent_dir).expect(&*format!("Could not create dir {:?}", parent_dir)); - - parent_dir - } - - fn create_fake_files(parent_dir: &Path, child_filenames: &[&str]) { - for filename in child_filenames { - let file = parent_dir.join(Path::new(filename)); - let mut source_file = File::create(&file).unwrap(); - write!(source_file, "This is a test file named '{}'", filename).unwrap(); - } - } - - // TODO: Test the case where number of immutable files is lower than 20 + use super::Progress; #[test] fn reports_progress_every_5_percent() { @@ -194,54 +109,15 @@ mod tests { } #[test] - fn list_immutable_file_should_skip_last_number() { - let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable"); - let entries = vec![ - "123.chunk", - "123.primary", - "123.secondary", - "125.chunk", - "125.primary", - "125.secondary", - "0124.chunk", - "0124.primary", - "0124.secondary", - "223.chunk", - "223.primary", - "223.secondary", - "0423.chunk", - "0423.primary", - "0423.secondary", - "0424.chunk", - "0424.primary", - "0424.secondary", - "21.chunk", - "21.primary", - "21.secondary", - ]; - create_fake_files(&target_dir, &entries); - let result = ImmutableFile::list_in_dir(target_dir.parent().unwrap()) - .expect("ImmutableFile::list_in_dir Failed"); - - assert_eq!(result.last().unwrap().number, 423); - assert_eq!( - result.len(), - entries.len() - 3, - "Expected to find {} files since The last (chunk, primary, secondary) trio is skipped, but found {}", - entries.len() - 3, - result.len(), - ); - } + fn reports_progress_when_total_lower_than_20() { + let mut progress = Progress { + index: 0, + total: 16, + }; - #[test] - fn list_immutable_file_should_works_in_a_empty_folder() { - let target_dir = - get_test_dir("list_immutable_file_should_works_even_in_a_empty_folder/immutable"); - let entries = vec![]; - create_fake_files(&target_dir, &entries); - let result = ImmutableFile::list_in_dir(target_dir.parent().unwrap()) - .expect("ImmutableFile::list_in_dir Failed"); - - assert!(result.is_empty()); + assert!(progress.report(4)); + assert!(progress.report(12)); + assert!(!progress.report(3)); + assert!(!progress.report(15)); } } diff --git a/mithril-common/src/digesters/immutable_file.rs b/mithril-common/src/digesters/immutable_file.rs new file mode 100644 index 00000000000..bdc85d062a8 --- /dev/null +++ b/mithril-common/src/digesters/immutable_file.rs @@ -0,0 +1,160 @@ +use crate::entities::ImmutableFileNumber; + +use std::ffi::OsStr; +use std::io; +use std::num::ParseIntError; +use std::path::{Path, PathBuf}; +use thiserror::Error; +use walkdir::WalkDir; + +fn is_immutable(path: &Path) -> bool { + let immutable = OsStr::new("immutable"); + path.iter().any(|component| component == immutable) +} + +#[derive(Debug)] +pub struct ImmutableFile { + pub path: PathBuf, + pub number: ImmutableFileNumber, +} + +#[derive(Error, Debug)] +pub enum ImmutableFileCreationError { + #[error("Couldn't extract the file stem for '{path:?}'")] + FileStemExtraction { path: PathBuf }, + #[error("Couldn't extract the filename as string for '{path:?}'")] + FileNameExtraction { path: PathBuf }, + #[error("Error while parsing immutable file number")] + FileNumberParsing(#[from] ParseIntError), +} + +#[derive(Error, Debug)] +pub enum ImmutableFileListingError { + #[error("metadata parsing failed")] + MetadataParsing(#[from] io::Error), + #[error("immutable file creation error")] + ImmutableFileCreation(#[from] ImmutableFileCreationError), +} + +impl ImmutableFile { + pub fn new(path: PathBuf) -> Result { + let filename = path + .file_stem() + .ok_or(ImmutableFileCreationError::FileStemExtraction { path: path.clone() })?; + let filename = filename + .to_str() + .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?; + let immutable_file_number = filename.parse::()?; + + Ok(Self { + path, + number: immutable_file_number, + }) + } + + /// List all [`ImmutableFile`] in a given directory. + /// + /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet + /// complete. + pub fn list_completed_in_dir( + dir: &Path, + ) -> Result, ImmutableFileListingError> { + let mut files: Vec = vec![]; + + for path in WalkDir::new(dir) + .into_iter() + .filter_map(|file| file.ok()) + .map(|f| f.path().to_owned()) + { + let metadata = path.metadata()?; + if metadata.is_file() && is_immutable(&path) { + let immutable_file = ImmutableFile::new(path)?; + files.push(immutable_file); + } + } + files.sort_by(|left, right| left.number.cmp(&right.number)); + + Ok(files.into_iter().rev().skip(3).rev().collect()) + } +} + +#[cfg(test)] +mod tests { + use super::ImmutableFile; + use std::fs; + use std::fs::File; + use std::io::prelude::*; + use std::path::{Path, PathBuf}; + + fn get_test_dir(subdir_name: &str) -> PathBuf { + let parent_dir = std::env::temp_dir().join("mithril_test").join(subdir_name); + + if parent_dir.exists() { + fs::remove_dir_all(&parent_dir) + .expect(&*format!("Could not remove dir {:?}", parent_dir)); + } + fs::create_dir_all(&parent_dir).expect(&*format!("Could not create dir {:?}", parent_dir)); + + parent_dir + } + + fn create_fake_files(parent_dir: &Path, child_filenames: &[&str]) { + for filename in child_filenames { + let file = parent_dir.join(Path::new(filename)); + let mut source_file = File::create(&file).unwrap(); + write!(source_file, "This is a test file named '{}'", filename).unwrap(); + } + } + + #[test] + fn list_immutable_file_should_skip_last_number() { + let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable"); + let entries = vec![ + "123.chunk", + "123.primary", + "123.secondary", + "125.chunk", + "125.primary", + "125.secondary", + "0124.chunk", + "0124.primary", + "0124.secondary", + "223.chunk", + "223.primary", + "223.secondary", + "0423.chunk", + "0423.primary", + "0423.secondary", + "0424.chunk", + "0424.primary", + "0424.secondary", + "21.chunk", + "21.primary", + "21.secondary", + ]; + create_fake_files(&target_dir, &entries); + let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap()) + .expect("ImmutableFile::list_in_dir Failed"); + + assert_eq!(result.last().unwrap().number, 423); + assert_eq!( + result.len(), + entries.len() - 3, + "Expected to find {} files since The last (chunk, primary, secondary) trio is skipped, but found {}", + entries.len() - 3, + result.len(), + ); + } + + #[test] + fn list_immutable_file_should_works_in_a_empty_folder() { + let target_dir = + get_test_dir("list_immutable_file_should_works_even_in_a_empty_folder/immutable"); + let entries = vec![]; + create_fake_files(&target_dir, &entries); + let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap()) + .expect("ImmutableFile::list_in_dir Failed"); + + assert!(result.is_empty()); + } +} diff --git a/mithril-common/src/digesters/mod.rs b/mithril-common/src/digesters/mod.rs index a695164ca43..a7f79831bcc 100644 --- a/mithril-common/src/digesters/mod.rs +++ b/mithril-common/src/digesters/mod.rs @@ -1,5 +1,7 @@ mod digester; mod immutable_digester; +mod immutable_file; pub use digester::{Digester, DigesterError, DigesterResult}; pub use immutable_digester::ImmutableDigester; +pub use immutable_file::{ImmutableFile, ImmutableFileCreationError, ImmutableFileListingError};