diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 3e83ff000c..1ba6b78afe 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -680,7 +680,7 @@ impl Chain { } // prepares the zip and return the corresponding Read - let txhashset_reader = txhashset::zip_read(self.db_root.clone(), &header, None)?; + let txhashset_reader = txhashset::zip_read(self.db_root.clone(), &header)?; Ok(( header.output_mmr_size, header.kernel_mmr_size, diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 51d7166af9..545c3a715e 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -32,12 +32,12 @@ use crate::util::secp::pedersen::{Commitment, RangeProof}; use crate::util::{file, secp_static, zip}; use croaring::Bitmap; use grin_store; -use grin_store::pmmr::{PMMRBackend, PMMR_FILES}; +use grin_store::pmmr::{clean_files_by_prefix, PMMRBackend, PMMR_FILES}; use std::collections::HashSet; use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::Instant; const HEADERHASHSET_SUBDIR: &'static str = "header"; const TXHASHSET_SUBDIR: &'static str = "txhashset"; @@ -1417,22 +1417,38 @@ impl<'a> Extension<'a> { /// Packages the txhashset data files into a zip and returns a Read to the /// resulting file -pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option) -> Result { - let ts = if let None = rand { - let now = SystemTime::now(); - now.duration_since(UNIX_EPOCH).unwrap().subsec_micros() - } else { - rand.unwrap() - }; - let txhashset_zip = format!("{}_{}.zip", TXHASHSET_ZIP, ts); +pub fn zip_read(root_dir: String, header: &BlockHeader) -> Result { + let txhashset_zip = format!("{}_{}.zip", TXHASHSET_ZIP, header.hash().to_string()); let txhashset_path = Path::new(&root_dir).join(TXHASHSET_SUBDIR); let zip_path = Path::new(&root_dir).join(txhashset_zip); - // create the zip archive - { + + // if file exist, just re-use it + let zip_file = File::open(zip_path.clone()); + if let Ok(zip) = zip_file { + return Ok(zip); + } else { + // clean up old zips. + // Theoretically, we only need clean-up those zip files older than STATE_SYNC_THRESHOLD. + // But practically, these zip files are not small ones, we just keep the zips in last one hour + let data_dir = Path::new(&root_dir); + let pattern = format!("{}_", TXHASHSET_ZIP); + if let Ok(n) = clean_files_by_prefix(data_dir.clone(), &pattern, 60 * 60) { + debug!( + "{} zip files have been clean up in folder: {:?}", + n, data_dir + ); + } + } + + // otherwise, create the zip archive + let path_to_be_cleanup = { // Temp txhashset directory - let temp_txhashset_path = - Path::new(&root_dir).join(format!("{}_zip_{}", TXHASHSET_SUBDIR, ts)); + let temp_txhashset_path = Path::new(&root_dir).join(format!( + "{}_zip_{}", + TXHASHSET_SUBDIR, + header.hash().to_string() + )); // Remove temp dir if it exist if temp_txhashset_path.exists() { fs::remove_dir_all(&temp_txhashset_path)?; @@ -1444,10 +1460,21 @@ pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option) -> Re // Compress zip zip::compress(&temp_txhashset_path, &File::create(zip_path.clone())?) .map_err(|ze| ErrorKind::Other(ze.to_string()))?; - } + + temp_txhashset_path + }; // open it again to read it back - let zip_file = File::open(zip_path)?; + let zip_file = File::open(zip_path.clone())?; + + // clean-up temp txhashset directory. + if let Err(e) = fs::remove_dir_all(&path_to_be_cleanup) { + warn!( + "txhashset zip file: {:?} fail to remove, err: {}", + zip_path.to_str(), + e + ); + } Ok(zip_file) } diff --git a/chain/tests/test_txhashset.rs b/chain/tests/test_txhashset.rs index a3bc4c8768..41be8d9c66 100644 --- a/chain/tests/test_txhashset.rs +++ b/chain/tests/test_txhashset.rs @@ -22,12 +22,12 @@ use std::fs::{self, File, OpenOptions}; use std::iter::FromIterator; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; use crate::chain::store::ChainStore; use crate::chain::txhashset; use crate::core::core::BlockHeader; use crate::util::file; +use grin_core::core::hash::Hashed; fn clean_output_dir(dir_name: &str) { let _ = fs::remove_dir_all(dir_name); @@ -35,52 +35,48 @@ fn clean_output_dir(dir_name: &str) { #[test] fn test_unexpected_zip() { - let now = SystemTime::now(); - let rand = now.duration_since(UNIX_EPOCH).unwrap().subsec_micros(); - let db_root = format!(".grin_txhashset_zip"); clean_output_dir(&db_root); { let chain_store = ChainStore::new(&db_root).unwrap(); let store = Arc::new(chain_store); txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap(); + let head = BlockHeader::default(); // First check if everything works out of the box - assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok()); - let zip_path = Path::new(&db_root).join(format!("txhashset_snapshot_{}.zip", rand)); + assert!(txhashset::zip_read(db_root.clone(), &head).is_ok()); + let zip_path = Path::new(&db_root).join(format!( + "txhashset_snapshot_{}.zip", + head.hash().to_string() + )); let zip_file = File::open(&zip_path).unwrap(); - assert!(txhashset::zip_write( - PathBuf::from(db_root.clone()), - zip_file, - &BlockHeader::default() - ) - .is_ok()); + assert!(txhashset::zip_write(PathBuf::from(db_root.clone()), zip_file, &head).is_ok()); // Remove temp txhashset dir - fs::remove_dir_all(Path::new(&db_root).join(format!("txhashset_zip_{}", rand))).unwrap(); + fs::remove_dir_all( + Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())), + ); // Then add strange files in the original txhashset folder write_file(db_root.clone()); - assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok()); + assert!(txhashset::zip_read(db_root.clone(), &head).is_ok()); // Check that the temp dir dos not contains the strange files - let txhashset_zip_path = Path::new(&db_root).join(format!("txhashset_zip_{}", rand)); + let txhashset_zip_path = + Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())); assert!(txhashset_contains_expected_files( - format!("txhashset_zip_{}", rand), + format!("txhashset_zip_{}", head.hash().to_string()), txhashset_zip_path.clone() )); - fs::remove_dir_all(Path::new(&db_root).join(format!("txhashset_zip_{}", rand))).unwrap(); + fs::remove_dir_all( + Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())), + ); let zip_file = File::open(zip_path).unwrap(); - assert!(txhashset::zip_write( - PathBuf::from(db_root.clone()), - zip_file, - &BlockHeader::default() - ) - .is_ok()); + assert!(txhashset::zip_write(PathBuf::from(db_root.clone()), zip_file, &head).is_ok()); // Check that the txhashset dir dos not contains the strange files let txhashset_path = Path::new(&db_root).join("txhashset"); assert!(txhashset_contains_expected_files( "txhashset".to_string(), txhashset_path.clone() )); - fs::remove_dir_all(Path::new(&db_root).join("txhashset")).unwrap(); + fs::remove_dir_all(Path::new(&db_root).join("txhashset")); } // Cleanup chain directory clean_output_dir(&db_root);