clean-up the txhashset zip file and temp folder (#2575)

* clean-up the txhashset zip file and temp folder
* adapt test_txhashset with the new zip_read function
* fix: the wrong folder when cleaning up old zips

chain/src/chain.rs

@@ -690,7 +690,7 @@ impl Chain {
 		}
 
 		// prepares the zip and return the corresponding Read
-		let txhashset_reader = txhashset::zip_read(self.db_root.clone(), &header, None)?;
+		let txhashset_reader = txhashset::zip_read(self.db_root.clone(), &header)?;
 		Ok((
 			header.output_mmr_size,
 			header.kernel_mmr_size,

chain/src/txhashset/txhashset.rs

@@ -32,13 +32,13 @@ use crate::util::secp::pedersen::{Commitment, RangeProof};
 use crate::util::{file, secp_static, zip};
 use croaring::Bitmap;
 use grin_store;
-use grin_store::pmmr::{PMMRBackend, PMMR_FILES};
+use grin_store::pmmr::{clean_files_by_prefix, PMMRBackend, PMMR_FILES};
 use grin_store::types::prune_noop;
 use std::collections::HashSet;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::{Instant, SystemTime, UNIX_EPOCH};
+use std::time::Instant;
 
 const HEADERHASHSET_SUBDIR: &'static str = "header";
 const TXHASHSET_SUBDIR: &'static str = "txhashset";
@@ -1383,22 +1383,38 @@ impl<'a> Extension<'a> {
 
 /// Packages the txhashset data files into a zip and returns a Read to the
 /// resulting file
-pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option<u32>) -> Result<File, Error> {
-	let ts = if let None = rand {
-		let now = SystemTime::now();
-		now.duration_since(UNIX_EPOCH).unwrap().subsec_micros()
-	} else {
-		rand.unwrap()
-	};
-	let txhashset_zip = format!("{}_{}.zip", TXHASHSET_ZIP, ts);
+pub fn zip_read(root_dir: String, header: &BlockHeader) -> Result<File, Error> {
+	let txhashset_zip = format!("{}_{}.zip", TXHASHSET_ZIP, header.hash().to_string());
 
 	let txhashset_path = Path::new(&root_dir).join(TXHASHSET_SUBDIR);
 	let zip_path = Path::new(&root_dir).join(txhashset_zip);
-	// create the zip archive
-	{
+
+	// if file exist, just re-use it
+	let zip_file = File::open(zip_path.clone());
+	if let Ok(zip) = zip_file {
+		return Ok(zip);
+	} else {
+		// clean up old zips.
+		// Theoretically, we only need clean-up those zip files older than STATE_SYNC_THRESHOLD.
+		// But practically, these zip files are not small ones, we just keep the zips in last one hour
+		let data_dir = Path::new(&root_dir);
+		let pattern = format!("{}_", TXHASHSET_ZIP);
+		if let Ok(n) = clean_files_by_prefix(data_dir.clone(), &pattern, 60 * 60) {
+			debug!(
+				"{} zip files have been clean up in folder: {:?}",
+				n, data_dir
+			);
+		}
+	}
+
+	// otherwise, create the zip archive
+	let path_to_be_cleanup = {
 		// Temp txhashset directory
-		let temp_txhashset_path =
-			Path::new(&root_dir).join(format!("{}_zip_{}", TXHASHSET_SUBDIR, ts));
+		let temp_txhashset_path = Path::new(&root_dir).join(format!(
+			"{}_zip_{}",
+			TXHASHSET_SUBDIR,
+			header.hash().to_string()
+		));
 		// Remove temp dir if it exist
 		if temp_txhashset_path.exists() {
 			fs::remove_dir_all(&temp_txhashset_path)?;
@@ -1410,10 +1426,21 @@ pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option<u32>) -> Re
 		// Compress zip
 		zip::compress(&temp_txhashset_path, &File::create(zip_path.clone())?)
 			.map_err(|ze| ErrorKind::Other(ze.to_string()))?;
-	}
+
+		temp_txhashset_path
+	};
 
 	// open it again to read it back
-	let zip_file = File::open(zip_path)?;
+	let zip_file = File::open(zip_path.clone())?;
+
+	// clean-up temp txhashset directory.
+	if let Err(e) = fs::remove_dir_all(&path_to_be_cleanup) {
+		warn!(
+			"txhashset zip file: {:?} fail to remove, err: {}",
+			zip_path.to_str(),
+			e
+		);
+	}
 	Ok(zip_file)
 }
 

chain/tests/test_txhashset.rs

@@ -23,48 +23,56 @@ use std::fs::{self, File, OpenOptions};
 use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::chain::store::ChainStore;
 use crate::chain::txhashset;
 use crate::core::core::BlockHeader;
 use crate::util::file;
+use grin_core::core::hash::Hashed;
 
 fn clean_output_dir(dir_name: &str) {
 	let _ = fs::remove_dir_all(dir_name);
 }
 
 #[test]
 fn test_unexpected_zip() {
-	let now = SystemTime::now();
-	let rand = now.duration_since(UNIX_EPOCH).unwrap().subsec_micros();
-
 	let db_root = format!(".grin_txhashset_zip");
 	clean_output_dir(&db_root);
 	let db_env = Arc::new(store::new_env(db_root.clone()));
 	let chain_store = ChainStore::new(db_env).unwrap();
 	let store = Arc::new(chain_store);
 	txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap();
+	let head = BlockHeader::default();
 	// First check if everything works out of the box
-	assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok());
-	let zip_path = Path::new(&db_root).join(format!("txhashset_snapshot_{}.zip", rand));
+	assert!(txhashset::zip_read(db_root.clone(), &head).is_ok());
+	let zip_path = Path::new(&db_root).join(format!(
+		"txhashset_snapshot_{}.zip",
+		head.hash().to_string()
+	));
 	let zip_file = File::open(&zip_path).unwrap();
-	assert!(txhashset::zip_write(db_root.clone(), zip_file, &BlockHeader::default()).is_ok());
+	assert!(txhashset::zip_write(db_root.clone(), zip_file, &head).is_ok());
 	// Remove temp txhashset dir
-	fs::remove_dir_all(Path::new(&db_root).join(format!("txhashset_zip_{}", rand))).unwrap();
+	assert!(fs::remove_dir_all(
+		Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string()))
+	)
+	.is_err());
 	// Then add strange files in the original txhashset folder
 	write_file(db_root.clone());
-	assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok());
+	assert!(txhashset::zip_read(db_root.clone(), &head).is_ok());
 	// Check that the temp dir dos not contains the strange files
-	let txhashset_zip_path = Path::new(&db_root).join(format!("txhashset_zip_{}", rand));
+	let txhashset_zip_path =
+		Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string()));
 	assert!(txhashset_contains_expected_files(
-		format!("txhashset_zip_{}", rand),
+		format!("txhashset_zip_{}", head.hash().to_string()),
 		txhashset_zip_path.clone()
 	));
-	fs::remove_dir_all(Path::new(&db_root).join(format!("txhashset_zip_{}", rand))).unwrap();
+	assert!(fs::remove_dir_all(
+		Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string()))
+	)
+	.is_err());
 
 	let zip_file = File::open(zip_path).unwrap();
-	assert!(txhashset::zip_write(db_root.clone(), zip_file, &BlockHeader::default()).is_ok());
+	assert!(txhashset::zip_write(db_root.clone(), zip_file, &head).is_ok());
 	// Check that the txhashset dir dos not contains the strange files
 	let txhashset_path = Path::new(&db_root).join("txhashset");
 	assert!(txhashset_contains_expected_files(