New: Compress json files from cache to zstd to reduce used space

CHANGELOG.md

@@ -14,12 +14,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Custom labels support [`1185d82`](https://github.com/ollm/OpenComic/commit/1185d82790a764e63d1d223362e4b44e203ff0e1)
 - Option to not enlarge images more than its original size [`e2f8598`](https://github.com/ollm/OpenComic/commit/e2f85983352529162822576bbb8e7da41cd31f39)
 - New image interpolation methods available: lanczos3, lanczos2, mitchell, cubic, nearest and others [`86cd705`](https://github.com/ollm/OpenComic/commit/86cd7053011b09b1a0a0b898775e32ede8cf5296)
-- Server connection support: smb://, ftp://, ftps://, scp://, sftp://, ssh://
+- Server connection support: smb://, ftp://, ftps://, scp://, sftp://, ssh:// [`52a09a9`](https://github.com/ollm/OpenComic/commit/52a09a9aad601a2e70b8f0011a6fddc7d3e9023a)
+- Compress json files from cache to zstd to reduce used space
 
 ##### 🐛 Bug Fixes
 
 - Error on detect file type from binary [`0f81947`](https://github.com/ollm/OpenComic/commit/0f819470d42ce996cd4f1f0a31665a605d2bc39a)
-- Zoom bug in vertical reading if global zoom disabled
+- Zoom bug in vertical reading if global zoom disabled [`f83d17f`](https://github.com/ollm/OpenComic/commit/f83d17fbf3cb581d8b8735050cb919fca623d8aa)
 
 ## [v1.0.0](https://github.com/ollm/OpenComic/releases/tag/v1.0.0) (09-12-2023)
 

package.json

@@ -68,6 +68,7 @@
 	"dependencies": {
 		"@electron/remote": "^2.1.0",
 		"@stifani/smb2": "^0.3.0",
+		"@toondepauw/node-zstd": "^1.2.0",
 		"7zip-bin": "^5.2.0",
 		"basic-ftp": "^5.0.4",
 		"electron-json-storage": "^4.6.0",

scripts/cache.js

@@ -1,3 +1,7 @@
+const zstd = require('@toondepauw/node-zstd');
+const zstdEncoder = new zstd.Encoder(5);
+const zstdDecoder = new zstd.Decoder();
+
 var queuedImages = [], processingTheImageQueue = false;
 
 var cacheFolder = p.join(electronRemote.app.getPath('cache'), 'opencomic');
@@ -177,7 +181,7 @@ function returnThumbnailsImages(images, callback, file = false)
 	return single ? thumbnail : thumbnails;
 }
 
-function writeFile(name, content)
+async function writeFile(name, content)
 {
 	fs.writeFile(p.join(cacheFolder, name), content, function(){}); 
 }
@@ -187,6 +191,22 @@ function writeFileSync(name, content)
 	fs.writeFileSync(p.join(cacheFolder, name), content, function(){}); 
 }
 
+async function writeJson(name, json)
+{
+	let path = p.join(cacheFolder, name+'.zstd');
+
+	let encoded = await zstdEncoder.encode(Buffer.from(JSON.stringify(json)));
+	fs.writeFile(path, encoded, function(){});
+}
+
+function writeJsonSync(name, json)
+{
+	let path = p.join(cacheFolder, name+'.zstd');
+
+	let encoded = zstdEncoder.encodeSync(Buffer.from(JSON.stringify(json)));
+	fs.writeFileSync(path, encoded, function(){});
+}
+
 function readFile(name)
 {
 	let path = p.join(cacheFolder, name);
@@ -199,10 +219,10 @@ function readFile(name)
 
 function readJson(name)
 {
-	let path = p.join(cacheFolder, name);
+	let path = p.join(cacheFolder, name+'.zstd');
 
 	if(fs.existsSync(path))
-		return JSON.parse(fs.readFileSync(path, 'utf8'));
+		return JSON.parse(zstdDecoder.decodeSync(fs.readFileSync(path)));
 	else
 		return false;
 }
@@ -342,6 +362,8 @@ module.exports = {
 	cleanQueue: cleanQueue,
 	writeFile: writeFile,
 	writeFileSync: writeFileSync,
+	writeJson: writeJson,
+	writeJsonSync: writeJsonSync,
 	readFile: readFile,
 	readJson: readJson,
 	existsFile: existsFile,

scripts/file-manager.js

@@ -204,10 +204,7 @@ var file = function(path, config = false) {
 		let mtime = !_isServer ? fs.statSync(firstCompressedFile(path)).mtime.getTime() : 1;
 		let compressed = this.openCompressed(path, _realPath, mtime);
 
-		let json = cache.readFile(compressed.cacheFile);
-
-		if(json)
-			json = JSON.parse(json);
+		let json = cache.readJson(compressed.cacheFile);
 
 		if(this.config.cache)
 		{
@@ -234,7 +231,7 @@ var file = function(path, config = false) {
 			files = this.setPosterFromMetadata(files, metadata.poster);
 
 		if(!json || json.mtime != mtime)
-			cache.writeFile(compressed.cacheFile, JSON.stringify({mtime: mtime, files: files, metadata: metadata}), {}, function(){});
+			cache.writeJson(compressed.cacheFile, {mtime: mtime, files: files, metadata: metadata});
 
 		return files;
 
@@ -263,7 +260,7 @@ var file = function(path, config = false) {
 
 		path = serverClient.fixPath(path || this.path);
 		let sha = sha1(path);
-		let cacheFile = 'server-files-'+sha+'.json';
+		let cacheFile = 'server-files-'+sha+'.json.zstd';
 
 		return cache.existsFile(cacheFile);
 
@@ -284,10 +281,7 @@ var file = function(path, config = false) {
 			if(_containsCompressed)
 				return this.readCompressed(path);
 
-			let json = cache.readFile(cacheFile);
-
-			if(json)
-				json = JSON.parse(json);
+			let json = cache.readJson(cacheFile);
 
 			if(json)
 				return json.files;

scripts/migration.js

@@ -0,0 +1,34 @@
+
+function compressJsonCache()
+{
+	let files = fs.readdirSync(cache.folder);
+
+	for(let i = 0, len = files.length; i < len; i++)
+	{
+		let file = files[i];
+
+		if(/\.json$/.test(file))
+		{
+			let json = fs.readFileSync(p.join(cache.folder, file));
+			json = JSON.parse(json);
+
+			cache.writeJsonSync(file, json);
+			fs.unlinkSync(p.join(cache.folder, file));
+		}
+	}
+}
+
+function start(data)
+{
+	let changes = data.config.changes;
+
+	if(changes < 75)
+		compressJsonCache();
+
+	return data;
+}
+
+module.exports = {
+	start: start,
+	compressJsonCache: compressJsonCache,
+};

scripts/opencomic.js

@@ -541,10 +541,9 @@ async function loadShoSho()
 
 /*Global functions*/
 
-function copy(data) {
-
+function copy(data)
+{
 	return JSON.parse(JSON.stringify(data));
-
 }
 
 function inArray(string, array)