updatedb made fast (#259)

Author: adityapatadia

.eslintrc.js

@@ -1,10 +1,11 @@
 module.exports = {
     "env": {
-        "node": true
+        "node": true,
+        "es6": true
     },
     "extends": "eslint:recommended",
     "parserOptions": {
-        "ecmaVersion": 5
+        "ecmaVersion": 2018
     },
     "rules": {
         "indent": [

scripts/updatedb.js

@@ -10,6 +10,7 @@ var https = require('https');
 var path = require('path');
 var url = require('url');
 var zlib = require('zlib');
+var readline = require('readline');
 
 fs.existsSync = fs.existsSync || path.existsSync;
 
@@ -340,9 +341,9 @@ function processLookupCountry(src, cb){
 		});
 }
 
-function processCountryData(src, dest, cb) {
+async function processCountryData(src, dest) {
 	var lines=0;
-	function processLine(line) {
+	async function processLine(line) {
 		var fields = CSVtoArray(line);
 
 		if (!fields || fields.length < 6) {
@@ -389,12 +390,18 @@ function processCountryData(src, dest, cb) {
 			}
 
 			b.write(cc, bsz - 2);
-	
-			fs.writeSync(datFile, b, 0, bsz, null);
 			if(Date.now() - tstart > 5000) {
 				tstart = Date.now();
 				process.stdout.write('\nStill working (' + lines + ') ...');
 			}
+
+			if(datFile._writableState.needDrain) {
+				return new Promise((resolve) => {
+					datFile.write(b, resolve);
+				});
+			} else {
+				return datFile.write(b);
+			}
 		}
 	}
 
@@ -406,24 +413,25 @@ function processCountryData(src, dest, cb) {
 
 	process.stdout.write('Processing Data (may take a moment) ...');
 	var tstart = Date.now();
-	var datFile = fs.openSync(dataFile, "w");
+	var datFile = fs.createWriteStream(dataFile);
 
-	lazy(fs.createReadStream(tmpDataFile))
-		.lines
-		.map(function(byteArray) {
-			return iconv.decode(byteArray, 'latin1');
-		})
-		.skip(1)
-		.map(processLine)
-		.on('pipe', function() {
-			console.log(chalk.green(' DONE'));
-			cb();
-		});
+	var rl = readline.createInterface({
+		input: fs.createReadStream(tmpDataFile),
+		crlfDelay: Infinity
+	});
+	var i = 0;
+	for await (var line of rl) {
+		i++;
+		if(i == 1) continue;
+		await processLine(line);
+	}
+	datFile.close();
+	console.log(' DONE'.green);
 }
 
-function processCityData(src, dest, cb) {
+async function processCityData(src, dest) {
 	var lines = 0;
-	function processLine(line) {
+	async function processLine(line) {
 		if (line.match(/^Copyright/) || !line.match(/\d/)) {
 			return;
 		}
@@ -500,11 +508,18 @@ function processCityData(src, dest, cb) {
 			b.writeInt32BE(area,20);
 		}
 
-		fs.writeSync(datFile, b, 0, b.length, null);
 		if(Date.now() - tstart > 5000) {
 			tstart = Date.now();
 			process.stdout.write('\nStill working (' + lines + ') ...');
 		}
+
+		if(datFile._writableState.needDrain) {
+			return new Promise((resolve) => {
+				datFile.write(b, resolve);
+			});
+		} else {
+			return datFile.write(b);
+		}
 	}
 
 	var dataFile = path.join(dataPath, dest);
@@ -514,16 +529,19 @@ function processCityData(src, dest, cb) {
 
 	process.stdout.write('Processing Data (may take a moment) ...');
 	var tstart = Date.now();
-	var datFile = fs.openSync(dataFile, "w");
+	var datFile = fs.createWriteStream(dataFile);
 
-	lazy(fs.createReadStream(tmpDataFile))
-		.lines
-		.map(function(byteArray) {
-			return iconv.decode(byteArray, 'latin1');
-		})
-		.skip(1)
-		.map(processLine)
-		.on('pipe', cb);
+	var rl = readline.createInterface({
+		input: fs.createReadStream(tmpDataFile),
+		crlfDelay: Infinity
+	});
+	var i = 0;
+	for await (var line of rl) {
+		i++;
+		if(i == 1) continue;
+		await processLine(line);
+	}
+	datFile.close();
 }
 
 function processCityDataNames(src, dest, cb) {
@@ -599,10 +617,10 @@ function processData(database, cb) {
 	if (type === 'country') {
 		if(Array.isArray(src)){
 			processLookupCountry(src[0], function() {
-				processCountryData(src[1], dest[1], function() {
-					processCountryData(src[2], dest[2], function() {
-						cb(null, database);
-					});
+				processCountryData(src[1], dest[1]).then(() => {
+					return processCountryData(src[2], dest[2]);
+				}).then(() => {
+					cb(null, database);
 				});
 			});
 		}
@@ -613,12 +631,12 @@ function processData(database, cb) {
 		}
 	} else if (type === 'city') {
 		processCityDataNames(src[0], dest[0], function() {
-			processCityData(src[1], dest[1], function() {
+			processCityData(src[1], dest[1]).then(() => {
 				console.log("city data processed");
-				processCityData(src[2], dest[2], function() {
-					console.log(chalk.green(' DONE'));
-					cb(null, database);
-				});
+				return processCityData(src[2], dest[2]);
+			}).then(() => {
+				console.log(chalk.green(' DONE'));
+				cb(null, database);
 			});
 		});
 	}