Merge pull request #28 from justindhillon/multithreading

Add Multithreading

bin/help.js

@@ -5,6 +5,9 @@ function help() {
     console.log('   link-inspector  Makes a file in an "output/" directory that');
     console.log('                   containes all the broken links found');
     console.log();
+    console.log('   Option     Meaning');
+    console.log('   -j         Sets the number of parallel threads');
+    console.log();
     console.log('"npx scan help" lists available commands');
     process.exit(1);
 }

bin/index.js

@@ -3,10 +3,22 @@ const help = require("./help.js");
 const scanLinks = require("./scan/scanLinks.js");
 
 const args = process.argv.slice(2); // Gets npx arguments
-const path = args[0]; // Gets <file/directory-path>
 
-// Error: no path given
-if (args.length < 1) {
+// Initialize variables for path and -j option
+let path = '';
+let jOptionValue = null;
+
+// Parse the arguments
+args.forEach((arg, index) => {
+    if (arg === '-j' && args[index + 1]) {
+        jOptionValue = parseInt(args[index + 1], 10); // Parse the next argument as an integer
+    } else if (index === 0 || (index === 1 && args[0] === '-j')) {
+        path = arg; // The path is expected to be the first or second argument
+    }
+});
+
+// Error handling for no path given
+if (!path) {
     console.error('Error: no path given');
     help();
 }
@@ -15,7 +27,8 @@ if (path === "help") {
     help();
 }
 
-scanLinks(path).then(() => {
+// Pass the -j option value to scanLinks, if provided
+scanLinks(path, jOptionValue).then(() => {
     process.exit(0);
 }).catch(() => {
     process.exit(1);

bin/scan/processPromises.js

@@ -0,0 +1,52 @@
+const readFile = require("./readFile.js");
+const getLinks = require("./getLinks.js");
+const writeBrokenLinks = require("./writeBrokenLinks.js");
+
+function createPromise(filePath, fluff) {
+    return new Promise(async (resolve, reject) => {
+        try {
+          const fileContent = readFile(filePath);
+          const links = getLinks(fileContent);
+
+          if (links !== null) {
+            await writeBrokenLinks(links, filePath, fluff);
+          }
+          resolve();
+        } catch (error) {
+          reject(error);
+        }
+    });
+}
+
+function processPromises(maxConcurrent, filePaths, fluff) {
+    let runningPromises = 0;
+    let currentIndex = 1;
+    let results = [];
+
+    return new Promise((resolve, reject) => {
+        const startNextPromise = () => {
+            if (currentIndex > filePaths.length) {
+                if (runningPromises === 0) {
+                    resolve(results);
+                }
+                return;
+            }
+
+            runningPromises++;
+            currentIndex++;
+            createPromise(filePaths[currentIndex - 2], fluff).then(() => {
+                runningPromises--;
+                startNextPromise();
+            }).catch(reject);
+
+            if (runningPromises < maxConcurrent) {
+                startNextPromise();
+            }
+        };
+
+        startNextPromise();
+    });
+}
+
+module.exports = processPromises;
+

bin/scan/scanLinks.js

@@ -1,12 +1,11 @@
-const getFilePaths = require("./getFilePaths.js");
-const readFile = require("./readFile.js");
-const getLinks = require("./getLinks.js");
-const writeBrokenLinks = require("./writeBrokenLinks.js");
-const help = require("../help.js");
 const fs = require('fs');
+const help = require("../help.js");
+const getFilePaths = require("./getFilePaths.js");
+const processPromises = require("./processPromises.js");
 
 // path is <file/directory path>
-async function scanLinks(path) {
+// j is concurrent threads
+async function scanLinks(path, j=1) {
   // Error: path does not exist
   if (!fs.existsSync(path)) {
     console.error('Error:', path, 'does not exist');
@@ -27,28 +26,11 @@ async function scanLinks(path) {
 
   console.log("If nothing is output below, no broken links where found");
 
-  let promises = filePaths.map(filePath => {
-    return new Promise(async (resolve, reject) => {
-      try {
-        const fileContent = readFile(filePath);
-        const links = getLinks(fileContent);
-
-        if (links !== null) {
-          await writeBrokenLinks(links, filePath, fluff);
-        }
-        resolve();
-      } catch (error) {
-        reject(error);
-      }
-    });
-  });
-
-  await Promise.all(promises)
+  await processPromises(j, filePaths, fluff)
     .then(() => {
       console.log("Finished!");
-    })
-    .catch(error => {
-      console.error("An error occurred:", error);
+    }).catch(error => {
+      console.error(error);
     });
 }
 

test/scanLinks.test.js

@@ -17,9 +17,8 @@ describe('scanLinks.js', () => {
             }
           });
 
-        await scanLinks("test/test_suite").then(async () => {
+        await scanLinks("test/test_suite", 10).then(async () => {
             const res = dircompare.compareSync("test/expected_output", "output/test_suite", options);
-            console.log(res.same);
             expect(res.same).toEqual(true);
         }).catch(() => {
             process.exit(1);