make the queueing system async

webspecs · Jan 27, 2015 · 19c7b14 · 19c7b14
1 parent 1374858
commit 19c7b14
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 31 deletions.
diff --git a/bin/server.js b/bin/server.js
@@ -50,14 +50,15 @@ app.post("/hook", function (req, res) {
     log.info("Processing request for " + repo + "#" + branch);
     if (!branch || !repo) return ok(res, "Could not find repo or branch in data.");
     log.info("Hook for " + repo + ", branch " + branch);
-    var wanted = JSON.parse(fs.readFileSync(wantedFile, "utf8"));
-    if (!wanted[repo]) return ok(res, "Repository not in the wanted list, maybe add it to the-index?");
-    if (!wanted[repo].branches[branch]) return ok(res, "Branch not in the wanted list, maybe add it to the-index?");
-    var stamp = queue.enqueue(repo, branch)
-    ,   msg = "Queued " + stamp + " for processing."
-    ;
-
-    return ok(res, msg);
+    fs.readFile(wantedFile, "utf8", function (err, content) {
+        var wanted = JSON.parse(content);
+        if (!wanted[repo]) return ok(res, "Repository not in the wanted list, maybe add it to the-index?");
+        if (!wanted[repo].branches[branch]) return ok(res, "Branch not in the wanted list, maybe add it to the-index?");
+        queue.enqueue(repo, branch, function (err, stamp) {
+            var msg = "Queued " + stamp + " for processing.";
+            ok(res, msg);
+        });
+    });
 });
 
 app.all("*", function (req, res) {
@@ -66,13 +67,13 @@ app.all("*", function (req, res) {
 });
 
 function poll () {
-    var next = queue.next();
-    // console.log("Polling", next);
-    if (!next) return setTimeout(poll, pollInterval);
-    log.info("Found item in queue, processing " + JSON.stringify(next));
-    man.processRepository(next, function (err) {
-        if (err) log.error(err);
-        process.nextTick(poll);
+    queue.next(function (err, next) {
+        if (!next) return setTimeout(poll, pollInterval);
+        log.info("Found item in queue, processing " + JSON.stringify(next));
+        man.processRepository(next, function (err) {
+            if (err) log.error(err);
+            process.nextTick(poll);
+        });
     });
 }
 poll();

diff --git a/lib/queue.js b/lib/queue.js
@@ -5,31 +5,41 @@ var fs = require("fs")
 ,   queueDir = jn(dataDir, "queue")
 ;
 
-exports.enqueue = function (repo, branch) {
+exports.enqueue = function (repo, branch, cb) {
     var stamp = Date.now() + "-" + process.hrtime()[1];
-    fs.writeFileSync(
+    fs.writeFile(
         jn(queueDir, stamp + ".json")
     ,   JSON.stringify({ repository: repo, branch: branch }, null, 4)
     ,   { encoding: "utf8" }
+    ,   function (err) {
+            if (err) throw err;
+            cb(null, stamp);
+        }
     );
-    return stamp;
 };
 
-exports.next = function () {
-    var queue = fs.readdirSync(queueDir)
+exports.next = function (cb) {
+    fs.readdir(queueDir, function (err, queue) {
+        if (err) throw err;
+        queue = queue
                     .sort(function (a, b) { // we have to do this to guarantee numeric comparison
                         if (a < b) return -1;
                         if (a > b) return 1;
                         return 0;
-                    })
-    ;
-    if (!queue.length) return null;
-    var top = queue[0]
-    ,   file = jn(queueDir, top)
-    ,   content = JSON.parse(fs.readFileSync(file))
-    ;
-    fs.unlinkSync(file);
-    return content;
+                    });
+        if (!queue.length) return null;
+        var top = queue[0]
+        ,   file = jn(queueDir, top)
+        ;
+        fs.readFile(file, function (err, content) {
+            if (err) throw err;
+            var content = JSON.parse(content);
+            fs.unlink(file, function (err) {
+                if (err) throw err;
+                cb(null, content);
+            });
+        });
+    });
 };
 
 // XXX
@@ -38,4 +48,3 @@ exports.next = function () {
 //      - on success, unlink
 //      - if there are limbo files at start, process those first
 //      - keep a counter of the files re-processed from limbo so we avoid loops
-//  - make this async
diff --git a/lib/rsync.js b/lib/rsync.js
@@ -1,7 +1,7 @@
 
 var exec = require("child_process").exec
 // XXX
-//  is it possible to exclude files that haven't changed apart from in there modified time?
+//  is it possible to exclude files that haven't changed apart from in their modified time?
 ,   excludes = ".git .gitignore README.md LICENSE CONTRIBUTING.md .DS_Store '*.php' '*.php3' '*.php4' '*.php5' '*.cgi'".split(" ")
 ,   lock = require("./lock")
 ,   log = require("./log")