implement updateNewerFiles option

.gitignore

@@ -1,6 +1,5 @@
 node_modules/
 ignore
-.vscode
 test/remote
 
 playground

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "hbenl.vscode-mocha-test-adapter"
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "mochaExplorer.files": "**/*.spec.js",
+    "mochaExplorer.timeout": 600000
+}

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 2.4.2
+
+    - Implement updateNewerFiles option
+
 ## 2.4.0
 
     - Add support for SFTP - thanks @der-On

README.md

@@ -41,6 +41,7 @@ const config = {
     forcePasv: true,
     // use sftp or ftp
     sftp: false,
+    updateNewerFiles: true // only upload files that have changed
 };
 
 ftpDeploy
@@ -118,4 +119,3 @@ npm test
 ## ToDo
 
 -   re-enable continueOnError
--   update newer files only (PR welcome)

package.json

@@ -1,7 +1,7 @@
 {
     "name": "ftp-deploy",
-    "version": "2.4.1",
-    "author": "Simon Hampton",
+    "version": "2.5.0",
+    "author": "Pascal GANAYE",
     "description": "Ftp a folder from your local disk to an ftp destination",
     "main": "src/ftp-deploy",
     "scripts": {
@@ -10,17 +10,18 @@
     },
     "dependencies": {
         "bluebird": "^3.7.2",
-        "minimatch": "3.0.4",
+        "memory-streams": "^0.1.3",
+        "minimatch": "^3.0.4",
         "promise-ftp": "^1.3.5",
         "read": "^1.0.7",
-        "ssh2-sftp-client": "^5.3.2",
+        "ssh2-sftp-client": "^7.2.1",
         "upath": "^2.0.1"
     },
     "devDependencies": {
-        "chai": "^4.3.0",
+        "chai": "^4.3.4",
         "delete": "^1.1.0",
-        "ftp-srv": "^4.4.0",
-        "mocha": "^8.2.1"
+        "ftp-srv": "^4.5.0",
+        "mocha": "^9.1.3"
     },
     "keywords": [
         "ftp",
@@ -53,4 +54,4 @@
     "prettier": {
         "tabWidth": 4
     }
-}
+}

src/ftp-deploy.js

@@ -5,10 +5,13 @@ const util = require("util");
 const events = require("events");
 const Promise = require("bluebird");
 const fs = require("fs");
+const os = require("os");
+const memoryStreams = require('memory-streams');
 
 var PromiseFtp = require("promise-ftp");
 var PromiseSftp = require("ssh2-sftp-client");
 const lib = require("./lib");
+const path = require('path');
 
 /* interim structure
 {
@@ -38,6 +41,69 @@ const FtpDeployer = function () {
         });
     };
 
+
+    this.downloadFilemap = function (remoteRoot, filemapName) {
+        return new Promise((resolve, reject) => {
+            this.ftp
+                .cwd(remoteRoot)
+                .then(() => this.ftp.get(filemapName))
+                .then((stream) => {
+                    var writer = new memoryStreams.WritableStream();
+                    stream.pipe(writer)
+                    stream.on('pause', () => {
+                        // I am not entirely sure while memory stream don't eat this 
+                        stream.resume();
+                    });
+                    stream.on('finish', (data, bcd) => {
+                        try {
+                            let obj = JSON.parse(writer.toString());
+                            resolve(obj?.filemap);
+                        } catch (error) {
+                            console.log("Invalid file-map")
+                            // we will upload everything
+                            resolve();
+                        }
+                    })
+                })
+                .catch((e) => {
+                    // probably file not found we resolve to undefined.
+                    resolve()
+                });
+        });
+
+    }
+
+    this.pruneFilemap = function (localFilemap, remoteFilemap, filemapToUpload) {
+        let filesAlreadyUploaded = 0;
+        let filesToUpload = 0;
+        if (!remoteFilemap) remoteFilemap = {};
+        for (let folder of Object.keys(localFilemap)) {
+            let previousFiles = remoteFilemap[folder] || [];
+            filemapToUpload[folder] = localFilemap[folder].filter(newFile => {
+                let previousFile = previousFiles.filter(previousFile => newFile.name == previousFile.name)[0];
+                let keep = (!previousFile || previousFile.mtime != newFile.mtime);
+                if (keep) filesAlreadyUploaded += 1;
+                else filesToUpload += 1;
+                return keep;
+            }).map(f => f.name);
+        }
+        this.emit("log", "files already uploaded", filesAlreadyUploaded);
+        this.emit("log", "files to upload", filesToUpload);
+    }
+
+    this.uploadFileMap = function (remoteRoot, filemapName, filemap) {
+        let filemapBuffer = Buffer.from(JSON.stringify({
+            date: new Date(),
+            user: os.userInfo().username,
+            client: os.hostname(),
+            filemap
+        }), "utf-8");
+        return this.ftp
+            .cwd(remoteRoot)
+            .then(() => this.ftp.put(filemapBuffer, filemapName));
+    }
+
+
     this.makeDir = function (newDirectory) {
         if (newDirectory === "/") {
             return Promise.resolve("unused");
@@ -78,7 +144,6 @@ const FtpDeployer = function () {
     // connects to the server, Resolves the config on success
     this.connect = (config) => {
         this.ftp = config.sftp ? new PromiseSftp() : new PromiseFtp();
-
         // sftp client does not provide a connection status
         // so instead provide one ourselfs
         if (config.sftp) {
@@ -124,20 +189,34 @@ const FtpDeployer = function () {
     // creates list of all files to upload and starts upload process
     this.checkLocalAndUpload = (config) => {
         try {
-            let filemap = lib.parseLocal(
+            let localFilemap = lib.parseLocal(
                 config.include,
                 config.exclude,
                 config.localRoot,
-                "/"
+                "/",
+                config.updateNewerFiles
             );
-            // console.log(filemap);
-            this.emit(
-                "log",
-                "Files found to upload: " + JSON.stringify(filemap)
-            );
-            this.eventObject["totalFilesCount"] = lib.countFiles(filemap);
+            let filemapName = config.filemapName || ".ftp-deploy-file-map.json"
+
+            let uploadFiles = (filemap) => {
+                // console.log(filemap);
+                this.emit("log",
+                    "Files found to upload: " + JSON.stringify(filemap)
+                );
+                this.eventObject["totalFilesCount"] = lib.countFiles(filemap);
+
+                return this.makeAllAndUpload(config, filemap);
+                return result;
+            };
+
+            if (config.updateNewerFiles) {
+                let filemapToUpload = {};
+                return this.downloadFilemap(config.remoteRoot, filemapName)
+                    .then((remoteFilemap) => this.pruneFilemap(localFilemap, remoteFilemap, filemapToUpload))
+                    .then(() => uploadFiles(filemapToUpload))
+                    .then(() => this.uploadFileMap(config.remoteRoot, filemapName, localFilemap));
+            } else return uploadFiles(localFilemap);
 
-            return this.makeAllAndUpload(config, filemap);
         } catch (e) {
             return Promise.reject(e);
         }
@@ -157,7 +236,7 @@ const FtpDeployer = function () {
                     this.emit(
                         "log",
                         "Deleting failed, trying to continue: " +
-                            JSON.stringify(err)
+                        JSON.stringify(err)
                     );
                     return Promise.resolve(config);
                 });

src/ftp-deploy.spec.js

@@ -4,31 +4,33 @@ const path = require("path");
 const fs = require("fs");
 const utils = require("util");
 
-// var assert = require("assert");
+const assert = require("assert");
 
 const statP = utils.promisify(fs.stat);
 
 const del = require("delete");
 const FtpDeploy = require("./ftp-deploy");
 
-const config = {
-    user: "anonymous",
-    password: "anon", // Optional, prompted if none given
-    host: "localhost",
-    port: 2121,
-    localRoot: path.join(__dirname, "../test/local"),
-    remoteRoot: "/ftp",
-    exclude: [],
-    include: ["folderA/**/*", "test-inside-root.txt"],
-    debugMode: true
-};
+function newTestConfig() {
+    return {
+        user: "anonymous",
+        password: "anon", // Optional, prompted if none given
+        host: "localhost",
+        port: 2121,
+        localRoot: path.join(__dirname, "../test/local"),
+        remoteRoot: "/ftp",
+        exclude: [],
+        include: ["folderA/**/*", "test-inside-root.txt"],
+        debugMode: true
+    };
+}
 
 describe("deploy tests", () => {
     const remoteDir = path.join(__dirname, "../test/remote/ftp");
 
     it("should fail if badly configured", () => {
         const d = new FtpDeploy();
-        const configError = Object.assign({}, config, { port: 212 });
+        const configError = Object.assign({}, newTestConfig(), { port: 212 });
         return del(remoteDir)
             .then(() => {
                 return d.deploy(configError);
@@ -46,7 +48,7 @@ describe("deploy tests", () => {
         const d = new FtpDeploy();
         return del(remoteDir)
             .then(() => {
-                let c2 = Object.assign({}, config, { include: [] });
+                let c2 = Object.assign({}, newTestConfig(), { include: [] });
                 return d.deploy(c2);
             })
             .catch(err => {
@@ -61,16 +63,19 @@ describe("deploy tests", () => {
         const d = new FtpDeploy();
         return del(remoteDir)
             .then(() => {
-                return d.deploy(config);
+                return d.deploy(newTestConfig());
             })
             .then(() => {
                 // Should reject if file does not exist
                 return statP(remoteDir + "/test-inside-root.txt");
             })
-            .catch(err => done(err));
+            .catch(err => {
+                throw err;
+            });
     });
     it("should put a dot file", () => {
         const d = new FtpDeploy();
+        const config = newTestConfig();
         return del(remoteDir)
             .then(() => {
                 config.include = [".*"];
@@ -81,4 +86,35 @@ describe("deploy tests", () => {
                 return statP(remoteDir + "/.testfile");
             });
     });
+    it("should put modified files again", () => {
+        const ftpDeploy = new FtpDeploy();
+        const config = newTestConfig();
+        config.updateNewerFiles = true;
+
+        const filesUploaded = [];
+        ftpDeploy.on("uploading", function (data) {
+            filesUploaded.push(data.filename);
+        });
+        return del(remoteDir)
+            .then(() => {
+                return ftpDeploy.deploy(config);
+            }).then(() => {
+                filesUploaded.length = 0;
+                return ftpDeploy.deploy(config);
+            }).then(() => {
+                assert(filesUploaded.length == 0);
+                // modify a single file")
+                fs.writeFileSync(path.resolve(__dirname, '../test/local/test-inside-root.txt'), "modified test inside root\r\n");
+                filesUploaded.length = 0;
+                return ftpDeploy.deploy(config);
+            }).then(() => {
+                assert(filesUploaded.length == 1);
+                // revert single file
+                fs.writeFileSync(path.resolve(__dirname, '../test/local/test-inside-root.txt'), "test inside root\r\n");
+                filesUploaded.length = 0;
+                return ftpDeploy.deploy(config);
+            }).then(() => {
+                assert(filesUploaded.length == 1);
+            });
+    });
 });

src/lib.js

@@ -64,15 +64,15 @@ function canIncludePath(includes, excludes, filePath) {
 }
 
 // A method for parsing the source location and storing the information into a suitably formated object
-function parseLocal(includes, excludes, localRootDir, relDir) {
+function parseLocal(includes, excludes, localRootDir, relDir, collectFilesSize) {
     // reducer
-    let handleItem = function(acc, item) {
+    let handleItem = function (acc, item) {
         const currItem = path.join(fullDir, item);
         const newRelDir = path.relative(localRootDir, currItem);
-
-        if (fs.lstatSync(currItem).isDirectory()) {
+        const stats = fs.lstatSync(currItem);
+        if (stats.isDirectory()) {
             // currItem is a directory. Recurse and attach to accumulator
-            let tmp = parseLocal(includes, excludes, localRootDir, newRelDir);
+            let tmp = parseLocal(includes, excludes, localRootDir, newRelDir, collectFilesSize);
             for (let key in tmp) {
                 if (tmp[key].length == 0) {
                     delete tmp[key];
@@ -84,7 +84,7 @@ function parseLocal(includes, excludes, localRootDir, relDir) {
             // acc[relDir] is always created at previous iteration
             if (canIncludePath(includes, excludes, newRelDir)) {
                 // console.log("including", currItem);
-                acc[relDir].push(item);
+                acc[relDir].push(collectFilesSize ? { name: item, mtime: stats.mtimeMs } : item);
                 return acc;
             }
         }
@@ -107,8 +107,7 @@ function parseLocal(includes, excludes, localRootDir, relDir) {
 }
 
 function countFiles(filemap) {
-    return Object.values(filemap).reduce((acc, item) => acc.concat(item))
-        .length;
+    return Object.values(filemap).reduce((acc, item) => acc.concat(item), []).length;
 }
 
 function deleteDir(ftp, dir) {