[wip] Upload newer

README.md

@@ -13,6 +13,7 @@ npm install --save-dev ftp-deploy
 ## Usage
 
 The most basic usage:
+
 ```js
 var FtpDeploy = require("ftp-deploy");
 var ftpDeploy = new FtpDeploy();
@@ -25,12 +26,20 @@ var config = {
     port: 21,
     localRoot: __dirname + "/local-folder",
     remoteRoot: "/public_html/remote-folder/",
-    // include: ["*", "**/*"],      // this would upload everything except dot files
-    include: ["*.php", "dist/*", ".*"],
+    // typical wordpress includes
+    include: ["*.php", "dist/*"],
+    // include: ["*", "**/*", ".*"],      // upload everything, including dot files
     // e.g. exclude sourcemaps, and ALL files in node_modules (including dot files)
-    exclude: ["dist/**/*.map", "node_modules/**", "node_modules/**/.*", ".git/**"],
-    // delete ALL existing files at destination before uploading, if true
+    exclude: [
+        "dist/**/*.map",
+        "node_modules/**",
+        "node_modules/**/.*",
+        ".git/**"
+    ],
+    // if true, delete ALL existing files (expcet dot files) at destination before uploading
     deleteRemote: false,
+    // if true, only uploads changed files (based on last modified date and file size)
+    newFilesOnly: false,
     // Passive mode is forced (EPSV command is not sent)
     forcePasv: true
 };
@@ -48,10 +57,11 @@ ftpDeploy.deploy(config, function(err, res) {
 });
 ```
 
-**Note:** 
- - in version 2 the config file expects a field of `user` rather than `username` in 1.x.
- - The config file is passed as-is to Promise-FTP.
- - I create a file - e.g. deploy.js - in the root of my source code and add a script to its `package.json` so that I can `npm run deploy`.
+**Note:**
+
+-   in version 2 the config file expects a field of `user` rather than `username` in 1.x.
+-   The config file is passed as-is to [promise-ftp](https://github.com/realtymaps/promise-ftp) - see their docs for more options.
+-   I create a file - e.g. deploy.js - in the root of my source code and add a script to its `package.json` so that I can `npm run deploy`.
 
 ```json
 "scripts": {
@@ -68,7 +78,7 @@ These are lists of [minimatch globs](https://github.com/isaacs/minimatch). ftp-d
 
 ## Events
 
-ftp-deploy reports to clients using events. To get the output you need to  implement watchers for "uploading", "uploaded" and "log":
+ftp-deploy reports progress using events. These can be accessed - if desired - by registering listeners for "uploading", "uploaded", "upload-error" and "log":
 
 ```js
 ftpDeploy.on("uploading", function(data) {
@@ -79,12 +89,12 @@ ftpDeploy.on("uploading", function(data) {
 ftpDeploy.on("uploaded", function(data) {
     console.log(data); // same data as uploading event
 });
-ftpDeploy.on("log", function(data) {
-    console.log(data); // same data as uploading event
-});
 ftpDeploy.on("upload-error", function(data) {
     console.log(data.err); // data will also include filename, relativePath, and other goodies
 });
+ftpDeploy.on("log", function(data) {
+    console.log(data); // same data as uploading event
+});
 ```
 
 ## Testing
@@ -105,6 +115,5 @@ npm test
 
 ## ToDo
 
- - re-enable continueOnError
- - update newer files only (PR welcome)
-
+-   re-enable continueOnError
+-   update newer files only (PR welcome)

package.json

@@ -2,7 +2,7 @@
     "name": "ftp-deploy",
     "version": "2.3.6",
     "author": "Simon Hampton",
-    "description": "Ftp a folder from your local disk to an ftp destination",
+    "description": "Deploy files to a remote Ftp location",
     "main": "src/ftp-deploy",
     "scripts": {
         "test": "mocha **/**.spec.js",
@@ -52,4 +52,4 @@
     "prettier": {
         "tabWidth": 4
     }
-}
+}

playground/test_script.js

@@ -9,10 +9,11 @@ const config = {
     host: "localhost",
     port: 2121,
     localRoot: path.join(__dirname, "local"),
-    remoteRoot: "/f",
-    deleteRemote: true,
-    exclude: [],
-    include: ["test-inside-root.txt"]
+    remoteRoot: "/ftp",
+    deleteRemote: false,
+    newFilesOnly: true,
+    exclude: ["*/index.html"],
+    include: ["index.html", "folderA/*"]
     // include: ["**/*", "*", ".*"]
 };
 

src/ftp-deploy.js

@@ -30,6 +30,8 @@ const FtpDeployer = function() {
     };
 
     this.makeAllAndUpload = function(remoteDir, filemap) {
+        // TODO pass on the full object
+
         let keys = Object.keys(filemap);
         return Promise.mapSeries(keys, key => {
             // console.log("Processing", key, filemap[key]);
@@ -46,37 +48,78 @@ const FtpDeployer = function() {
     };
     // Creates a remote directory and uploads all of the files in it
     // Resolves a confirmation message on success
-    this.makeAndUpload = (config, relDir, fnames) => {
-        let newDirectory = upath.join(config.remoteRoot, relDir);
-        return this.makeDir(newDirectory, true).then(() => {
-            // console.log("newDirectory", newDirectory);
-            return Promise.mapSeries(fnames, fname => {
-                let tmpFileName = upath.join(config.localRoot, relDir, fname);
-                let tmp = fs.readFileSync(tmpFileName);
-                this.eventObject["filename"] = upath.join(relDir, fname);
-
-                this.emit("uploading", this.eventObject);
-
-                return this.ftp
-                    .put(tmp, upath.join(config.remoteRoot, relDir, fname))
-                    .then(() => {
-                        this.eventObject.transferredFileCount++;
-                        this.emit("uploaded", this.eventObject);
-                        return Promise.resolve("uploaded " + tmpFileName);
-                    })
-                    .catch(err => {
-                        this.eventObject["error"] = err;
-                        this.emit("upload-error", this.eventObject);
-                        // if continue on error....
-                        return Promise.reject(err);
+    this.makeAndUpload = (config, relDir, localFileMetas) => {
+        let newRemoteDir = upath.join(config.remoteRoot, relDir);
+        // console.log("newRemoteDir", newRemoteDir);
+        // ensure directory we need exists. Will resolve if dir already exists
+        return this.makeDir(newRemoteDir)
+            .then(() => {
+                if (config.newFilesOnly) {
+                    return this.ftp.list(newRemoteDir).then(remoteStats => {
+                        return remoteStats.reduce((acc, item) => {
+                            acc[item.name] = {
+                                size: item.size,
+                                date: new Date(item.date).getTime()
+                            };
+                            return acc;
+                        }, {});
                     });
+                } else {
+                    // as we will not be checking for new files, no need to stat the remote directory
+                    return Promise.resolve({});
+                }
+            })
+            .then(remoteStats => {
+                return Promise.mapSeries(localFileMetas, meta => {
+                    // console.log("remoteStats", remoteStats[meta.fname], meta);
+                    let tmpLocalName = upath.join(
+                        config.localRoot,
+                        relDir,
+                        meta.fname
+                    );
+
+                    if (
+                        config.newFilesOnly &&
+                        remoteStats[meta.fname] &&
+                        remoteStats[meta.fname].size == meta.size &&
+                        remoteStats[meta.fname].date >= meta.mtime
+                    ) {
+                        this.emit("log", "skipping: " + meta.fname);
+                        return Promise.resolve("skipped " + tmpLocalName);
+                    }
+
+                    let localFile = fs.readFileSync(tmpLocalName);
+                    this.eventObject["filename"] = upath.join(
+                        relDir,
+                        meta.fname
+                    );
+
+                    this.emit("uploading", this.eventObject);
+
+                    return this.ftp
+                        .put(
+                            localFile,
+                            upath.join(config.remoteRoot, relDir, meta.fname)
+                        )
+                        .then(() => {
+                            this.eventObject.transferredFileCount++;
+                            this.emit("uploaded", this.eventObject);
+                            return Promise.resolve("uploaded " + tmpLocalName);
+                        })
+                        .catch(err => {
+                            this.eventObject["error"] = err;
+                            this.emit("upload-error", this.eventObject);
+                            // if continue on error....
+                            return Promise.reject(err);
+                        });
+                });
             });
-        });
     };
 
     // connects to the server, Resolves the config on success
     this.connect = config => {
         this.ftp = new PromiseFtp();
+        this.emit("log", "Attempting to connect to: " + config.host);
 
         return this.ftp.connect(config).then(serverMessage => {
             this.emit("log", "Connected to: " + config.host);

src/lib.js

@@ -70,9 +70,12 @@ function parseLocal(includes, excludes, localRootDir, relDir) {
         const currItem = path.join(fullDir, item);
         const newRelDir = path.relative(localRootDir, currItem);
 
-        if (fs.lstatSync(currItem).isDirectory()) {
+        const stat = fs.lstatSync(currItem);
+
+        if (stat.isDirectory()) {
             // currItem is a directory. Recurse and attach to accumulator
             let tmp = parseLocal(includes, excludes, localRootDir, newRelDir);
+            // remove any empty directories
             for (let key in tmp) {
                 if (tmp[key].length == 0) {
                     delete tmp[key];
@@ -83,8 +86,9 @@ function parseLocal(includes, excludes, localRootDir, relDir) {
             // currItem is a file
             // acc[relDir] is always created at previous iteration
             if (canIncludePath(includes, excludes, newRelDir)) {
-                // console.log("including", currItem);
-                acc[relDir].push(item);
+                let tmp = { fname: item, mtime: stat.mtimeMs, size: stat.size };
+                // console.log("including", tmp);
+                acc[relDir].push(tmp);
                 return acc;
             }
         }
@@ -133,21 +137,22 @@ mkDirExists = (ftp, dir) => {
     // Make the directory using recursive expand
     return ftp.mkdir(dir, true).catch(err => {
         if (err.message.startsWith("EEXIST")) {
+            // directory already exists - convert this to good news
             return Promise.resolve();
         } else {
+            // something really went wrong
             console.log("[mkDirExists]", err.message);
-            // console.log(Object.getOwnPropertyNames(err));
             return Promise.reject(err);
         }
     });
 };
 
 module.exports = {
-    checkIncludes: checkIncludes,
-    getPassword: getPassword,
-    parseLocal: parseLocal,
-    canIncludePath: canIncludePath,
-    countFiles: countFiles,
-    mkDirExists: mkDirExists,
-    deleteDir: deleteDir
+    checkIncludes,
+    getPassword,
+    parseLocal,
+    canIncludePath,
+    countFiles,
+    mkDirExists,
+    deleteDir
 };

src/lib.spec.js

@@ -48,46 +48,55 @@ describe("canIncludePath", () => {
     });
 });
 
-describe("dirParseSync", () => {
+describe("parseLocal tests", () => {
     it("should throw on a bad start directory", () => {
         const testDir = "./throw";
-        assert.throws(() => lib.parseLocal(["*"], testDir, testDir), Error);
+        assert.throws(() =>
+            simplify(lib.parseLocal(["*"], testDir, testDir), Error)
+        );
     });
     it("should traverse simple directory", () => {
         const rootDir = path.join(__dirname, "../test/simple");
-        assert.deepEqual(lib.parseLocal(["*"], [], rootDir, "/"), {
+        assert.deepEqual(simplify(lib.parseLocal(["*"], [], rootDir, "/")), {
             "/": ["test-inside-root.txt"],
             inner: ["test-inside-root.excl"]
         });
     });
     it("should respect a negate (!)", () => {
         const rootDir = path.join(__dirname, "../test/simple");
-        assert.deepEqual(lib.parseLocal(["!*.excl"], [], rootDir, "/"), {
-            "/": ["test-inside-root.txt"]
-        });
+        assert.deepEqual(
+            simplify(lib.parseLocal(["!*.excl"], [], rootDir, "/")),
+            {
+                "/": ["test-inside-root.txt"]
+            }
+        );
     });
     it("should respect excludes (directory)", () => {
         const rootDir = path.join(__dirname, "../test/local");
         assert.deepEqual(
-            lib.parseLocal(
-                [".*", "*", "*/**"],
-                [".*", "*", "*/**"],
-                rootDir,
-                "/"
+            simplify(
+                lib.parseLocal(
+                    [".*", "*", "*/**"],
+                    [".*", "*", "*/**"],
+                    rootDir,
+                    "/"
+                )
             ),
             { "/": [] }
         );
     });
     it("should exclude dot files/dirs", () => {
         const rootDir = path.join(__dirname, "../test/test2");
         assert.deepEqual(
-            lib.parseLocal(
-                ["*", "*/**"],
-                ["n_modules/**/*", "n_modules/**/.*"],
-                rootDir,
-                "/"
+            simplify(
+                lib.parseLocal(
+                    ["*", "*/**"],
+                    ["n_modules/**/*", "n_modules/**/.*"],
+                    rootDir,
+                    "/"
+                )
             ),
-            { "/": [],  src: [ 'index.js' ] }
+            { "/": [], src: ["index.js"] }
         );
     });
     it("should traverse test directory", () => {
@@ -96,7 +105,7 @@ describe("dirParseSync", () => {
             "folderA/folderB/FolderC": ["test-inside-c.txt"]
         });
         assert.deepEqual(
-            lib.parseLocal(["*"], [".excludeme/**/*"], rootDir, "/"),
+            simplify(lib.parseLocal(["*"], [".excludeme/**/*"], rootDir, "/")),
             exp2
         );
     });
@@ -111,3 +120,20 @@ let exp = {
         "test-inside-d-2.txt"
     ]
 };
+
+/*
+simplify({ '/': [ { 'test-inside-root.txt': 1525592919000 } ],
+  inner: [ { 'test-inside-root.excl': 1550948905974.2664 } ] })
+
+{
+    "/": ["test-inside-root.txt"],
+    inner: ["test-inside-root.excl"]
+}
+*/
+function simplify(obj) {
+    let keys = Object.keys(obj);
+    return keys.reduce((acc, key) => {
+        acc[key] = obj[key].map(o => o.fname);
+        return acc;
+    }, {});
+}