diff --git a/README.md b/README.md index c195e00..fd0d904 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ npm install --save-dev ftp-deploy ## Usage The most basic usage: + ```js var FtpDeploy = require("ftp-deploy"); var ftpDeploy = new FtpDeploy(); @@ -25,12 +26,20 @@ var config = { port: 21, localRoot: __dirname + "/local-folder", remoteRoot: "/public_html/remote-folder/", - // include: ["*", "**/*"], // this would upload everything except dot files - include: ["*.php", "dist/*", ".*"], + // typical wordpress includes + include: ["*.php", "dist/*"], + // include: ["*", "**/*", ".*"], // upload everything, including dot files // e.g. exclude sourcemaps, and ALL files in node_modules (including dot files) - exclude: ["dist/**/*.map", "node_modules/**", "node_modules/**/.*", ".git/**"], - // delete ALL existing files at destination before uploading, if true + exclude: [ + "dist/**/*.map", + "node_modules/**", + "node_modules/**/.*", + ".git/**" + ], + // if true, delete ALL existing files (expcet dot files) at destination before uploading deleteRemote: false, + // if true, only uploads changed files (based on last modified date and file size) + newFilesOnly: false, // Passive mode is forced (EPSV command is not sent) forcePasv: true }; @@ -48,10 +57,11 @@ ftpDeploy.deploy(config, function(err, res) { }); ``` -**Note:** - - in version 2 the config file expects a field of `user` rather than `username` in 1.x. - - The config file is passed as-is to Promise-FTP. - - I create a file - e.g. deploy.js - in the root of my source code and add a script to its `package.json` so that I can `npm run deploy`. +**Note:** + +- in version 2 the config file expects a field of `user` rather than `username` in 1.x. +- The config file is passed as-is to [promise-ftp](https://github.com/realtymaps/promise-ftp) - see their docs for more options. +- I create a file - e.g. deploy.js - in the root of my source code and add a script to its `package.json` so that I can `npm run deploy`. ```json "scripts": { @@ -68,7 +78,7 @@ These are lists of [minimatch globs](https://github.com/isaacs/minimatch). ftp-d ## Events -ftp-deploy reports to clients using events. To get the output you need to implement watchers for "uploading", "uploaded" and "log": +ftp-deploy reports progress using events. These can be accessed - if desired - by registering listeners for "uploading", "uploaded", "upload-error" and "log": ```js ftpDeploy.on("uploading", function(data) { @@ -79,12 +89,12 @@ ftpDeploy.on("uploading", function(data) { ftpDeploy.on("uploaded", function(data) { console.log(data); // same data as uploading event }); -ftpDeploy.on("log", function(data) { - console.log(data); // same data as uploading event -}); ftpDeploy.on("upload-error", function(data) { console.log(data.err); // data will also include filename, relativePath, and other goodies }); +ftpDeploy.on("log", function(data) { + console.log(data); // same data as uploading event +}); ``` ## Testing @@ -105,6 +115,5 @@ npm test ## ToDo - - re-enable continueOnError - - update newer files only (PR welcome) - +- re-enable continueOnError +- update newer files only (PR welcome) diff --git a/package.json b/package.json index 100e910..ba3cb53 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "ftp-deploy", "version": "2.3.6", "author": "Simon Hampton", - "description": "Ftp a folder from your local disk to an ftp destination", + "description": "Deploy files to a remote Ftp location", "main": "src/ftp-deploy", "scripts": { "test": "mocha **/**.spec.js", @@ -52,4 +52,4 @@ "prettier": { "tabWidth": 4 } -} \ No newline at end of file +} diff --git a/playground/test_script.js b/playground/test_script.js index 56038ba..5eff939 100644 --- a/playground/test_script.js +++ b/playground/test_script.js @@ -9,10 +9,11 @@ const config = { host: "localhost", port: 2121, localRoot: path.join(__dirname, "local"), - remoteRoot: "/f", - deleteRemote: true, - exclude: [], - include: ["test-inside-root.txt"] + remoteRoot: "/ftp", + deleteRemote: false, + newFilesOnly: true, + exclude: ["*/index.html"], + include: ["index.html", "folderA/*"] // include: ["**/*", "*", ".*"] }; diff --git a/src/ftp-deploy.js b/src/ftp-deploy.js index 0f1c55a..7713953 100644 --- a/src/ftp-deploy.js +++ b/src/ftp-deploy.js @@ -30,6 +30,8 @@ const FtpDeployer = function() { }; this.makeAllAndUpload = function(remoteDir, filemap) { + // TODO pass on the full object + let keys = Object.keys(filemap); return Promise.mapSeries(keys, key => { // console.log("Processing", key, filemap[key]); @@ -46,37 +48,78 @@ const FtpDeployer = function() { }; // Creates a remote directory and uploads all of the files in it // Resolves a confirmation message on success - this.makeAndUpload = (config, relDir, fnames) => { - let newDirectory = upath.join(config.remoteRoot, relDir); - return this.makeDir(newDirectory, true).then(() => { - // console.log("newDirectory", newDirectory); - return Promise.mapSeries(fnames, fname => { - let tmpFileName = upath.join(config.localRoot, relDir, fname); - let tmp = fs.readFileSync(tmpFileName); - this.eventObject["filename"] = upath.join(relDir, fname); - - this.emit("uploading", this.eventObject); - - return this.ftp - .put(tmp, upath.join(config.remoteRoot, relDir, fname)) - .then(() => { - this.eventObject.transferredFileCount++; - this.emit("uploaded", this.eventObject); - return Promise.resolve("uploaded " + tmpFileName); - }) - .catch(err => { - this.eventObject["error"] = err; - this.emit("upload-error", this.eventObject); - // if continue on error.... - return Promise.reject(err); + this.makeAndUpload = (config, relDir, localFileMetas) => { + let newRemoteDir = upath.join(config.remoteRoot, relDir); + // console.log("newRemoteDir", newRemoteDir); + // ensure directory we need exists. Will resolve if dir already exists + return this.makeDir(newRemoteDir) + .then(() => { + if (config.newFilesOnly) { + return this.ftp.list(newRemoteDir).then(remoteStats => { + return remoteStats.reduce((acc, item) => { + acc[item.name] = { + size: item.size, + date: new Date(item.date).getTime() + }; + return acc; + }, {}); }); + } else { + // as we will not be checking for new files, no need to stat the remote directory + return Promise.resolve({}); + } + }) + .then(remoteStats => { + return Promise.mapSeries(localFileMetas, meta => { + // console.log("remoteStats", remoteStats[meta.fname], meta); + let tmpLocalName = upath.join( + config.localRoot, + relDir, + meta.fname + ); + + if ( + config.newFilesOnly && + remoteStats[meta.fname] && + remoteStats[meta.fname].size == meta.size && + remoteStats[meta.fname].date >= meta.mtime + ) { + this.emit("log", "skipping: " + meta.fname); + return Promise.resolve("skipped " + tmpLocalName); + } + + let localFile = fs.readFileSync(tmpLocalName); + this.eventObject["filename"] = upath.join( + relDir, + meta.fname + ); + + this.emit("uploading", this.eventObject); + + return this.ftp + .put( + localFile, + upath.join(config.remoteRoot, relDir, meta.fname) + ) + .then(() => { + this.eventObject.transferredFileCount++; + this.emit("uploaded", this.eventObject); + return Promise.resolve("uploaded " + tmpLocalName); + }) + .catch(err => { + this.eventObject["error"] = err; + this.emit("upload-error", this.eventObject); + // if continue on error.... + return Promise.reject(err); + }); + }); }); - }); }; // connects to the server, Resolves the config on success this.connect = config => { this.ftp = new PromiseFtp(); + this.emit("log", "Attempting to connect to: " + config.host); return this.ftp.connect(config).then(serverMessage => { this.emit("log", "Connected to: " + config.host); diff --git a/src/lib.js b/src/lib.js index 8d66eb1..8f97bc6 100644 --- a/src/lib.js +++ b/src/lib.js @@ -70,9 +70,12 @@ function parseLocal(includes, excludes, localRootDir, relDir) { const currItem = path.join(fullDir, item); const newRelDir = path.relative(localRootDir, currItem); - if (fs.lstatSync(currItem).isDirectory()) { + const stat = fs.lstatSync(currItem); + + if (stat.isDirectory()) { // currItem is a directory. Recurse and attach to accumulator let tmp = parseLocal(includes, excludes, localRootDir, newRelDir); + // remove any empty directories for (let key in tmp) { if (tmp[key].length == 0) { delete tmp[key]; @@ -83,8 +86,9 @@ function parseLocal(includes, excludes, localRootDir, relDir) { // currItem is a file // acc[relDir] is always created at previous iteration if (canIncludePath(includes, excludes, newRelDir)) { - // console.log("including", currItem); - acc[relDir].push(item); + let tmp = { fname: item, mtime: stat.mtimeMs, size: stat.size }; + // console.log("including", tmp); + acc[relDir].push(tmp); return acc; } } @@ -133,21 +137,22 @@ mkDirExists = (ftp, dir) => { // Make the directory using recursive expand return ftp.mkdir(dir, true).catch(err => { if (err.message.startsWith("EEXIST")) { + // directory already exists - convert this to good news return Promise.resolve(); } else { + // something really went wrong console.log("[mkDirExists]", err.message); - // console.log(Object.getOwnPropertyNames(err)); return Promise.reject(err); } }); }; module.exports = { - checkIncludes: checkIncludes, - getPassword: getPassword, - parseLocal: parseLocal, - canIncludePath: canIncludePath, - countFiles: countFiles, - mkDirExists: mkDirExists, - deleteDir: deleteDir + checkIncludes, + getPassword, + parseLocal, + canIncludePath, + countFiles, + mkDirExists, + deleteDir }; diff --git a/src/lib.spec.js b/src/lib.spec.js index e8446ec..9da45b6 100644 --- a/src/lib.spec.js +++ b/src/lib.spec.js @@ -48,32 +48,39 @@ describe("canIncludePath", () => { }); }); -describe("dirParseSync", () => { +describe("parseLocal tests", () => { it("should throw on a bad start directory", () => { const testDir = "./throw"; - assert.throws(() => lib.parseLocal(["*"], testDir, testDir), Error); + assert.throws(() => + simplify(lib.parseLocal(["*"], testDir, testDir), Error) + ); }); it("should traverse simple directory", () => { const rootDir = path.join(__dirname, "../test/simple"); - assert.deepEqual(lib.parseLocal(["*"], [], rootDir, "/"), { + assert.deepEqual(simplify(lib.parseLocal(["*"], [], rootDir, "/")), { "/": ["test-inside-root.txt"], inner: ["test-inside-root.excl"] }); }); it("should respect a negate (!)", () => { const rootDir = path.join(__dirname, "../test/simple"); - assert.deepEqual(lib.parseLocal(["!*.excl"], [], rootDir, "/"), { - "/": ["test-inside-root.txt"] - }); + assert.deepEqual( + simplify(lib.parseLocal(["!*.excl"], [], rootDir, "/")), + { + "/": ["test-inside-root.txt"] + } + ); }); it("should respect excludes (directory)", () => { const rootDir = path.join(__dirname, "../test/local"); assert.deepEqual( - lib.parseLocal( - [".*", "*", "*/**"], - [".*", "*", "*/**"], - rootDir, - "/" + simplify( + lib.parseLocal( + [".*", "*", "*/**"], + [".*", "*", "*/**"], + rootDir, + "/" + ) ), { "/": [] } ); @@ -81,13 +88,15 @@ describe("dirParseSync", () => { it("should exclude dot files/dirs", () => { const rootDir = path.join(__dirname, "../test/test2"); assert.deepEqual( - lib.parseLocal( - ["*", "*/**"], - ["n_modules/**/*", "n_modules/**/.*"], - rootDir, - "/" + simplify( + lib.parseLocal( + ["*", "*/**"], + ["n_modules/**/*", "n_modules/**/.*"], + rootDir, + "/" + ) ), - { "/": [], src: [ 'index.js' ] } + { "/": [], src: ["index.js"] } ); }); it("should traverse test directory", () => { @@ -96,7 +105,7 @@ describe("dirParseSync", () => { "folderA/folderB/FolderC": ["test-inside-c.txt"] }); assert.deepEqual( - lib.parseLocal(["*"], [".excludeme/**/*"], rootDir, "/"), + simplify(lib.parseLocal(["*"], [".excludeme/**/*"], rootDir, "/")), exp2 ); }); @@ -111,3 +120,20 @@ let exp = { "test-inside-d-2.txt" ] }; + +/* +simplify({ '/': [ { 'test-inside-root.txt': 1525592919000 } ], + inner: [ { 'test-inside-root.excl': 1550948905974.2664 } ] }) + +{ + "/": ["test-inside-root.txt"], + inner: ["test-inside-root.excl"] +} +*/ +function simplify(obj) { + let keys = Object.keys(obj); + return keys.reduce((acc, key) => { + acc[key] = obj[key].map(o => o.fname); + return acc; + }, {}); +}