From 1f85b758872167ddca2c6dd18fe5d7cef280b466 Mon Sep 17 00:00:00 2001 From: Stuart Miller Date: Thu, 8 Jun 2017 13:30:19 +1200 Subject: [PATCH 01/12] Write tar files to a stream instead of to the filesystem first. This improves performance when writing to a stream (i.e. there is no need to wait for the contents to be written to disk first), and can help solve EMFILE errors. This is achieved by using tar-stream instead of regular tar. This commit introduces a small shim in front of the file system and tar-stream to allow both direct disk writes and tar file stream writes. It then uses the appropriate shim (file system or tar-stream) throughout the codebase to create directories and store files. --- .editorconfig | 4 + index.js | 612 ++++++++++++++++++++++++-------------------------- index.min.js | 308 +++++++++++++------------ package.json | 3 +- 4 files changed, 462 insertions(+), 465 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5e9d853 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,4 @@ +[*] +indent_style=space +indent_size=2 + diff --git a/index.js b/index.js index 9d57b59..4b30672 100644 --- a/index.js +++ b/index.js @@ -13,18 +13,99 @@ * initialize module */ var systemRegex = /^system\./; -var fs = require('graceful-fs'); +var fs = require('graceful-fs').gracefulify(require('fs-extra')); var path = require('path'); var BSON; var logger; var meta; +var fileSystemDocumentStore = function (root) { + var dbDir = root; + var makeDir = function (pathname, next) { + fs.stat(pathname, function (err, stats) { + + if (err && err.code === 'ENOENT') { // no file or dir + logger('make dir at ' + pathname); + return fs.mkdirp(pathname, function (err) { + + next(err, pathname); + }); + + } else if (stats && stats.isDirectory() === false) { // pathname is a file + logger('unlink file at ' + pathname); + return fs.unlink(pathname, function (err) { + + if (err) { // unlink fail. permission maybe + return next(err); + } + + logger('make dir at ' + pathname); + fs.mkdir(pathname, function (err) { + + next(err, pathname); + }); + }); + + } else { // already a dir + next(null, pathname); + } + }); + }; + return { + addDatabase: function (dbName, next) { + dbDir = path.join(root, dbName); + return makeDir(dbDir, next); + }, + addCollection: function addCollection(relativePath, next) { + var pathname = path.join(dbDir, relativePath); + return makeDir(pathname, next); + }, + store: function store(collectionName, relativePath, content, callback) { + fs.writeFile(path.join(dbDir, collectionName, relativePath), content, callback); + }, + close: function () { + } + }; +}; + +var streamingDocumentStore = function (root, stream) { + var tar = require('tar-stream'); + var pack = tar.pack(); // pack is a streams2 stream + pack.pipe(stream); + + var dbDir = root; + return { + addDatabase: function addDatabase(dbName, next) { + dbDir = path.join(root, dbName); + pack.entry({name: dbDir, type: 'directory'}); + next(); + }, + + addCollection: function addCollection(filename, next) { + if (filename !== '') { + pack.entry({name: path.join(dbDir, filename), type: 'directory'}); + } + next(); + }, + + store: function store(collectionName, filename, content, callback) { + pack.entry({name: path.join(dbDir, collectionName, filename)}, content); + if (callback) { + callback(); + } + }, + close: function close() { + pack.finalize(); + } + }; +}; + /* * functions */ /** * error handler - * + * * @function error * @param {Object} err - raised error */ @@ -37,142 +118,53 @@ function error(err) { /** * save collection metadata to file - * + * * @function writeMetadata * @param {Object} collection - db collection * @param {String} metadata - path of metadata * @param {Function} next - callback */ -function writeMetadata(collection, metadata, next) { - - return collection.indexes(function(err, indexes) { - - if (err) { - return next(err); - } - - fs.writeFile(metadata + collection.collectionName, JSON.stringify(indexes), - next); - }); -} - -/** - * make dir - * - * @function makeDir - * @param {String} pathname - pathname of dir - * @param {Function} next - callback - */ -function makeDir(pathname, next) { - - fs.stat(pathname, function(err, stats) { - - if (err && err.code === 'ENOENT') { // no file or dir - logger('make dir at ' + pathname); - return fs.mkdir(pathname, function(err) { - - next(err, pathname); - }); - - } else if (stats && stats.isDirectory() === false) { // pathname is a file - logger('unlink file at ' + pathname); - return fs.unlink(pathname, function(err) { - - if (err) { // unlink fail. permission maybe - return next(err); - } - - logger('make dir at ' + pathname); - fs.mkdir(pathname, function(err) { - - next(err, pathname); - }); - }); - - } else { // already a dir - next(null, pathname); - } - }); -} - -/** - * remove dir - * - * @function rmDir - * @param {String} pathname - path of dir - * @param {Function} [next] - callback - */ -function rmDir(pathname, next) { - - fs.readdirSync(pathname).forEach(function(first) { // database - - var database = pathname + first; - if (fs.statSync(database).isDirectory() === false) { - return next(Error('path is not a Directory')); - } - - var metadata = ''; - var collections = fs.readdirSync(database); - var metadataPath = path.join(database, '.metadata'); - if (fs.existsSync(metadataPath) === true) { - metadata = metadataPath + path.sep; - delete collections[collections.indexOf('.metadata')]; // undefined is not a dir - } - - collections.forEach(function(second) { // collection - - var collection = path.join(database, second); - if (fs.statSync(collection).isDirectory() === false) { - return; - } - - fs.readdirSync(collection).forEach(function(third) { // document - - var document = path.join(collection, third); - fs.unlinkSync(document); - return next ? next(null, document) : ''; - }); - - if (metadata !== '') { - fs.unlinkSync(metadata + second); +function writeMetadata(documentStore) { + return function (collection, metadata, next) { + return collection.indexes(function (err, indexes) { + if (err) { + return next(err); } - fs.rmdirSync(collection); + documentStore.store('.metadata', collection.collectionName, JSON.stringify(indexes), next); }); - - if (metadata !== '') { - fs.rmdirSync(metadata); - } - return fs.rmdirSync(database); - }); + }; } + /** * JSON parser async - * + * * @function toJson * @param {Objecy} doc - document from stream * @param {String} collectionPath - path of collection */ -function toJsonAsync(doc, collectionPath) { - - fs.writeFile(collectionPath + doc._id + '.json', JSON.stringify(doc)); +function toJsonAsync(documentStore) { + return function (doc, collectionPath) { + documentStore.store(collectionPath, doc._id + '.json', JSON.stringify(doc)); + }; } /** * BSON parser async - * + * * @function toBson * @param {Objecy} doc - document from stream * @param {String} collectionPath - path of collection */ -function toBsonAsync(doc, collectionPath) { - - fs.writeFile(collectionPath + doc._id + '.bson', BSON.serialize(doc)); +function toBsonAsync(documentStore) { + return function (doc, collectionPath) { + documentStore.store(collectionPath, doc._id + '.bson', BSON.serialize(doc)); + }; } /** * get data from all available collections - * + * * @function allCollections * @param {Object} db - database * @param {String} name - path of dir @@ -181,52 +173,54 @@ function toBsonAsync(doc, collectionPath) { * @param {Function} parser - data parser * @param {Function} next - callback */ -function allCollections(db, name, query, metadata, parser, next) { +function allCollections(documentStore) { + return function (db, name, query, metadata, parser, next) { - return db.collections(function(err, collections) { + return db.collections(function (err, collections) { - if (err) { - return next(err); - } - - var last = ~~collections.length, index = 0; - if (last === 0) { // empty set - return next(err); - } - - collections.forEach(function(collection) { + if (err) { + return next(err); + } - if (systemRegex.test(collection.collectionName) === true) { - return last === ++index ? next(null) : null; + var last = ~~collections.length, index = 0; + if (last === 0) { // empty set + return next(err); } - logger('select collection ' + collection.collectionName); - makeDir(name + collection.collectionName + path.sep, function(err, name) { + collections.forEach(function (collection) { - if (err) { - return last === ++index ? next(err) : error(err); + if (systemRegex.test(collection.collectionName) === true) { + return last === ++index ? next(null) : null; } - meta(collection, metadata, function() { + logger('select collection ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function (err) { + + if (err) { + return last === ++index ? next(err) : error(err); + } - var stream = collection.find(query).snapshot(true).stream(); + meta(collection, metadata, function () { - stream.once('end', function() { + var stream = collection.find(query).snapshot(true).stream(); - return last === ++index ? next(null) : null; - }).on('data', function(doc) { + stream.once('end', function () { - parser(doc, name); + return last === ++index ? next(null) : null; + }).on('data', function (doc) { + + parser(doc, collection.collectionName); + }); }); }); }); }); - }); + }; } /** * get data from all available collections without query (parallelCollectionScan) - * + * * @function allCollectionsScan * @param {Object} db - database * @param {String} name - path of dir @@ -235,70 +229,72 @@ function allCollections(db, name, query, metadata, parser, next) { * @param {Function} parser - data parser * @param {Function} next - callback */ -function allCollectionsScan(db, name, numCursors, metadata, parser, next) { - - return db.collections(function(err, collections) { - - if (err) { - return next(err); - } +function allCollectionsScan(documentStore) { + return function (db, name, numCursors, metadata, parser, next) { - var last = ~~collections.length, index = 0; - if (last === 0) { // empty set - return next(null); - } + return db.collections(function (err, collections) { - collections.forEach(function(collection) { + if (err) { + return next(err); + } - if (systemRegex.test(collection.collectionName) === true) { - return last === ++index ? next(null) : null; + var last = ~~collections.length, index = 0; + if (last === 0) { // empty set + return next(null); } - logger('select collection scan ' + collection.collectionName); - makeDir(name + collection.collectionName + path.sep, function(err, name) { + collections.forEach(function (collection) { - if (err) { - return last === ++index ? next(err) : error(err); + if (systemRegex.test(collection.collectionName) === true) { + return last === ++index ? next(null) : null; } - meta(collection, metadata, function() { + logger('select collection scan ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function (err) { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function(err, cursors) { + if (err) { + return last === ++index ? next(err) : error(err); + } - if (err) { - return last === ++index ? next(err) : error(err); - } + meta(collection, metadata, function () { - var ii, cursorsDone; - ii = cursorsDone = ~~cursors.length; - if (ii === 0) { // empty set - return last === ++index ? next(null) : null; - } + collection.parallelCollectionScan({ + numCursors: numCursors + }, function (err, cursors) { + + if (err) { + return last === ++index ? next(err) : error(err); + } - for (var i = 0; i < ii; ++i) { - cursors[i].once('end', function() { + var ii, cursorsDone; + ii = cursorsDone = ~~cursors.length; + if (ii === 0) { // empty set + return last === ++index ? next(null) : null; + } - // No more cursors let's ensure we got all results - if (--cursorsDone === 0) { - return last === ++index ? next(null) : null; - } - }).on('data', function(doc) { + for (var i = 0; i < ii; ++i) { + cursors[i].once('end', function () { - parser(doc, name); - }); - } + // No more cursors let's ensure we got all results + if (--cursorsDone === 0) { + return last === ++index ? next(null) : null; + } + }).on('data', function (doc) { + + parser(doc, collection.collectionName); + }); + } + }); }); }); }); }); - }); + }; } /** * get data from some collections - * + * * @function someCollections * @param {Object} db - database * @param {String} name - path of dir @@ -308,50 +304,52 @@ function allCollectionsScan(db, name, numCursors, metadata, parser, next) { * @param {Function} next - callback * @param {Array} collections - selected collections */ -function someCollections(db, name, query, metadata, parser, next, collections) { +function someCollections(documentStore) { + return function (db, name, query, metadata, parser, next, collections) { - var last = ~~collections.length, index = 0; - if (last === 0) { - return next(null); - } - - collections.forEach(function(collection) { - - db.collection(collection, { - strict: true - }, function(err, collection) { + var last = ~~collections.length, index = 0; + if (last === 0) { + return next(null); + } - if (err) { // returns an error if the collection does not exist - return last === ++index ? next(err) : error(err); - } + collections.forEach(function (collection) { - logger('select collection ' + collection.collectionName); - makeDir(name + collection.collectionName + path.sep, function(err, name) { + db.collection(collection, { + strict: true + }, function (err, collection) { - if (err) { + if (err) { // returns an error if the collection does not exist return last === ++index ? next(err) : error(err); } - meta(collection, metadata, function() { + logger('select collection ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function (err) { + + if (err) { + return last === ++index ? next(err) : error(err); + } + + meta(collection, metadata, function () { - var stream = collection.find(query).snapshot(true).stream(); + var stream = collection.find(query).snapshot(true).stream(); - stream.once('end', function() { + stream.once('end', function () { - return last === ++index ? next(null) : null; - }).on('data', function(doc) { + return last === ++index ? next(null) : null; + }).on('data', function (doc) { - parser(doc, name); + parser(doc, collection.collectionName); + }); }); }); }); }); - }); + }; } /** * get data from some collections without query (parallelCollectionScan) - * + * * @function someCollectionsScan * @param {Object} db - database * @param {String} name - path of dir @@ -361,69 +359,71 @@ function someCollections(db, name, query, metadata, parser, next, collections) { * @param {Function} next - callback * @param {Array} collections - selected collections */ -function someCollectionsScan(db, name, numCursors, metadata, parser, next, - collections) { +function someCollectionsScan(documentStore) { + return function (db, name, numCursors, metadata, parser, next, + collections) { - var last = ~~collections.length, index = 0; - if (last === 0) { // empty set - return next(null); - } - - collections.forEach(function(collection) { - - db.collection(collection, { - strict: true - }, function(err, collection) { + var last = ~~collections.length, index = 0; + if (last === 0) { // empty set + return next(null); + } - if (err) { // returns an error if the collection does not exist - return last === ++index ? next(err) : error(err); - } + collections.forEach(function (collection) { - logger('select collection scan ' + collection.collectionName); - makeDir(name + collection.collectionName + path.sep, function(err, name) { + db.collection(collection, { + strict: true + }, function (err, collection) { - if (err) { + if (err) { // returns an error if the collection does not exist return last === ++index ? next(err) : error(err); } - meta(collection, metadata, function() { + logger('select collection scan ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function (err) { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function(err, cursors) { + if (err) { + return last === ++index ? next(err) : error(err); + } - if (err) { - return last === ++index ? next(err) : error(err); - } + meta(collection, metadata, function () { - var ii, cursorsDone; - ii = cursorsDone = ~~cursors.length; - if (ii === 0) { // empty set - return last === ++index ? next(null) : null; - } + collection.parallelCollectionScan({ + numCursors: numCursors + }, function (err, cursors) { + + if (err) { + return last === ++index ? next(err) : error(err); + } - for (var i = 0; i < ii; ++i) { - cursors[i].once('end', function() { + var ii, cursorsDone; + ii = cursorsDone = ~~cursors.length; + if (ii === 0) { // empty set + return last === ++index ? next(null) : null; + } - // No more cursors let's ensure we got all results - if (--cursorsDone === 0) { - return last === ++index ? next(null) : null; - } - }).on('data', function(doc) { + for (var i = 0; i < ii; ++i) { + cursors[i].once('end', function () { - parser(doc, name); - }); - } + // No more cursors let's ensure we got all results + if (--cursorsDone === 0) { + return last === ++index ? next(null) : null; + } + }).on('data', function (doc) { + + parser(doc, collection.collectionName); + }); + } + }); }); }); }); }); - }); + }; } /** * function wrapper - * + * * @function wrapper * @param {Object} my - parsed options */ @@ -437,31 +437,31 @@ function wrapper(my) { case 'bson': BSON = require('bson'); BSON = new BSON(); - parser = toBsonAsync; + parser = toBsonAsync(my.documentStore); break; case 'json': // JSON error on ObjectId, Date and Long - parser = toJsonAsync; + parser = toJsonAsync(my.documentStore); break; default: throw new Error('missing parser option'); } } - var discriminator = allCollections; + var discriminator = allCollections(my.documentStore); if (my.collections !== null) { - discriminator = someCollections; + discriminator = someCollections(my.documentStore); if (my.numCursors) { - discriminator = someCollectionsScan; + discriminator = someCollectionsScan(my.documentStore); my.query = my.numCursors; // override } } else if (my.numCursors) { - discriminator = allCollectionsScan; + discriminator = allCollectionsScan(my.documentStore); my.query = my.numCursors; // override } if (my.logger === null) { - logger = function() { + logger = function () { return; }; @@ -479,7 +479,7 @@ function wrapper(my) { logger('backup start'); var log = require('mongodb').Logger; log.setLevel('info'); - log.setCurrentLogger(function(msg) { + log.setCurrentLogger(function (msg) { return logger(msg); }); @@ -487,9 +487,9 @@ function wrapper(my) { var metadata = ''; if (my.metadata === true) { - meta = writeMetadata; + meta = writeMetadata(my.documentStore); } else { - meta = function(a, b, c) { + meta = function (a, b, c) { return c(); }; @@ -497,7 +497,7 @@ function wrapper(my) { /** * latest callback - * + * * @return {Null} */ function callback(err) { @@ -512,87 +512,48 @@ function wrapper(my) { } } - require('mongodb').MongoClient.connect(my.uri, my.options, function(err, db) { + require('mongodb').MongoClient.connect(my.uri, my.options, function (err, db) { logger('db open'); if (err) { return callback(err); } - var root = my.tar === null ? my.root : my.dir; - makeDir(root, function(err, name) { - - if (err) { - return callback(err); - } - - makeDir(name + db.databaseName + path.sep, function(err, name) { - - function go() { - - // waiting for `db.fsyncLock()` on node driver - return discriminator(db, name, my.query, metadata, parser, - function(err) { - - logger('db close'); - db.close(); - if (err) { - return callback(err); - } - - if (my.tar) { - makeDir(my.root, function(e, name) { + my.documentStore.addDatabase(db.databaseName, function (err, name) { - if (err) { - error(err); - } - - var dest; - if (my.stream) { // user stream - logger('send tar file to stream'); - dest = my.stream; - } else { // filesystem stream - logger('make tar file at ' + name + my.tar); - dest = fs.createWriteStream(name + my.tar); - } + function go() { - var packer = require('tar').Pack().on('error', callback).on( - 'end', function() { + // waiting for `db.fsyncLock()` on node driver + return discriminator(db, name, my.query, metadata, parser, + function (err) { - rmDir(root); - callback(null); - }); - - require('fstream').Reader({ - path: root + db.databaseName, - type: 'Directory' - }).on('error', callback).pipe(packer).pipe(dest); - }); + logger('db close'); + db.close(); + if (err) { + return callback(err); + } - } else { - callback(null); - } - }, my.collections); - } + my.documentStore.close(); + callback(null); + }, my.collections); + } - if (err) { - return callback(err); - } + if (err) { + return callback(err); + } - if (my.metadata === false) { - go(); - } else { - metadata = name + '.metadata' + path.sep; - makeDir(metadata, go); - } - }); + if (my.metadata === false) { + go(); + } else { + my.documentStore.addCollection('.metadata', go); + } }); }); } /** * option setting - * + * * @exports backup * @function backup * @param {Object} options - various options. Check README.md @@ -612,7 +573,6 @@ function backup(options) { } var my = { - dir: path.join(__dirname, 'dump', path.sep), uri: String(opt.uri), root: path.resolve(String(opt.root || '')) + path.sep, stream: opt.stream || null, @@ -626,9 +586,19 @@ function backup(options) { options: typeof opt.options === 'object' ? opt.options : {}, metadata: Boolean(opt.metadata) }; + if (my.tar && !my.stream) { + my.stream = fs.createWriteStream(path.join(my.root, my.tar)); + } if (my.stream) { my.tar = true; // override + my.documentStore = streamingDocumentStore(my.root, my.stream); + } else { + my.documentStore = fileSystemDocumentStore(my.root); } return wrapper(my); } module.exports = backup; + + + + diff --git a/index.min.js b/index.min.js index 6a35041..4e0293e 100644 --- a/index.min.js +++ b/index.min.js @@ -4,149 +4,133 @@ function error(err) { err && logger(err.message); } -function writeMetadata(collection, metadata, next) { - return collection.indexes(function(err, indexes) { - if (err) return next(err); - fs.writeFile(metadata + collection.collectionName, JSON.stringify(indexes), next); - }); -} - -function makeDir(pathname, next) { - fs.stat(pathname, function(err, stats) { - return err && "ENOENT" === err.code ? (logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { - next(err, pathname); - })) : stats && !1 === stats.isDirectory() ? (logger("unlink file at " + pathname), - fs.unlink(pathname, function(err) { +function writeMetadata(documentStore) { + return function(collection, metadata, next) { + return collection.indexes(function(err, indexes) { if (err) return next(err); - logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { - next(err, pathname); - }); - })) : void next(null, pathname); - }); -} - -function rmDir(pathname, next) { - fs.readdirSync(pathname).forEach(function(first) { - var database = pathname + first; - if (!1 === fs.statSync(database).isDirectory()) return next(Error("path is not a Directory")); - var metadata = "", collections = fs.readdirSync(database), metadataPath = path.join(database, ".metadata"); - return !0 === fs.existsSync(metadataPath) && (metadata = metadataPath + path.sep, - delete collections[collections.indexOf(".metadata")]), collections.forEach(function(second) { - var collection = path.join(database, second); - !1 !== fs.statSync(collection).isDirectory() && (fs.readdirSync(collection).forEach(function(third) { - var document = path.join(collection, third); - return fs.unlinkSync(document), next ? next(null, document) : ""; - }), "" !== metadata && fs.unlinkSync(metadata + second), fs.rmdirSync(collection)); - }), "" !== metadata && fs.rmdirSync(metadata), fs.rmdirSync(database); - }); + documentStore.store(".metadata", collection.collectionName, JSON.stringify(indexes), next); + }); + }; } -function toJsonAsync(doc, collectionPath) { - fs.writeFile(collectionPath + doc._id + ".json", JSON.stringify(doc)); +function toJsonAsync(documentStore) { + return function(doc, collectionPath) { + documentStore.store(collectionPath, doc._id + ".json", JSON.stringify(doc)); + }; } -function toBsonAsync(doc, collectionPath) { - fs.writeFile(collectionPath + doc._id + ".bson", BSON.serialize(doc)); +function toBsonAsync(documentStore) { + return function(doc, collectionPath) { + documentStore.store(collectionPath, doc._id + ".bson", BSON.serialize(doc)); + }; } -function allCollections(db, name, query, metadata, parser, next) { - return db.collections(function(err, collections) { - if (err) return next(err); - var last = ~~collections.length, index = 0; - if (0 === last) return next(err); - collections.forEach(function(collection) { - if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; - logger("select collection " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { - if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.find(query).snapshot(!0).stream().once("end", function() { - return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, name); +function allCollections(documentStore) { + return function(db, name, query, metadata, parser, next) { + return db.collections(function(err, collections) { + if (err) return next(err); + var last = ~~collections.length, index = 0; + if (0 === last) return next(err); + collections.forEach(function(collection) { + if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; + logger("select collection " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { + if (err) return last === ++index ? next(err) : error(err); + meta(collection, metadata, function() { + collection.find(query).snapshot(!0).stream().once("end", function() { + return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, collection.collectionName); + }); }); }); }); }); - }); + }; } -function allCollectionsScan(db, name, numCursors, metadata, parser, next) { - return db.collections(function(err, collections) { - if (err) return next(err); - var last = ~~collections.length, index = 0; - if (0 === last) return next(null); - collections.forEach(function(collection) { - if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; - logger("select collection scan " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { - if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function(err, cursors) { - if (err) return last === ++index ? next(err) : error(err); - var ii, cursorsDone; - if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; - for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { - if (0 == --cursorsDone) return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, name); +function allCollectionsScan(documentStore) { + return function(db, name, numCursors, metadata, parser, next) { + return db.collections(function(err, collections) { + if (err) return next(err); + var last = ~~collections.length, index = 0; + if (0 === last) return next(null); + collections.forEach(function(collection) { + if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; + logger("select collection scan " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { + if (err) return last === ++index ? next(err) : error(err); + meta(collection, metadata, function() { + collection.parallelCollectionScan({ + numCursors: numCursors + }, function(err, cursors) { + if (err) return last === ++index ? next(err) : error(err); + var ii, cursorsDone; + if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; + for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { + if (0 == --cursorsDone) return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, collection.collectionName); + }); }); }); }); }); }); - }); + }; } -function someCollections(db, name, query, metadata, parser, next, collections) { - var last = ~~collections.length, index = 0; - if (0 === last) return next(null); - collections.forEach(function(collection) { - db.collection(collection, { - strict: !0 - }, function(err, collection) { - if (err) return last === ++index ? next(err) : error(err); - logger("select collection " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { +function someCollections(documentStore) { + return function(db, name, query, metadata, parser, next, collections) { + var last = ~~collections.length, index = 0; + if (0 === last) return next(null); + collections.forEach(function(collection) { + db.collection(collection, { + strict: !0 + }, function(err, collection) { if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.find(query).snapshot(!0).stream().once("end", function() { - return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, name); + logger("select collection " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { + if (err) return last === ++index ? next(err) : error(err); + meta(collection, metadata, function() { + collection.find(query).snapshot(!0).stream().once("end", function() { + return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, collection.collectionName); + }); }); }); }); }); - }); + }; } -function someCollectionsScan(db, name, numCursors, metadata, parser, next, collections) { - var last = ~~collections.length, index = 0; - if (0 === last) return next(null); - collections.forEach(function(collection) { - db.collection(collection, { - strict: !0 - }, function(err, collection) { - if (err) return last === ++index ? next(err) : error(err); - logger("select collection scan " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { +function someCollectionsScan(documentStore) { + return function(db, name, numCursors, metadata, parser, next, collections) { + var last = ~~collections.length, index = 0; + if (0 === last) return next(null); + collections.forEach(function(collection) { + db.collection(collection, { + strict: !0 + }, function(err, collection) { if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function(err, cursors) { - if (err) return last === ++index ? next(err) : error(err); - var ii, cursorsDone; - if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; - for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { - if (0 == --cursorsDone) return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, name); + logger("select collection scan " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { + if (err) return last === ++index ? next(err) : error(err); + meta(collection, metadata, function() { + collection.parallelCollectionScan({ + numCursors: numCursors + }, function(err, cursors) { + if (err) return last === ++index ? next(err) : error(err); + var ii, cursorsDone; + if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; + for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { + if (0 == --cursorsDone) return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, collection.collectionName); + }); }); }); }); }); }); - }); + }; } function wrapper(my) { @@ -156,19 +140,19 @@ function wrapper(my) { var parser; if ("function" == typeof my.parser) parser = my.parser; else switch (my.parser.toLowerCase()) { case "bson": - BSON = require("bson"), BSON = new BSON(), parser = toBsonAsync; + BSON = require("bson"), BSON = new BSON(), parser = toBsonAsync(my.documentStore); break; case "json": - parser = toJsonAsync; + parser = toJsonAsync(my.documentStore); break; default: throw new Error("missing parser option"); } - var discriminator = allCollections; - if (null !== my.collections ? (discriminator = someCollections, my.numCursors && (discriminator = someCollectionsScan, - my.query = my.numCursors)) : my.numCursors && (discriminator = allCollectionsScan, + var discriminator = allCollections(my.documentStore); + if (null !== my.collections ? (discriminator = someCollections(my.documentStore), + my.numCursors && (discriminator = someCollectionsScan(my.documentStore), my.query = my.numCursors)) : my.numCursors && (discriminator = allCollectionsScan(my.documentStore), my.query = my.numCursors), null === my.logger) logger = function() {}; else { (logger = require("logger-request")({ filename: my.logger, @@ -186,35 +170,19 @@ function wrapper(my) { }); } var metadata = ""; - meta = !0 === my.metadata ? writeMetadata : function(a, b, c) { + meta = !0 === my.metadata ? writeMetadata(my.documentStore) : function(a, b, c) { return c(); }, require("mongodb").MongoClient.connect(my.uri, my.options, function(err, db) { if (logger("db open"), err) return callback(err); - var root = null === my.tar ? my.root : my.dir; - makeDir(root, function(err, name) { + my.documentStore.addDatabase(db.databaseName, function(err, name) { + function go() { + return discriminator(db, name, my.query, metadata, parser, function(err) { + if (logger("db close"), db.close(), err) return callback(err); + my.documentStore.close(), callback(null); + }, my.collections); + } if (err) return callback(err); - makeDir(name + db.databaseName + path.sep, function(err, name) { - function go() { - return discriminator(db, name, my.query, metadata, parser, function(err) { - if (logger("db close"), db.close(), err) return callback(err); - my.tar ? makeDir(my.root, function(e, name) { - err && error(err); - var dest; - my.stream ? (logger("send tar file to stream"), dest = my.stream) : (logger("make tar file at " + name + my.tar), - dest = fs.createWriteStream(name + my.tar)); - var packer = require("tar").Pack().on("error", callback).on("end", function() { - rmDir(root), callback(null); - }); - require("fstream").Reader({ - path: root + db.databaseName, - type: "Directory" - }).on("error", callback).pipe(packer).pipe(dest); - }) : callback(null); - }, my.collections); - } - if (err) return callback(err); - !1 === my.metadata ? go() : makeDir(metadata = name + ".metadata" + path.sep, go); - }); + !1 === my.metadata ? go() : my.documentStore.addCollection(".metadata", go); }); }); } @@ -227,7 +195,6 @@ function backup(options) { if (fs.existsSync(opt.root) && !fs.statSync(opt.root).isDirectory()) throw new Error("root option is not a directory"); } var my = { - dir: path.join(__dirname, "dump", path.sep), uri: String(opt.uri), root: path.resolve(String(opt.root || "")) + path.sep, stream: opt.stream || null, @@ -241,9 +208,64 @@ function backup(options) { options: "object" == typeof opt.options ? opt.options : {}, metadata: Boolean(opt.metadata) }; - return my.stream && (my.tar = !0), wrapper(my); + return my.tar && !my.stream && (my.stream = fs.createWriteStream(path.join(my.root, my.tar))), + my.stream ? (my.tar = !0, my.documentStore = streamingDocumentStore(my.root, my.stream)) : my.documentStore = fileSystemDocumentStore(my.root), + wrapper(my); } -var systemRegex = /^system\./, fs = require("graceful-fs"), path = require("path"), BSON, logger, meta; +var systemRegex = /^system\./, fs = require("graceful-fs").gracefulify(require("fs-extra")), path = require("path"), BSON, logger, meta, fileSystemDocumentStore = function(root) { + var dbDir = root, makeDir = function(pathname, next) { + fs.stat(pathname, function(err, stats) { + return err && "ENOENT" === err.code ? (logger("make dir at " + pathname), fs.mkdirp(pathname, function(err) { + next(err, pathname); + })) : stats && !1 === stats.isDirectory() ? (logger("unlink file at " + pathname), + fs.unlink(pathname, function(err) { + if (err) return next(err); + logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { + next(err, pathname); + }); + })) : void next(null, pathname); + }); + }; + return { + addDatabase: function(dbName, next) { + return dbDir = path.join(root, dbName), makeDir(dbDir, next); + }, + addCollection: function(relativePath, next) { + var pathname = path.join(dbDir, relativePath); + return makeDir(pathname, next); + }, + store: function(collectionName, relativePath, content, callback) { + fs.writeFile(path.join(dbDir, collectionName, relativePath), content, callback); + }, + close: function() {} + }; +}, streamingDocumentStore = function(root, stream) { + var pack = require("tar-stream").pack(); + pack.pipe(stream); + var dbDir = root; + return { + addDatabase: function(dbName, next) { + dbDir = path.join(root, dbName), pack.entry({ + name: dbDir, + type: "directory" + }), next(); + }, + addCollection: function(filename, next) { + "" !== filename && pack.entry({ + name: path.join(dbDir, filename), + type: "directory" + }), next(); + }, + store: function(collectionName, filename, content, callback) { + pack.entry({ + name: path.join(dbDir, collectionName, filename) + }, content), callback && callback(); + }, + close: function() { + pack.finalize(); + } + }; +}; module.exports = backup; diff --git a/package.json b/package.json index a0d71b4..82fdcff 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,12 @@ "main": "index.min.js", "dependencies": { "bson": "1.0.4", + "fs-extra": "^3.0.1", "fstream": "1.0.11", "graceful-fs": "4.1.11", "logger-request": "3.8.0", "mongodb": "2.2.26", - "tar": "2.2.1" + "tar-stream": "^1.5.4" }, "devDependencies": { "grunt": "~1.0", From 82b80c6e74e47a06c0710942b1970fb48113713f Mon Sep 17 00:00:00 2001 From: Stuart Miller Date: Fri, 9 Jun 2017 09:48:17 +1200 Subject: [PATCH 02/12] Revert index.min.js for code review purposes --- index.min.js | 308 ++++++++++++++++++++++++--------------------------- 1 file changed, 143 insertions(+), 165 deletions(-) diff --git a/index.min.js b/index.min.js index 4e0293e..6a35041 100644 --- a/index.min.js +++ b/index.min.js @@ -4,133 +4,149 @@ function error(err) { err && logger(err.message); } -function writeMetadata(documentStore) { - return function(collection, metadata, next) { - return collection.indexes(function(err, indexes) { +function writeMetadata(collection, metadata, next) { + return collection.indexes(function(err, indexes) { + if (err) return next(err); + fs.writeFile(metadata + collection.collectionName, JSON.stringify(indexes), next); + }); +} + +function makeDir(pathname, next) { + fs.stat(pathname, function(err, stats) { + return err && "ENOENT" === err.code ? (logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { + next(err, pathname); + })) : stats && !1 === stats.isDirectory() ? (logger("unlink file at " + pathname), + fs.unlink(pathname, function(err) { if (err) return next(err); - documentStore.store(".metadata", collection.collectionName, JSON.stringify(indexes), next); - }); - }; + logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { + next(err, pathname); + }); + })) : void next(null, pathname); + }); } -function toJsonAsync(documentStore) { - return function(doc, collectionPath) { - documentStore.store(collectionPath, doc._id + ".json", JSON.stringify(doc)); - }; +function rmDir(pathname, next) { + fs.readdirSync(pathname).forEach(function(first) { + var database = pathname + first; + if (!1 === fs.statSync(database).isDirectory()) return next(Error("path is not a Directory")); + var metadata = "", collections = fs.readdirSync(database), metadataPath = path.join(database, ".metadata"); + return !0 === fs.existsSync(metadataPath) && (metadata = metadataPath + path.sep, + delete collections[collections.indexOf(".metadata")]), collections.forEach(function(second) { + var collection = path.join(database, second); + !1 !== fs.statSync(collection).isDirectory() && (fs.readdirSync(collection).forEach(function(third) { + var document = path.join(collection, third); + return fs.unlinkSync(document), next ? next(null, document) : ""; + }), "" !== metadata && fs.unlinkSync(metadata + second), fs.rmdirSync(collection)); + }), "" !== metadata && fs.rmdirSync(metadata), fs.rmdirSync(database); + }); } -function toBsonAsync(documentStore) { - return function(doc, collectionPath) { - documentStore.store(collectionPath, doc._id + ".bson", BSON.serialize(doc)); - }; +function toJsonAsync(doc, collectionPath) { + fs.writeFile(collectionPath + doc._id + ".json", JSON.stringify(doc)); } -function allCollections(documentStore) { - return function(db, name, query, metadata, parser, next) { - return db.collections(function(err, collections) { - if (err) return next(err); - var last = ~~collections.length, index = 0; - if (0 === last) return next(err); - collections.forEach(function(collection) { - if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; - logger("select collection " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { - if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.find(query).snapshot(!0).stream().once("end", function() { - return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, collection.collectionName); - }); - }); - }); - }); - }); - }; +function toBsonAsync(doc, collectionPath) { + fs.writeFile(collectionPath + doc._id + ".bson", BSON.serialize(doc)); } -function allCollectionsScan(documentStore) { - return function(db, name, numCursors, metadata, parser, next) { - return db.collections(function(err, collections) { - if (err) return next(err); - var last = ~~collections.length, index = 0; - if (0 === last) return next(null); - collections.forEach(function(collection) { - if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; - logger("select collection scan " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { - if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function(err, cursors) { - if (err) return last === ++index ? next(err) : error(err); - var ii, cursorsDone; - if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; - for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { - if (0 == --cursorsDone) return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, collection.collectionName); - }); - }); +function allCollections(db, name, query, metadata, parser, next) { + return db.collections(function(err, collections) { + if (err) return next(err); + var last = ~~collections.length, index = 0; + if (0 === last) return next(err); + collections.forEach(function(collection) { + if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; + logger("select collection " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { + if (err) return last === ++index ? next(err) : error(err); + meta(collection, metadata, function() { + collection.find(query).snapshot(!0).stream().once("end", function() { + return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, name); }); }); }); }); - }; + }); } -function someCollections(documentStore) { - return function(db, name, query, metadata, parser, next, collections) { +function allCollectionsScan(db, name, numCursors, metadata, parser, next) { + return db.collections(function(err, collections) { + if (err) return next(err); var last = ~~collections.length, index = 0; if (0 === last) return next(null); collections.forEach(function(collection) { - db.collection(collection, { - strict: !0 - }, function(err, collection) { + if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; + logger("select collection scan " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { if (err) return last === ++index ? next(err) : error(err); - logger("select collection " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { - if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.find(query).snapshot(!0).stream().once("end", function() { - return last === ++index ? next(null) : null; + meta(collection, metadata, function() { + collection.parallelCollectionScan({ + numCursors: numCursors + }, function(err, cursors) { + if (err) return last === ++index ? next(err) : error(err); + var ii, cursorsDone; + if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; + for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { + if (0 == --cursorsDone) return last === ++index ? next(null) : null; }).on("data", function(doc) { - parser(doc, collection.collectionName); + parser(doc, name); }); }); }); }); }); - }; + }); } -function someCollectionsScan(documentStore) { - return function(db, name, numCursors, metadata, parser, next, collections) { - var last = ~~collections.length, index = 0; - if (0 === last) return next(null); - collections.forEach(function(collection) { - db.collection(collection, { - strict: !0 - }, function(err, collection) { +function someCollections(db, name, query, metadata, parser, next, collections) { + var last = ~~collections.length, index = 0; + if (0 === last) return next(null); + collections.forEach(function(collection) { + db.collection(collection, { + strict: !0 + }, function(err, collection) { + if (err) return last === ++index ? next(err) : error(err); + logger("select collection " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { if (err) return last === ++index ? next(err) : error(err); - logger("select collection scan " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { - if (err) return last === ++index ? next(err) : error(err); - meta(collection, metadata, function() { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function(err, cursors) { - if (err) return last === ++index ? next(err) : error(err); - var ii, cursorsDone; - if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; - for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { - if (0 == --cursorsDone) return last === ++index ? next(null) : null; - }).on("data", function(doc) { - parser(doc, collection.collectionName); - }); + meta(collection, metadata, function() { + collection.find(query).snapshot(!0).stream().once("end", function() { + return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, name); + }); + }); + }); + }); + }); +} + +function someCollectionsScan(db, name, numCursors, metadata, parser, next, collections) { + var last = ~~collections.length, index = 0; + if (0 === last) return next(null); + collections.forEach(function(collection) { + db.collection(collection, { + strict: !0 + }, function(err, collection) { + if (err) return last === ++index ? next(err) : error(err); + logger("select collection scan " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { + if (err) return last === ++index ? next(err) : error(err); + meta(collection, metadata, function() { + collection.parallelCollectionScan({ + numCursors: numCursors + }, function(err, cursors) { + if (err) return last === ++index ? next(err) : error(err); + var ii, cursorsDone; + if (0 === (ii = cursorsDone = ~~cursors.length)) return last === ++index ? next(null) : null; + for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { + if (0 == --cursorsDone) return last === ++index ? next(null) : null; + }).on("data", function(doc) { + parser(doc, name); }); }); }); }); }); - }; + }); } function wrapper(my) { @@ -140,19 +156,19 @@ function wrapper(my) { var parser; if ("function" == typeof my.parser) parser = my.parser; else switch (my.parser.toLowerCase()) { case "bson": - BSON = require("bson"), BSON = new BSON(), parser = toBsonAsync(my.documentStore); + BSON = require("bson"), BSON = new BSON(), parser = toBsonAsync; break; case "json": - parser = toJsonAsync(my.documentStore); + parser = toJsonAsync; break; default: throw new Error("missing parser option"); } - var discriminator = allCollections(my.documentStore); - if (null !== my.collections ? (discriminator = someCollections(my.documentStore), - my.numCursors && (discriminator = someCollectionsScan(my.documentStore), my.query = my.numCursors)) : my.numCursors && (discriminator = allCollectionsScan(my.documentStore), + var discriminator = allCollections; + if (null !== my.collections ? (discriminator = someCollections, my.numCursors && (discriminator = someCollectionsScan, + my.query = my.numCursors)) : my.numCursors && (discriminator = allCollectionsScan, my.query = my.numCursors), null === my.logger) logger = function() {}; else { (logger = require("logger-request")({ filename: my.logger, @@ -170,19 +186,35 @@ function wrapper(my) { }); } var metadata = ""; - meta = !0 === my.metadata ? writeMetadata(my.documentStore) : function(a, b, c) { + meta = !0 === my.metadata ? writeMetadata : function(a, b, c) { return c(); }, require("mongodb").MongoClient.connect(my.uri, my.options, function(err, db) { if (logger("db open"), err) return callback(err); - my.documentStore.addDatabase(db.databaseName, function(err, name) { - function go() { - return discriminator(db, name, my.query, metadata, parser, function(err) { - if (logger("db close"), db.close(), err) return callback(err); - my.documentStore.close(), callback(null); - }, my.collections); - } + var root = null === my.tar ? my.root : my.dir; + makeDir(root, function(err, name) { if (err) return callback(err); - !1 === my.metadata ? go() : my.documentStore.addCollection(".metadata", go); + makeDir(name + db.databaseName + path.sep, function(err, name) { + function go() { + return discriminator(db, name, my.query, metadata, parser, function(err) { + if (logger("db close"), db.close(), err) return callback(err); + my.tar ? makeDir(my.root, function(e, name) { + err && error(err); + var dest; + my.stream ? (logger("send tar file to stream"), dest = my.stream) : (logger("make tar file at " + name + my.tar), + dest = fs.createWriteStream(name + my.tar)); + var packer = require("tar").Pack().on("error", callback).on("end", function() { + rmDir(root), callback(null); + }); + require("fstream").Reader({ + path: root + db.databaseName, + type: "Directory" + }).on("error", callback).pipe(packer).pipe(dest); + }) : callback(null); + }, my.collections); + } + if (err) return callback(err); + !1 === my.metadata ? go() : makeDir(metadata = name + ".metadata" + path.sep, go); + }); }); }); } @@ -195,6 +227,7 @@ function backup(options) { if (fs.existsSync(opt.root) && !fs.statSync(opt.root).isDirectory()) throw new Error("root option is not a directory"); } var my = { + dir: path.join(__dirname, "dump", path.sep), uri: String(opt.uri), root: path.resolve(String(opt.root || "")) + path.sep, stream: opt.stream || null, @@ -208,64 +241,9 @@ function backup(options) { options: "object" == typeof opt.options ? opt.options : {}, metadata: Boolean(opt.metadata) }; - return my.tar && !my.stream && (my.stream = fs.createWriteStream(path.join(my.root, my.tar))), - my.stream ? (my.tar = !0, my.documentStore = streamingDocumentStore(my.root, my.stream)) : my.documentStore = fileSystemDocumentStore(my.root), - wrapper(my); + return my.stream && (my.tar = !0), wrapper(my); } -var systemRegex = /^system\./, fs = require("graceful-fs").gracefulify(require("fs-extra")), path = require("path"), BSON, logger, meta, fileSystemDocumentStore = function(root) { - var dbDir = root, makeDir = function(pathname, next) { - fs.stat(pathname, function(err, stats) { - return err && "ENOENT" === err.code ? (logger("make dir at " + pathname), fs.mkdirp(pathname, function(err) { - next(err, pathname); - })) : stats && !1 === stats.isDirectory() ? (logger("unlink file at " + pathname), - fs.unlink(pathname, function(err) { - if (err) return next(err); - logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { - next(err, pathname); - }); - })) : void next(null, pathname); - }); - }; - return { - addDatabase: function(dbName, next) { - return dbDir = path.join(root, dbName), makeDir(dbDir, next); - }, - addCollection: function(relativePath, next) { - var pathname = path.join(dbDir, relativePath); - return makeDir(pathname, next); - }, - store: function(collectionName, relativePath, content, callback) { - fs.writeFile(path.join(dbDir, collectionName, relativePath), content, callback); - }, - close: function() {} - }; -}, streamingDocumentStore = function(root, stream) { - var pack = require("tar-stream").pack(); - pack.pipe(stream); - var dbDir = root; - return { - addDatabase: function(dbName, next) { - dbDir = path.join(root, dbName), pack.entry({ - name: dbDir, - type: "directory" - }), next(); - }, - addCollection: function(filename, next) { - "" !== filename && pack.entry({ - name: path.join(dbDir, filename), - type: "directory" - }), next(); - }, - store: function(collectionName, filename, content, callback) { - pack.entry({ - name: path.join(dbDir, collectionName, filename) - }, content), callback && callback(); - }, - close: function() { - pack.finalize(); - } - }; -}; +var systemRegex = /^system\./, fs = require("graceful-fs"), path = require("path"), BSON, logger, meta; module.exports = backup; From 36b035eb27ae26fe5de2dc0ad20fd22f7e07bdf0 Mon Sep 17 00:00:00 2001 From: Stuart Miller Date: Wed, 14 Jun 2017 10:04:25 +1200 Subject: [PATCH 03/12] Store documentStore as a global variable to make the diff easier to read. --- index.js | 360 +++++++++++++++++++++++++++---------------------------- 1 file changed, 174 insertions(+), 186 deletions(-) diff --git a/index.js b/index.js index 4b30672..ff6eee5 100644 --- a/index.js +++ b/index.js @@ -19,28 +19,30 @@ var BSON; var logger; var meta; -var fileSystemDocumentStore = function (root) { +var documentStore = undefined; + +var fileSystemDocumentStore = function(root) { var dbDir = root; - var makeDir = function (pathname, next) { - fs.stat(pathname, function (err, stats) { + var makeDir = function(pathname, next) { + fs.stat(pathname, function(err, stats) { if (err && err.code === 'ENOENT') { // no file or dir logger('make dir at ' + pathname); - return fs.mkdirp(pathname, function (err) { + return fs.mkdirp(pathname, function(err) { next(err, pathname); }); } else if (stats && stats.isDirectory() === false) { // pathname is a file logger('unlink file at ' + pathname); - return fs.unlink(pathname, function (err) { + return fs.unlink(pathname, function(err) { if (err) { // unlink fail. permission maybe return next(err); } logger('make dir at ' + pathname); - fs.mkdir(pathname, function (err) { + fs.mkdir(pathname, function(err) { next(err, pathname); }); @@ -52,7 +54,7 @@ var fileSystemDocumentStore = function (root) { }); }; return { - addDatabase: function (dbName, next) { + addDatabase: function(dbName, next) { dbDir = path.join(root, dbName); return makeDir(dbDir, next); }, @@ -63,12 +65,12 @@ var fileSystemDocumentStore = function (root) { store: function store(collectionName, relativePath, content, callback) { fs.writeFile(path.join(dbDir, collectionName, relativePath), content, callback); }, - close: function () { + close: function() { } }; }; -var streamingDocumentStore = function (root, stream) { +var streamingDocumentStore = function(root, stream) { var tar = require('tar-stream'); var pack = tar.pack(); // pack is a streams2 stream pack.pipe(stream); @@ -124,15 +126,15 @@ function error(err) { * @param {String} metadata - path of metadata * @param {Function} next - callback */ -function writeMetadata(documentStore) { - return function (collection, metadata, next) { - return collection.indexes(function (err, indexes) { - if (err) { - return next(err); - } - documentStore.store('.metadata', collection.collectionName, JSON.stringify(indexes), next); - }); - }; +function writeMetadata(collection, metadata, next) { + + return collection.indexes(function(err, indexes) { + + if (err) { + return next(err); + } + documentStore.store('.metadata', collection.collectionName, JSON.stringify(indexes), next); + }); } @@ -143,10 +145,9 @@ function writeMetadata(documentStore) { * @param {Objecy} doc - document from stream * @param {String} collectionPath - path of collection */ -function toJsonAsync(documentStore) { - return function (doc, collectionPath) { - documentStore.store(collectionPath, doc._id + '.json', JSON.stringify(doc)); - }; +function toJsonAsync(doc, collectionPath) { + + documentStore.store(collectionPath, doc._id + '.json', JSON.stringify(doc)); } /** @@ -156,10 +157,9 @@ function toJsonAsync(documentStore) { * @param {Objecy} doc - document from stream * @param {String} collectionPath - path of collection */ -function toBsonAsync(documentStore) { - return function (doc, collectionPath) { - documentStore.store(collectionPath, doc._id + '.bson', BSON.serialize(doc)); - }; +function toBsonAsync(doc, collectionPath) { + + documentStore.store(collectionPath, doc._id + '.bson', BSON.serialize(doc)); } /** @@ -173,49 +173,47 @@ function toBsonAsync(documentStore) { * @param {Function} parser - data parser * @param {Function} next - callback */ -function allCollections(documentStore) { - return function (db, name, query, metadata, parser, next) { +function allCollections(db, name, query, metadata, parser, next) { - return db.collections(function (err, collections) { + return db.collections(function(err, collections) { - if (err) { - return next(err); - } + if (err) { + return next(err); + } - var last = ~~collections.length, index = 0; - if (last === 0) { // empty set - return next(err); - } + var last = ~~collections.length, index = 0; + if (last === 0) { // empty set + return next(err); + } - collections.forEach(function (collection) { + collections.forEach(function(collection) { - if (systemRegex.test(collection.collectionName) === true) { - return last === ++index ? next(null) : null; - } + if (systemRegex.test(collection.collectionName) === true) { + return last === ++index ? next(null) : null; + } - logger('select collection ' + collection.collectionName); - documentStore.addCollection(collection.collectionName, function (err) { + logger('select collection ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function(err) { - if (err) { - return last === ++index ? next(err) : error(err); - } + if (err) { + return last === ++index ? next(err) : error(err); + } - meta(collection, metadata, function () { + meta(collection, metadata, function() { - var stream = collection.find(query).snapshot(true).stream(); + var stream = collection.find(query).snapshot(true).stream(); - stream.once('end', function () { + stream.once('end', function() { - return last === ++index ? next(null) : null; - }).on('data', function (doc) { + return last === ++index ? next(null) : null; + }).on('data', function(doc) { - parser(doc, collection.collectionName); - }); + parser(doc, collection.collectionName); }); }); }); }); - }; + }); } /** @@ -229,67 +227,65 @@ function allCollections(documentStore) { * @param {Function} parser - data parser * @param {Function} next - callback */ -function allCollectionsScan(documentStore) { - return function (db, name, numCursors, metadata, parser, next) { +function allCollectionsScan(db, name, numCursors, metadata, parser, next) { - return db.collections(function (err, collections) { + return db.collections(function(err, collections) { - if (err) { - return next(err); - } + if (err) { + return next(err); + } - var last = ~~collections.length, index = 0; - if (last === 0) { // empty set - return next(null); - } + var last = ~~collections.length, index = 0; + if (last === 0) { // empty set + return next(null); + } - collections.forEach(function (collection) { + collections.forEach(function(collection) { - if (systemRegex.test(collection.collectionName) === true) { - return last === ++index ? next(null) : null; - } + if (systemRegex.test(collection.collectionName) === true) { + return last === ++index ? next(null) : null; + } - logger('select collection scan ' + collection.collectionName); - documentStore.addCollection(collection.collectionName, function (err) { + logger('select collection scan ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function(err) { - if (err) { - return last === ++index ? next(err) : error(err); - } + if (err) { + return last === ++index ? next(err) : error(err); + } - meta(collection, metadata, function () { + meta(collection, metadata, function() { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function (err, cursors) { + collection.parallelCollectionScan({ + numCursors: numCursors + }, function(err, cursors) { - if (err) { - return last === ++index ? next(err) : error(err); - } + if (err) { + return last === ++index ? next(err) : error(err); + } - var ii, cursorsDone; - ii = cursorsDone = ~~cursors.length; - if (ii === 0) { // empty set - return last === ++index ? next(null) : null; - } + var ii, cursorsDone; + ii = cursorsDone = ~~cursors.length; + if (ii === 0) { // empty set + return last === ++index ? next(null) : null; + } - for (var i = 0; i < ii; ++i) { - cursors[i].once('end', function () { + for (var i = 0; i < ii; ++i) { + cursors[i].once('end', function() { - // No more cursors let's ensure we got all results - if (--cursorsDone === 0) { - return last === ++index ? next(null) : null; - } - }).on('data', function (doc) { + // No more cursors let's ensure we got all results + if (--cursorsDone === 0) { + return last === ++index ? next(null) : null; + } + }).on('data', function(doc) { - parser(doc, collection.collectionName); - }); - } - }); + parser(doc, collection.collectionName); + }); + } }); }); }); }); - }; + }); } /** @@ -304,47 +300,45 @@ function allCollectionsScan(documentStore) { * @param {Function} next - callback * @param {Array} collections - selected collections */ -function someCollections(documentStore) { - return function (db, name, query, metadata, parser, next, collections) { +function someCollections(db, name, query, metadata, parser, next, collections) { - var last = ~~collections.length, index = 0; - if (last === 0) { - return next(null); - } + var last = ~~collections.length, index = 0; + if (last === 0) { + return next(null); + } - collections.forEach(function (collection) { + collections.forEach(function(collection) { - db.collection(collection, { - strict: true - }, function (err, collection) { + db.collection(collection, { + strict: true + }, function(err, collection) { - if (err) { // returns an error if the collection does not exist - return last === ++index ? next(err) : error(err); - } + if (err) { // returns an error if the collection does not exist + return last === ++index ? next(err) : error(err); + } - logger('select collection ' + collection.collectionName); - documentStore.addCollection(collection.collectionName, function (err) { + logger('select collection ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function(err) { - if (err) { - return last === ++index ? next(err) : error(err); - } + if (err) { + return last === ++index ? next(err) : error(err); + } - meta(collection, metadata, function () { + meta(collection, metadata, function() { - var stream = collection.find(query).snapshot(true).stream(); + var stream = collection.find(query).snapshot(true).stream(); - stream.once('end', function () { + stream.once('end', function() { - return last === ++index ? next(null) : null; - }).on('data', function (doc) { + return last === ++index ? next(null) : null; + }).on('data', function(doc) { - parser(doc, collection.collectionName); - }); + parser(doc, collection.collectionName); }); }); }); }); - }; + }); } /** @@ -359,66 +353,64 @@ function someCollections(documentStore) { * @param {Function} next - callback * @param {Array} collections - selected collections */ -function someCollectionsScan(documentStore) { - return function (db, name, numCursors, metadata, parser, next, - collections) { +function someCollectionsScan(db, name, numCursors, metadata, parser, next, + collections) { - var last = ~~collections.length, index = 0; - if (last === 0) { // empty set - return next(null); - } + var last = ~~collections.length, index = 0; + if (last === 0) { // empty set + return next(null); + } - collections.forEach(function (collection) { + collections.forEach(function(collection) { - db.collection(collection, { - strict: true - }, function (err, collection) { + db.collection(collection, { + strict: true + }, function(err, collection) { - if (err) { // returns an error if the collection does not exist - return last === ++index ? next(err) : error(err); - } + if (err) { // returns an error if the collection does not exist + return last === ++index ? next(err) : error(err); + } - logger('select collection scan ' + collection.collectionName); - documentStore.addCollection(collection.collectionName, function (err) { + logger('select collection scan ' + collection.collectionName); + documentStore.addCollection(collection.collectionName, function(err) { - if (err) { - return last === ++index ? next(err) : error(err); - } + if (err) { + return last === ++index ? next(err) : error(err); + } - meta(collection, metadata, function () { + meta(collection, metadata, function() { - collection.parallelCollectionScan({ - numCursors: numCursors - }, function (err, cursors) { + collection.parallelCollectionScan({ + numCursors: numCursors + }, function(err, cursors) { - if (err) { - return last === ++index ? next(err) : error(err); - } + if (err) { + return last === ++index ? next(err) : error(err); + } - var ii, cursorsDone; - ii = cursorsDone = ~~cursors.length; - if (ii === 0) { // empty set - return last === ++index ? next(null) : null; - } + var ii, cursorsDone; + ii = cursorsDone = ~~cursors.length; + if (ii === 0) { // empty set + return last === ++index ? next(null) : null; + } - for (var i = 0; i < ii; ++i) { - cursors[i].once('end', function () { + for (var i = 0; i < ii; ++i) { + cursors[i].once('end', function() { - // No more cursors let's ensure we got all results - if (--cursorsDone === 0) { - return last === ++index ? next(null) : null; - } - }).on('data', function (doc) { + // No more cursors let's ensure we got all results + if (--cursorsDone === 0) { + return last === ++index ? next(null) : null; + } + }).on('data', function(doc) { - parser(doc, collection.collectionName); - }); - } - }); + parser(doc, collection.collectionName); + }); + } }); }); }); }); - }; + }); } /** @@ -437,31 +429,31 @@ function wrapper(my) { case 'bson': BSON = require('bson'); BSON = new BSON(); - parser = toBsonAsync(my.documentStore); + parser = toBsonAsync; break; case 'json': // JSON error on ObjectId, Date and Long - parser = toJsonAsync(my.documentStore); + parser = toJsonAsync; break; default: throw new Error('missing parser option'); } } - var discriminator = allCollections(my.documentStore); + var discriminator = allCollections; if (my.collections !== null) { - discriminator = someCollections(my.documentStore); + discriminator = someCollections; if (my.numCursors) { - discriminator = someCollectionsScan(my.documentStore); + discriminator = someCollectionsScan; my.query = my.numCursors; // override } } else if (my.numCursors) { - discriminator = allCollectionsScan(my.documentStore); + discriminator = allCollectionsScan; my.query = my.numCursors; // override } if (my.logger === null) { - logger = function () { + logger = function() { return; }; @@ -479,7 +471,7 @@ function wrapper(my) { logger('backup start'); var log = require('mongodb').Logger; log.setLevel('info'); - log.setCurrentLogger(function (msg) { + log.setCurrentLogger(function(msg) { return logger(msg); }); @@ -487,9 +479,9 @@ function wrapper(my) { var metadata = ''; if (my.metadata === true) { - meta = writeMetadata(my.documentStore); + meta = writeMetadata; } else { - meta = function (a, b, c) { + meta = function(a, b, c) { return c(); }; @@ -512,20 +504,20 @@ function wrapper(my) { } } - require('mongodb').MongoClient.connect(my.uri, my.options, function (err, db) { + require('mongodb').MongoClient.connect(my.uri, my.options, function(err, db) { logger('db open'); if (err) { return callback(err); } - my.documentStore.addDatabase(db.databaseName, function (err, name) { + documentStore.addDatabase(db.databaseName, function(err, name) { function go() { // waiting for `db.fsyncLock()` on node driver return discriminator(db, name, my.query, metadata, parser, - function (err) { + function(err) { logger('db close'); db.close(); @@ -533,7 +525,7 @@ function wrapper(my) { return callback(err); } - my.documentStore.close(); + documentStore.close(); callback(null); }, my.collections); } @@ -545,7 +537,7 @@ function wrapper(my) { if (my.metadata === false) { go(); } else { - my.documentStore.addCollection('.metadata', go); + documentStore.addCollection('.metadata', go); } }); }); @@ -591,14 +583,10 @@ function backup(options) { } if (my.stream) { my.tar = true; // override - my.documentStore = streamingDocumentStore(my.root, my.stream); + documentStore = streamingDocumentStore(my.root, my.stream); } else { - my.documentStore = fileSystemDocumentStore(my.root); + documentStore = fileSystemDocumentStore(my.root); } return wrapper(my); } module.exports = backup; - - - - From 4427cdeb5e9f7b7e5f939a603e07c8223db39844 Mon Sep 17 00:00:00 2001 From: Cristian Cortez Date: Tue, 5 Feb 2019 19:13:02 +0000 Subject: [PATCH 04/12] Patch lib to work with MongoDB 4.x --- index.js | 25 +++++++++++++++++-------- package.json | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index ff6eee5..f7178b9 100644 --- a/index.js +++ b/index.js @@ -201,7 +201,11 @@ function allCollections(db, name, query, metadata, parser, next) { meta(collection, metadata, function() { - var stream = collection.find(query).snapshot(true).stream(); + var stream = collection.find(query) + // NOTE: snapshot was deprecated + // See: http://mongodb.github.io/node-mongodb-native/3.1/api/Cursor.html#snapshot + // .snapshot(true) + .stream(); stream.once('end', function() { @@ -326,7 +330,11 @@ function someCollections(db, name, query, metadata, parser, next, collections) { meta(collection, metadata, function() { - var stream = collection.find(query).snapshot(true).stream(); + var stream = collection.find(query) + // NOTE: snapshot was deprecated + // See: http://mongodb.github.io/node-mongodb-native/3.1/api/Cursor.html#snapshot + // .snapshot(true) + .stream(); stream.once('end', function() { @@ -504,14 +512,13 @@ function wrapper(my) { } } - require('mongodb').MongoClient.connect(my.uri, my.options, function(err, db) { + require('mongodb').MongoClient.connect(my.uri, my.options).then(function(database) { + const databaseName = database.s.options.dbName; + let db = database.db(databaseName); logger('db open'); - if (err) { - return callback(err); - } - documentStore.addDatabase(db.databaseName, function(err, name) { + documentStore.addDatabase(databaseName, function(err, name) { function go() { @@ -520,7 +527,7 @@ function wrapper(my) { function(err) { logger('db close'); - db.close(); + database.close(); if (err) { return callback(err); } @@ -540,6 +547,8 @@ function wrapper(my) { documentStore.addCollection('.metadata', go); } }); + }).catch((err) => { + return callback(err); }); } diff --git a/package.json b/package.json index 82fdcff..1cd1208 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "fstream": "1.0.11", "graceful-fs": "4.1.11", "logger-request": "3.8.0", - "mongodb": "2.2.26", + "mongodb": "3.1.13", "tar-stream": "^1.5.4" }, "devDependencies": { From bda120e13f969c7f576110b8fd72551787cd2e73 Mon Sep 17 00:00:00 2001 From: Cristian Cortez Date: Tue, 5 Feb 2019 19:24:54 +0000 Subject: [PATCH 05/12] Lint fixes --- index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index f7178b9..d8775f3 100644 --- a/index.js +++ b/index.js @@ -19,7 +19,7 @@ var BSON; var logger; var meta; -var documentStore = undefined; +var documentStore; var fileSystemDocumentStore = function(root) { var dbDir = root; @@ -513,8 +513,8 @@ function wrapper(my) { } require('mongodb').MongoClient.connect(my.uri, my.options).then(function(database) { - const databaseName = database.s.options.dbName; - let db = database.db(databaseName); + var databaseName = database.s.options.dbName; + var db = database.db(databaseName); logger('db open'); @@ -547,7 +547,7 @@ function wrapper(my) { documentStore.addCollection('.metadata', go); } }); - }).catch((err) => { + }).catch(function (err) { return callback(err); }); } From dbe4ad9144e4362189035ba3903c44954dd915fb Mon Sep 17 00:00:00 2001 From: Cristian Cortez Date: Tue, 5 Feb 2019 19:59:23 +0000 Subject: [PATCH 06/12] Fixing internal tar directory --- index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index d8775f3..9485d0d 100644 --- a/index.js +++ b/index.js @@ -573,9 +573,14 @@ function backup(options) { } } + var rootPath = opt.root || ''; + if (rootPath !== '') { + rootPath = path.resolve(String(opt.root || '')); + } + var my = { uri: String(opt.uri), - root: path.resolve(String(opt.root || '')) + path.sep, + root: rootPath + path.sep, stream: opt.stream || null, parser: opt.parser || 'bson', numCursors: ~~opt.numCursors, From 87aacda77e5e40b0b96957bd15027b870c1ee28e Mon Sep 17 00:00:00 2001 From: Phoscur Date: Sun, 12 May 2019 16:35:18 +0200 Subject: [PATCH 07/12] Fix #27 Unrecognized field 'snapshot' --- index.js | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/index.js b/index.js index 9485d0d..f631ab3 100644 --- a/index.js +++ b/index.js @@ -201,11 +201,7 @@ function allCollections(db, name, query, metadata, parser, next) { meta(collection, metadata, function() { - var stream = collection.find(query) - // NOTE: snapshot was deprecated - // See: http://mongodb.github.io/node-mongodb-native/3.1/api/Cursor.html#snapshot - // .snapshot(true) - .stream(); + var stream = collection.find({ $query: query, $snapshot: true }).stream(); stream.once('end', function() { @@ -330,11 +326,7 @@ function someCollections(db, name, query, metadata, parser, next, collections) { meta(collection, metadata, function() { - var stream = collection.find(query) - // NOTE: snapshot was deprecated - // See: http://mongodb.github.io/node-mongodb-native/3.1/api/Cursor.html#snapshot - // .snapshot(true) - .stream(); + var stream = collection.find({ $query: query, $snapshot: true }).stream(); stream.once('end', function() { From 62d6cdd3d69fd81bce99d8f60258d3b59077d444 Mon Sep 17 00:00:00 2001 From: Phoscur Date: Sun, 12 May 2019 18:29:45 +0200 Subject: [PATCH 08/12] Fix npm audit --- package-lock.json | 405 ++++++++++++++++++---------------------------- package.json | 4 +- 2 files changed, 156 insertions(+), 253 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0a4e3d6..cccd341 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,23 +10,12 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, - "align-text": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", - "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", - "dev": true, - "optional": true, - "requires": { - "kind-of": "^3.0.2", - "longest": "^1.0.1", - "repeat-string": "^1.5.2" - } - }, "amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", - "dev": true + "dev": true, + "optional": true }, "ansi-regex": { "version": "2.1.1", @@ -41,12 +30,20 @@ "dev": true }, "argparse": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz", - "integrity": "sha1-c9g7wmP4bpf4zE9rrhsOkKfSLIY=", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "requires": { "sprintf-js": "~1.0.2" + }, + "dependencies": { + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + } } }, "array-find-index": { @@ -202,17 +199,6 @@ "map-obj": "^1.0.0" } }, - "center-align": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", - "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", - "dev": true, - "optional": true, - "requires": { - "align-text": "^0.1.3", - "lazy-cache": "^1.0.3" - } - }, "chalk": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", @@ -236,31 +222,25 @@ "glob": "^7.1.1" } }, - "cliui": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", - "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", + "coffeescript": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-1.10.0.tgz", + "integrity": "sha1-56qDAZF+9iGzXYo580jc3R234z4=", + "dev": true + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", "dev": true, - "optional": true, "requires": { - "center-align": "^0.1.1", - "right-align": "^0.1.1", - "wordwrap": "0.0.2" - }, - "dependencies": { - "wordwrap": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", - "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=", - "dev": true, - "optional": true - } + "color-name": "1.1.3" } }, - "coffee-script": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.10.0.tgz", - "integrity": "sha1-EpOLz5vhlI+gBvkuDEyegXBRCMA=", + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "dev": true }, "colors": { @@ -678,12 +658,12 @@ "dev": true }, "grunt": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/grunt/-/grunt-1.0.1.tgz", - "integrity": "sha1-6HeHZOlEsY8yuw8QuQeEdcnftWs=", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grunt/-/grunt-1.0.4.tgz", + "integrity": "sha512-PYsMOrOC+MsdGEkFVwMaMyc6Ob7pKmq+deg1Sjr+vvMWp35sztfwKE7qoN51V+UEtHsyNuMcGdgMLFkBHvMxHQ==", "dev": true, "requires": { - "coffee-script": "~1.10.0", + "coffeescript": "~1.10.0", "dateformat": "~1.0.12", "eventemitter2": "~0.4.13", "exit": "~0.1.1", @@ -691,14 +671,15 @@ "glob": "~7.0.0", "grunt-cli": "~1.2.0", "grunt-known-options": "~1.1.0", - "grunt-legacy-log": "~1.0.0", - "grunt-legacy-util": "~1.0.0", + "grunt-legacy-log": "~2.0.0", + "grunt-legacy-util": "~1.1.1", "iconv-lite": "~0.4.13", - "js-yaml": "~3.5.2", - "minimatch": "~3.0.0", + "js-yaml": "~3.13.0", + "minimatch": "~3.0.2", + "mkdirp": "~0.5.1", "nopt": "~3.0.6", "path-is-absolute": "~1.0.0", - "rimraf": "~2.2.8" + "rimraf": "~2.6.2" }, "dependencies": { "glob": { @@ -726,12 +707,6 @@ "nopt": "~3.0.6", "resolve": "~1.1.0" } - }, - "rimraf": { - "version": "2.2.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz", - "integrity": "sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI=", - "dev": true } } }, @@ -765,22 +740,21 @@ "dev": true }, "grunt-known-options": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/grunt-known-options/-/grunt-known-options-1.1.0.tgz", - "integrity": "sha1-pCdO6zL6dl2lp6OxcSYXzjsUQUk=", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/grunt-known-options/-/grunt-known-options-1.1.1.tgz", + "integrity": "sha512-cHwsLqoighpu7TuYj5RonnEuxGVFnztcUqTqp5rXFGYL4OuPFofwC4Ycg7n9fYwvK6F5WbYgeVOwph9Crs2fsQ==", "dev": true }, "grunt-legacy-log": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/grunt-legacy-log/-/grunt-legacy-log-1.0.0.tgz", - "integrity": "sha1-+4bxgJhHvAfcR4Q/ns1srLYt8tU=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/grunt-legacy-log/-/grunt-legacy-log-2.0.0.tgz", + "integrity": "sha512-1m3+5QvDYfR1ltr8hjiaiNjddxGdQWcH0rw1iKKiQnF0+xtgTazirSTGu68RchPyh1OBng1bBUjLmX8q9NpoCw==", "dev": true, "requires": { "colors": "~1.1.2", - "grunt-legacy-log-utils": "~1.0.0", + "grunt-legacy-log-utils": "~2.0.0", "hooker": "~0.2.3", - "lodash": "~3.10.1", - "underscore.string": "~3.2.3" + "lodash": "~4.17.5" }, "dependencies": { "colors": { @@ -792,36 +766,65 @@ } }, "grunt-legacy-log-utils": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/grunt-legacy-log-utils/-/grunt-legacy-log-utils-1.0.0.tgz", - "integrity": "sha1-p7ji0Ps1taUPSvmG/BEnSevJbz0=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/grunt-legacy-log-utils/-/grunt-legacy-log-utils-2.0.1.tgz", + "integrity": "sha512-o7uHyO/J+i2tXG8r2bZNlVk20vlIFJ9IEYyHMCQGfWYru8Jv3wTqKZzvV30YW9rWEjq0eP3cflQ1qWojIe9VFA==", "dev": true, "requires": { - "chalk": "~1.1.1", - "lodash": "~4.3.0" + "chalk": "~2.4.1", + "lodash": "~4.17.10" }, "dependencies": { - "lodash": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.3.0.tgz", - "integrity": "sha1-79nEpuxT87BUEkKZFcPkgk5NJaQ=", + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } } } }, "grunt-legacy-util": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/grunt-legacy-util/-/grunt-legacy-util-1.0.0.tgz", - "integrity": "sha1-OGqnjcbtUJhsKxiVcmWxtIq7m4Y=", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/grunt-legacy-util/-/grunt-legacy-util-1.1.1.tgz", + "integrity": "sha512-9zyA29w/fBe6BIfjGENndwoe1Uy31BIXxTH3s8mga0Z5Bz2Sp4UCjkeyv2tI449ymkx3x26B+46FV4fXEddl5A==", "dev": true, "requires": { "async": "~1.5.2", "exit": "~0.1.1", "getobject": "~0.1.0", "hooker": "~0.2.3", - "lodash": "~4.3.0", - "underscore.string": "~3.2.3", - "which": "~1.2.1" + "lodash": "~4.17.10", + "underscore.string": "~3.3.4", + "which": "~1.3.0" }, "dependencies": { "async": { @@ -830,11 +833,14 @@ "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true }, - "lodash": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.3.0.tgz", - "integrity": "sha1-79nEpuxT87BUEkKZFcPkgk5NJaQ=", - "dev": true + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } } } }, @@ -858,53 +864,15 @@ } }, "handlebars": { - "version": "4.0.11", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.11.tgz", - "integrity": "sha1-Ywo13+ApS8KB7a5v/F0yn8eYLcw=", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", + "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", "dev": true, "requires": { - "async": "^1.4.0", + "neo-async": "^2.6.0", "optimist": "^0.6.1", - "source-map": "^0.4.4", - "uglify-js": "^2.6" - }, - "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", - "dev": true - }, - "source-map": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", - "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", - "dev": true, - "requires": { - "amdefine": ">=0.0.4" - } - }, - "uglify-js": { - "version": "2.8.29", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", - "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=", - "dev": true, - "optional": true, - "requires": { - "source-map": "~0.5.1", - "uglify-to-browserify": "~1.0.0", - "yargs": "~3.10.0" - }, - "dependencies": { - "source-map": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", - "dev": true, - "optional": true - } - } - } + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" } }, "has-ansi": { @@ -980,10 +948,13 @@ } }, "iconv-lite": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", - "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==", - "dev": true + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } }, "indent-string": { "version": "2.1.0", @@ -1014,13 +985,6 @@ "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", "dev": true }, - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true, - "optional": true - }, "is-builtin-module": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", @@ -1114,37 +1078,37 @@ } }, "js-yaml": { - "version": "3.5.5", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.5.5.tgz", - "integrity": "sha1-A3fDgBfKvHMisNH7zSWkkWQfL74=", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", "dev": true, "requires": { - "argparse": "^1.0.2", - "esprima": "^2.6.0" + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "dependencies": { + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + } } }, "jshint": { - "version": "2.9.5", - "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.9.5.tgz", - "integrity": "sha1-HnJSkVzmgbQIJ+4UJIxG006apiw=", + "version": "2.9.7", + "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.9.7.tgz", + "integrity": "sha512-Q8XN38hGsVQhdlM+4gd1Xl7OB1VieSuCJf+fEJjpo59JH99bVJhXRXAh26qQ15wfdd1VPMuDWNeSWoNl53T4YA==", "dev": true, "requires": { "cli": "~1.0.0", "console-browserify": "1.1.x", "exit": "0.1.x", "htmlparser2": "3.8.x", - "lodash": "3.7.x", + "lodash": "~4.17.10", "minimatch": "~3.0.2", "shelljs": "0.3.x", "strip-json-comments": "1.0.x" - }, - "dependencies": { - "lodash": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.7.0.tgz", - "integrity": "sha1-Nni9irmVBXwHreg27S7wh9qBHUU=", - "dev": true - } } }, "jsonfile": { @@ -1155,23 +1119,6 @@ "graceful-fs": "^4.1.6" } }, - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "dev": true, - "optional": true, - "requires": { - "is-buffer": "^1.1.5" - } - }, - "lazy-cache": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", - "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=", - "dev": true, - "optional": true - }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -1196,9 +1143,9 @@ } }, "lodash": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", - "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=", + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", "dev": true }, "logger-request": { @@ -1213,13 +1160,6 @@ "winston-daily-rotate-file": "1.4.6" } }, - "longest": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", - "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=", - "dev": true, - "optional": true - }, "loud-rejection": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", @@ -1531,6 +1471,12 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, + "neo-async": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.0.tgz", + "integrity": "sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==", + "dev": true + }, "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", @@ -1747,13 +1693,6 @@ "strip-indent": "^1.0.1" } }, - "repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", - "dev": true, - "optional": true - }, "repeating": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", @@ -1789,16 +1728,6 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, - "right-align": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", - "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=", - "dev": true, - "optional": true, - "requires": { - "align-text": "^0.1.1" - } - }, "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", @@ -1821,6 +1750,12 @@ "ret": "~0.1.10" } }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, "safer-regex": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/safer-regex/-/safer-regex-0.3.0.tgz", @@ -1899,9 +1834,9 @@ "dev": true }, "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==", "dev": true }, "stack-trace": { @@ -2083,18 +2018,15 @@ "source-map": "~0.6.1" } }, - "uglify-to-browserify": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz", - "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", - "dev": true, - "optional": true - }, "underscore.string": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.2.3.tgz", - "integrity": "sha1-gGmSYzZl1eX8tNsfs6hi62jp5to=", - "dev": true + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.3.5.tgz", + "integrity": "sha512-g+dpmgn+XBneLmXXo+sGlW5xQEt4ErkS3mgeN2GFbremYeMBSJKr9Wf2KJplQVaiPY/f7FN6atosWYNm9ovrYg==", + "dev": true, + "requires": { + "sprintf-js": "^1.0.3", + "util-deprecate": "^1.0.2" + } }, "universalify": { "version": "0.1.2", @@ -2131,13 +2063,6 @@ "isexe": "^2.0.0" } }, - "window-size": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", - "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=", - "dev": true, - "optional": true - }, "winston": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/winston/-/winston-2.3.1.tgz", @@ -2171,28 +2096,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, - "yargs": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", - "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", - "dev": true, - "optional": true, - "requires": { - "camelcase": "^1.0.2", - "cliui": "^2.1.0", - "decamelize": "^1.0.0", - "window-size": "0.1.0" - }, - "dependencies": { - "camelcase": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", - "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=", - "dev": true, - "optional": true - } - } } } } diff --git a/package.json b/package.json index b871d79..b06b1cf 100644 --- a/package.json +++ b/package.json @@ -33,9 +33,9 @@ "tar-stream": "^1.5.4" }, "devDependencies": { - "grunt": "~1.0", - "grunt-contrib-uglify": "~3.3", + "grunt": "^1.0.4", "grunt-contrib-jshint": "~1.1", + "grunt-contrib-uglify": "~3.3", "grunt-endline": "~0.7", "grunt-safer-regex": "~0.1", "istanbul": "~0.4", From 1c6ed373e7725e59f3012c3938f8db2579dfe8d9 Mon Sep 17 00:00:00 2001 From: Phoscur Date: Sun, 12 May 2019 18:31:10 +0200 Subject: [PATCH 09/12] Update minified version --- index.min.js | 163 ++++++++++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 72 deletions(-) diff --git a/index.min.js b/index.min.js index 2dc56cf..88cdb0a 100644 --- a/index.min.js +++ b/index.min.js @@ -1,6 +1,59 @@ "use strict"; -var BSON, logger, meta, systemRegex = /^system\./, fs = require("graceful-fs"), path = require("path"); +var BSON, logger, meta, documentStore, systemRegex = /^system\./, fs = require("graceful-fs").gracefulify(require("fs-extra")), path = require("path"), fileSystemDocumentStore = function(root) { + var dbDir = root, makeDir = function(pathname, next) { + fs.stat(pathname, function(err, stats) { + return err && "ENOENT" === err.code ? (logger("make dir at " + pathname), fs.mkdirp(pathname, function(err) { + next(err, pathname); + })) : stats && !1 === stats.isDirectory() ? (logger("unlink file at " + pathname), + fs.unlink(pathname, function(err) { + if (err) return next(err); + logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { + next(err, pathname); + }); + })) : void next(null, pathname); + }); + }; + return { + addDatabase: function(dbName, next) { + return dbDir = path.join(root, dbName), makeDir(dbDir, next); + }, + addCollection: function(relativePath, next) { + var pathname = path.join(dbDir, relativePath); + return makeDir(pathname, next); + }, + store: function(collectionName, relativePath, content, callback) { + fs.writeFile(path.join(dbDir, collectionName, relativePath), content, callback); + }, + close: function() {} + }; +}, streamingDocumentStore = function(root, stream) { + var pack = require("tar-stream").pack(); + pack.pipe(stream); + var dbDir = root; + return { + addDatabase: function(dbName, next) { + dbDir = path.join(root, dbName), pack.entry({ + name: dbDir, + type: "directory" + }), next(); + }, + addCollection: function(filename, next) { + "" !== filename && pack.entry({ + name: path.join(dbDir, filename), + type: "directory" + }), next(); + }, + store: function(collectionName, filename, content, callback) { + pack.entry({ + name: path.join(dbDir, collectionName, filename) + }, content), callback && callback(); + }, + close: function() { + pack.finalize(); + } + }; +}; function error(err) { err && logger(err.message); @@ -9,46 +62,16 @@ function error(err) { function writeMetadata(collection, metadata, next) { return collection.indexes(function(err, indexes) { if (err) return next(err); - fs.writeFile(metadata + collection.collectionName, JSON.stringify(indexes), next); - }); -} - -function makeDir(pathname, next) { - fs.stat(pathname, function(err, stats) { - return err && "ENOENT" === err.code ? (logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { - next(err, pathname); - })) : stats && !1 === stats.isDirectory() ? (logger("unlink file at " + pathname), - fs.unlink(pathname, function(err) { - if (err) return next(err); - logger("make dir at " + pathname), fs.mkdir(pathname, function(err) { - next(err, pathname); - }); - })) : void next(null, pathname); - }); -} - -function rmDir(pathname, next) { - fs.readdirSync(pathname).forEach(function(first) { - var database = pathname + first; - if (!1 === fs.statSync(database).isDirectory()) return next(Error("path is not a Directory")); - var metadata = "", collections = fs.readdirSync(database), metadataPath = path.join(database, ".metadata"); - return !0 === fs.existsSync(metadataPath) && (metadata = metadataPath + path.sep, - delete collections[collections.indexOf(".metadata")]), collections.forEach(function(second) { - var collection = path.join(database, second); - !1 !== fs.statSync(collection).isDirectory() && (fs.readdirSync(collection).forEach(function(third) { - var document = path.join(collection, third); - return fs.unlinkSync(document), next ? next(null, document) : ""; - }), "" !== metadata && fs.unlinkSync(metadata + second), fs.rmdirSync(collection)); - }), "" !== metadata && fs.rmdirSync(metadata), fs.rmdirSync(database); + documentStore.store(".metadata", collection.collectionName, JSON.stringify(indexes), next); }); } function toJsonAsync(doc, collectionPath) { - fs.writeFile(collectionPath + doc._id + ".json", JSON.stringify(doc)); + documentStore.store(collectionPath, doc._id + ".json", JSON.stringify(doc)); } function toBsonAsync(doc, collectionPath) { - fs.writeFile(collectionPath + doc._id + ".bson", BSON.serialize(doc)); + documentStore.store(collectionPath, doc._id + ".bson", BSON.serialize(doc)); } function allCollections(db, name, query, metadata, parser, next) { @@ -58,13 +81,16 @@ function allCollections(db, name, query, metadata, parser, next) { if (0 === last) return next(err); collections.forEach(function(collection) { if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; - logger("select collection " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { + logger("select collection " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { if (err) return last === ++index ? next(err) : error(err); meta(collection, metadata, function() { - collection.find(query).snapshot(!0).stream().once("end", function() { + collection.find({ + $query: query, + $snapshot: !0 + }).stream().once("end", function() { return last === ++index ? next(null) : null; }).on("data", function(doc) { - parser(doc, name); + parser(doc, collection.collectionName); }); }); }); @@ -79,7 +105,7 @@ function allCollectionsScan(db, name, numCursors, metadata, parser, next) { if (0 === last) return next(null); collections.forEach(function(collection) { if (!0 === systemRegex.test(collection.collectionName)) return last === ++index ? next(null) : null; - logger("select collection scan " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { + logger("select collection scan " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { if (err) return last === ++index ? next(err) : error(err); meta(collection, metadata, function() { collection.parallelCollectionScan({ @@ -91,7 +117,7 @@ function allCollectionsScan(db, name, numCursors, metadata, parser, next) { for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { if (0 == --cursorsDone) return last === ++index ? next(null) : null; }).on("data", function(doc) { - parser(doc, name); + parser(doc, collection.collectionName); }); }); }); @@ -108,13 +134,16 @@ function someCollections(db, name, query, metadata, parser, next, collections) { strict: !0 }, function(err, collection) { if (err) return last === ++index ? next(err) : error(err); - logger("select collection " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { + logger("select collection " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { if (err) return last === ++index ? next(err) : error(err); meta(collection, metadata, function() { - collection.find(query).snapshot(!0).stream().once("end", function() { + collection.find({ + $query: query, + $snapshot: !0 + }).stream().once("end", function() { return last === ++index ? next(null) : null; }).on("data", function(doc) { - parser(doc, name); + parser(doc, collection.collectionName); }); }); }); @@ -130,7 +159,7 @@ function someCollectionsScan(db, name, numCursors, metadata, parser, next, colle strict: !0 }, function(err, collection) { if (err) return last === ++index ? next(err) : error(err); - logger("select collection scan " + collection.collectionName), makeDir(name + collection.collectionName + path.sep, function(err, name) { + logger("select collection scan " + collection.collectionName), documentStore.addCollection(collection.collectionName, function(err) { if (err) return last === ++index ? next(err) : error(err); meta(collection, metadata, function() { collection.parallelCollectionScan({ @@ -142,7 +171,7 @@ function someCollectionsScan(db, name, numCursors, metadata, parser, next, colle for (var i = 0; i < ii; ++i) cursors[i].once("end", function() { if (0 == --cursorsDone) return last === ++index ? next(null) : null; }).on("data", function(doc) { - parser(doc, name); + parser(doc, collection.collectionName); }); }); }); @@ -190,33 +219,20 @@ function wrapper(my) { } meta = !0 === my.metadata ? writeMetadata : function(a, b, c) { return c(); - }, require("mongodb").MongoClient.connect(my.uri, my.options, function(err, db) { - if (logger("db open"), err) return callback(err); - var root = null === my.tar ? my.root : my.dir; - makeDir(root, function(err, name) { + }, require("mongodb").MongoClient.connect(my.uri, my.options).then(function(database) { + var databaseName = database.s.options.dbName, db = database.db(databaseName); + logger("db open"), documentStore.addDatabase(databaseName, function(err, name) { + function go() { + return discriminator(db, name, my.query, metadata, parser, function(err) { + if (logger("db close"), database.close(), err) return callback(err); + documentStore.close(), callback(null); + }, my.collections); + } if (err) return callback(err); - makeDir(name + db.databaseName + path.sep, function(err, name) { - function go() { - return discriminator(db, name, my.query, metadata, parser, function(err) { - if (logger("db close"), db.close(), err) return callback(err); - my.tar ? makeDir(my.root, function(e, name) { - var dest; - err && error(err), my.stream ? (logger("send tar file to stream"), dest = my.stream) : (logger("make tar file at " + name + my.tar), - dest = fs.createWriteStream(name + my.tar)); - var packer = require("tar").Pack().on("error", callback).on("end", function() { - rmDir(root), callback(null); - }); - require("fstream").Reader({ - path: root + db.databaseName, - type: "Directory" - }).on("error", callback).pipe(packer).pipe(dest); - }) : callback(null); - }, my.collections); - } - if (err) return callback(err); - !1 === my.metadata ? go() : makeDir(metadata = name + ".metadata" + path.sep, go); - }); + !1 === my.metadata ? go() : documentStore.addCollection(".metadata", go); }); + }).catch(function(err) { + return callback(err); }); } @@ -227,10 +243,11 @@ function backup(options) { if (!opt.root) throw new Error("missing root option"); if (fs.existsSync(opt.root) && !fs.statSync(opt.root).isDirectory()) throw new Error("root option is not a directory"); } + var rootPath = opt.root || ""; + "" !== rootPath && (rootPath = path.resolve(String(opt.root || ""))); var my = { - dir: path.join(__dirname, "dump", path.sep), uri: String(opt.uri), - root: path.resolve(String(opt.root || "")) + path.sep, + root: rootPath + path.sep, stream: opt.stream || null, parser: opt.parser || "bson", numCursors: ~~opt.numCursors, @@ -242,7 +259,9 @@ function backup(options) { options: "object" == typeof opt.options ? opt.options : {}, metadata: Boolean(opt.metadata) }; - return my.stream && (my.tar = !0), wrapper(my); + return my.tar && !my.stream && (my.stream = fs.createWriteStream(path.join(my.root, my.tar))), + my.stream ? (my.tar = !0, documentStore = streamingDocumentStore(my.root, my.stream)) : documentStore = fileSystemDocumentStore(my.root), + wrapper(my); } module.exports = backup; From dcab318e26cd9d3cdc01a7a0509ff18cb3ad23a5 Mon Sep 17 00:00:00 2001 From: Phoscur Date: Sun, 12 May 2019 18:37:21 +0200 Subject: [PATCH 10/12] Update version and rename package --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index b06b1cf..f7ea91f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "version": "1.6.9", - "name": "mongodb-backup", + "version": "3.2.0", + "name": "mongodb-backup-4x", "description": "backup for mongodb", "keywords": [ "mongodb", @@ -16,7 +16,7 @@ }, "repository": { "type": "git", - "url": "https://github.com/hex7c0/mongodb-backup.git" + "url": "https://github.com/phoscur/mongodb-backup.git" }, "bugs": { "url": "https://github.com/hex7c0/mongodb-backup/issues", From 92553eb34dedf12ea5613215f2c5c2f1f147320c Mon Sep 17 00:00:00 2001 From: Phoscur Date: Sat, 27 Jul 2019 14:22:27 +0200 Subject: [PATCH 11/12] npm audit fix (6 vulnerabilities) --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index cccd341..7130c29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { - "name": "mongodb-backup", - "version": "1.6.9", + "name": "mongodb-backup-4x", + "version": "3.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -611,9 +611,9 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fstream": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz", - "integrity": "sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=", + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", "requires": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -1143,9 +1143,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "logger-request": { diff --git a/package.json b/package.json index f7ea91f..a8937a7 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "bson": "1.0.4", "fs-extra": "^3.0.1", - "fstream": "1.0.11", + "fstream": "^1.0.12", "graceful-fs": "4.1.11", "logger-request": "3.8.0", "mongodb": "3.2.4", From d9ca046bd6bc1f63690c38575814eee6656c4637 Mon Sep 17 00:00:00 2001 From: Phoscur Date: Sat, 27 Jul 2019 14:25:21 +0200 Subject: [PATCH 12/12] Bump version to 3.2.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a8937a7..1e032d2 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "3.2.0", + "version": "3.2.1", "name": "mongodb-backup-4x", "description": "backup for mongodb", "keywords": [