From 050272a40f4f212fa2fb6b1cb02c9e11062aea0a Mon Sep 17 00:00:00 2001 From: JulienBrks Date: Wed, 14 Oct 2015 12:49:36 +0800 Subject: [PATCH 1/5] Fix calcHash function cannot replace all '+' to '-' and '/' to '_' --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 8da4625..ec33134 100644 --- a/index.js +++ b/index.js @@ -111,6 +111,6 @@ module.exports = function (qiniu, option) { var hash = new Buffer(1 + sha1.length); hash[0] = 0x16; sha1.copy(hash, 1); - return hash.toString('base64').replace('+', '-').replace('/', '_'); + return hash.toString('base64').replace(/+/g, '-').replace(/-/g, '_'); } }; From 0fa37f3d89ee333e1498156a8152e28b7dc10250 Mon Sep 17 00:00:00 2001 From: JulienBrks Date: Wed, 14 Oct 2015 12:54:09 +0800 Subject: [PATCH 2/5] Fix calcHash function replace error --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index ec33134..f850630 100644 --- a/index.js +++ b/index.js @@ -111,6 +111,6 @@ module.exports = function (qiniu, option) { var hash = new Buffer(1 + sha1.length); hash[0] = 0x16; sha1.copy(hash, 1); - return hash.toString('base64').replace(/+/g, '-').replace(/-/g, '_'); + return hash.toString('base64').replace(/+/g, '-').replace(/\//g, '_'); } }; From 762503f7bc22173cddccab7b1ccb6f4148e0e4c0 Mon Sep 17 00:00:00 2001 From: JulienBrks Date: Wed, 14 Oct 2015 13:08:31 +0800 Subject: [PATCH 3/5] fix calcHash function '+' need to be transferred --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index f850630..715c049 100644 --- a/index.js +++ b/index.js @@ -111,6 +111,6 @@ module.exports = function (qiniu, option) { var hash = new Buffer(1 + sha1.length); hash[0] = 0x16; sha1.copy(hash, 1); - return hash.toString('base64').replace(/+/g, '-').replace(/\//g, '_'); + return hash.toString('base64').replace(/\+/g, '-').replace(/\//g, '_'); } }; From 302953ea75b52a0877bdb312233b302e829a01c4 Mon Sep 17 00:00:00 2001 From: JulienBrks Date: Tue, 3 Nov 2015 14:49:51 +0800 Subject: [PATCH 4/5] add qetag.js from github.com/qiniu/qetag --- .gitignore | 1 + index.js | 80 ++++++++++++++++++++++-------------------------------- qetag.js | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 47 deletions(-) create mode 100644 qetag.js diff --git a/.gitignore b/.gitignore index 78f2710..3674c4c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ .idea/ +npm-debug.log diff --git a/index.js b/index.js index 715c049..8c29045 100644 --- a/index.js +++ b/index.js @@ -11,6 +11,7 @@ var util = require('util') var crypto = require('crypto') var minimatch = require('minimatch') var uploadedFiles = 0; +var getEtag = require('getEtag'); module.exports = function (qiniu, option) { option = option || {}; @@ -33,40 +34,42 @@ module.exports = function (qiniu, option) { var fileKey = option.dir + ((!option.dir || option.dir[option.dir.length - 1]) === '/' ? '' : '/') + (option.versioning ? version + '/' : '') + filePath; var fileHash = calcHash(file); + getEtag(file, function (fileHash) { + qs.push(Q.nbind(qn.stat, qn)(fileKey) + .spread(function (stat) { + // Skip when hash equal + if (stat.hash === fileHash) return false; - qs.push(Q.nbind(qn.stat, qn)(fileKey) - .spread(function (stat) { - // Skip when hash equal - if (stat.hash === fileHash) return false; + // Then delete + return Q.nbind(qn.delete, qn)(fileKey) + }, function () { + // Upload when not exists + return true; + }) + .then(function (isUpload) { + if (isUpload === false) return false; + return Q.nbind(qn.upload, qn)(file._contents, {key: fileKey}) + }) + .then(function (stat) { + // No upload + if (stat === false) { + log('Skip:', colors.grey(filePath)); + return; + } - // Then delete - return Q.nbind(qn.delete, qn)(fileKey) - }, function () { - // Upload when not exists - return true; - }) - .then(function (isUpload) { - if (isUpload === false) return false; - return Q.nbind(qn.upload, qn)(file._contents, {key: fileKey}) - }) - .then(function (stat) { - // No upload - if (stat === false) { - log('Skip:', colors.grey(filePath)); - return; - } - - // Record hash - uploadedFiles++; + // Record hash + uploadedFiles++; - log('Upload:', colors.green(filePath), '→', colors.green(fileKey)); - }, function (err) { - log('Error', colors.red(filePath), new PluginError('gulp-qiniu', err).message); - that.emit('Error', colors.red(filePath), new PluginError('gulp-qiniu', err)); - }) - ) + log('Upload:', colors.green(filePath), '→', colors.green(fileKey)); + }, function (err) { + log('Error', colors.red(filePath), new PluginError('gulp-qiniu', err).message); + that.emit('Error', colors.red(filePath), new PluginError('gulp-qiniu', err)); + }) + ) - next(); + next(); + }); + }, function () { Q.all(qs) .then(function (rets) { @@ -96,21 +99,4 @@ module.exports = function (qiniu, option) { } return target; } - - /** - * Calc qiniu etag - * - * @param file - * @returns {*} - */ - function calcHash(file) { - if (file.size > 1 << 22) return false; - var shasum = crypto.createHash('sha1'); - shasum.update(file._contents); - var sha1 = shasum.digest(); - var hash = new Buffer(1 + sha1.length); - hash[0] = 0x16; - sha1.copy(hash, 1); - return hash.toString('base64').replace(/\+/g, '-').replace(/\//g, '_'); - } }; diff --git a/qetag.js b/qetag.js new file mode 100644 index 0000000..f7b04a2 --- /dev/null +++ b/qetag.js @@ -0,0 +1,79 @@ +// 计算文件的eTag，参数为buffer或者readableStream或者文件路径 +function getEtag(buffer,callback){ + + // 判断传入的参数是buffer还是stream还是filepath + var mode = 'buffer'; + + if(typeof buffer === 'string'){ + buffer = require('fs').createReadStream(buffer); + mode='stream'; + }else if(buffer instanceof require('stream')){ + mode='stream'; + } + + // sha1算法 + var sha1 = function(content){ + var crypto = require('crypto'); + var sha1 = crypto.createHash('sha1'); + sha1.update(content); + return sha1.digest(); + }; + + // 以4M为单位分割 + var blockSize = 4*1024*1024; + var sha1String = []; + var prefix = 0x16; + var blockCount = 0; + + switch(mode){ + case 'buffer': + var bufferSize = buffer.length; + blockCount = Math.ceil(bufferSize / blockSize); + + for(var i=0;i 1){ + prefix = 0x96; + sha1Buffer = sha1(sha1Buffer); + } + + sha1Buffer = Buffer.concat( + [new Buffer([prefix]),sha1Buffer], + sha1Buffer.length + 1 + ); + + return sha1Buffer.toString('base64') + .replace(/\//g,'_').replace(/\+/g,'-'); + + } + +} + +module.exports = getEtag; From d7f9eeab2b2af4893766d7c5523b6c429f2ee402 Mon Sep 17 00:00:00 2001 From: JulienBrks Date: Tue, 3 Nov 2015 15:30:40 +0800 Subject: [PATCH 5/5] =?UTF-8?q?fix(getEtag):=20=E4=BB=A3=E7=A0=81=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 8c29045..7924f8b 100644 --- a/index.js +++ b/index.js @@ -11,7 +11,7 @@ var util = require('util') var crypto = require('crypto') var minimatch = require('minimatch') var uploadedFiles = 0; -var getEtag = require('getEtag'); +var getEtag = require('./qetag'); module.exports = function (qiniu, option) { option = option || {}; @@ -33,8 +33,7 @@ module.exports = function (qiniu, option) { if (isIgnore) return next(); var fileKey = option.dir + ((!option.dir || option.dir[option.dir.length - 1]) === '/' ? '' : '/') + (option.versioning ? version + '/' : '') + filePath; - var fileHash = calcHash(file); - getEtag(file, function (fileHash) { + getEtag(file.path, function (fileHash) { qs.push(Q.nbind(qn.stat, qn)(fileKey) .spread(function (stat) { // Skip when hash equal