feat: avoid doing multiple RPC requests for files.add, fixes #522 (#595)

* chore: test against go-ipfs 0.4.11

* feat: avoid doing multiple RPC requests for files.add, fixes #522

* new bundle sizes, so much space saved!

* use converter

* add progress tracking back

Author: daviddias

examples/sub-module/bundles-size-KBs.csv

@@ -1,28 +1,28 @@
 name, bundled (KBs), minified (KBs)
-IPFS, 1685.41, 661.64
-add, 968.11, 344.13
-bitswap, 684.80, 232.13
-block, 724.77, 250.23
-bootstrap, 685.64, 232.39
-cat, 725.05, 250.33
-commands, 683.89, 231.76
-config, 686.87, 233.10
-dht, 688.51, 233.41
-diag, 684.97, 232.18
-files, 1120.38, 404.30
-get, 907.74, 318.35
-id, 684.31, 231.95
-key, 684.59, 232.03
-log, 685.40, 232.38
-ls, 684.00, 231.80
-mount, 684.18, 231.86
-name, 684.63, 232.06
-object, 923.66, 340.57
-pin, 685.51, 232.39
-ping, 684.59, 231.90
-pubsub, 740.40, 249.54
-refs, 684.39, 231.94
-repo, 684.56, 232.02
-swarm, 1324.18, 527.03
-update, 684.45, 231.96
-version, 684.21, 231.88
+IPFS, 1412.20, 573.44
+add, 591.17, 198.23
+bitswap, 590.14, 197.96
+block, 630.50, 216.31
+bootstrap, 590.97, 198.22
+cat, 630.78, 216.41
+commands, 589.22, 197.59
+config, 592.21, 198.93
+dht, 593.86, 199.24
+diag, 590.31, 198.00
+files, 669.07, 235.88
+get, 661.57, 233.16
+id, 589.65, 197.78
+key, 589.93, 197.86
+log, 590.74, 198.20
+ls, 589.35, 197.63
+mount, 589.53, 197.69
+name, 589.97, 197.88
+object, 833.17, 307.73
+pin, 590.86, 198.22
+ping, 589.94, 197.73
+pubsub, 595.31, 199.76
+refs, 589.74, 197.77
+repo, 589.91, 197.85
+swarm, 1239.42, 498.59
+update, 589.79, 197.79
+version, 589.55, 197.71

examples/sub-module/test-modules-size.sh

@@ -6,7 +6,7 @@ modules=($(ls modules/))
 echo "name, bundled (KBs), minified (KBs)"
 
 # Full IPFS module
-webpack --display none --config webpack.confg.js complete-module.js complete-bundle.js
+webpack --display none --config webpack.config.js complete-module.js complete-bundle.js
 babili complete-bundle.js -o complete-bundle-minified.js
 
 ipfsBundleSize=($(wc -c < complete-bundle.js | awk '{b=$1/1024; printf "%.2f\n", b}' | sed 's/,/./g'))

package.json

@@ -67,7 +67,7 @@
     "gulp": "^3.9.1",
     "interface-ipfs-core": "~0.32.1",
     "hapi": "^16.6.2",
-    "ipfsd-ctl": "~0.23.0",
+    "ipfsd-ctl": "~0.24.0",
     "pre-commit": "^1.2.2",
     "socket.io": "^2.0.3",
     "socket.io-client": "^2.0.3",

src/files/add.js

@@ -2,8 +2,8 @@
 
 const isStream = require('is-stream')
 const promisify = require('promisify-es6')
-const DAGNodeStream = require('../utils/dagnode-stream')
 const ProgressStream = require('../utils/progress-stream')
+const converter = require('../utils/converter')
 
 module.exports = (send) => {
   return promisify((files, opts, callback) => {
@@ -44,9 +44,8 @@ module.exports = (send) => {
 
     const request = { path: 'add', files: files, qs: qs, progress: opts.progress }
 
-    // Transform the response stream to DAGNode values
-    const transform = (res, callback) => DAGNodeStream
-      .streamToValue(send, ProgressStream.fromStream(opts.progress, res), callback)
-    send.andTransform(request, transform, callback)
+    send.andTransform(request, (response, cb) => {
+      converter(ProgressStream.fromStream(opts.progress, response), cb)
+    }, callback)
   })
 }

src/util/fs-add.js

@@ -2,7 +2,7 @@
 
 const isNode = require('detect-node')
 const promisify = require('promisify-es6')
-const DAGNodeStream = require('../utils/dagnode-stream')
+const converter = require('../utils/converter')
 const moduleConfig = require('../utils/module-config')
 
 module.exports = (arg) => {
@@ -31,14 +31,7 @@ module.exports = (arg) => {
       return callback(new Error('"path" must be a string'))
     }
 
-    const request = {
-      path: 'add',
-      qs: opts,
-      files: path
-    }
-
-    // Transform the response stream to DAGNode values
-    const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback)
-    send.andTransform(request, transform, callback)
+    const request = { path: 'add', files: path, qs: opts }
+    send.andTransform(request, converter, callback)
   })
 }

src/util/url-add.js

@@ -4,7 +4,7 @@ const promisify = require('promisify-es6')
 const once = require('once')
 const parseUrl = require('url').parse
 const request = require('../utils/request')
-const DAGNodeStream = require('../utils/dagnode-stream')
+const converter = require('../utils/converter')
 const moduleConfig = require('../utils/module-config')
 
 module.exports = (arg) => {
@@ -25,12 +25,12 @@ module.exports = (arg) => {
       opts = {}
     }
 
+    callback = once(callback)
+
     if (!validUrl(url)) {
       return callback(new Error('"url" param must be an http(s) url'))
     }
 
-    callback = once(callback)
-
     requestWithRedirect(url, opts, send, callback)
   })
 }
@@ -52,14 +52,9 @@ const requestWithRedirect = (url, opts, send, callback) => {
       }
       requestWithRedirect(redirection, opts, send, callback)
     } else {
-      const params = {
-        path: 'add',
-        qs: opts,
-        files: res
-      }
-      // Transform the response stream to DAGNode values
-      const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback)
-      send.andTransform(params, transform, callback)
+      const request = { path: 'add', files: res, qs: opts }
+
+      send.andTransform(request, converter, callback)
     }
   }).end()
 }

src/utils/dagnode-stream.js renamed to src/utils/converter.js

@@ -3,18 +3,18 @@
 const pump = require('pump')
 const TransformStream = require('readable-stream').Transform
 const streamToValue = require('./stream-to-value')
-const getDagNode = require('./get-dagnode')
 
 /*
   Transforms a stream of {Name, Hash} objects to include size
   of the DAG object.
 
-  Usage: inputStream.pipe(DAGNodeStream({ send: send }))
+  Usage: inputStream.pipe(new Converter())
 
   Input object format:
   {
     Name: '/path/to/file/foo.txt',
     Hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP'
+    Size: '20'
   }
 
   Output object format:
@@ -24,38 +24,34 @@ const getDagNode = require('./get-dagnode')
     size: 20
   }
 */
-class DAGNodeStream extends TransformStream {
+class ConverterStream extends TransformStream {
   constructor (options) {
     const opts = Object.assign(options || {}, { objectMode: true })
     super(opts)
-    this._send = opts.send
   }
 
-  static streamToValue (send, inputStream, callback) {
-    const outputStream = pump(inputStream, new DAGNodeStream({ send: send }), (err) => {
-      if (err) {
-        callback(err)
-      }
+  _transform (obj, enc, callback) {
+    this.push({
+      path: obj.Name,
+      hash: obj.Hash,
+      size: parseInt(obj.Size, 10)
     })
-    streamToValue(outputStream, callback)
+
+    callback(null)
   }
+}
 
-  _transform (obj, enc, callback) {
-    getDagNode(this._send, obj.Hash, (err, node) => {
+function converter (inputStream, callback) {
+  const outputStream = pump(
+    inputStream,
+    new ConverterStream(),
+    (err) => {
       if (err) {
-        return callback(err)
-      }
-
-      const result = {
-        path: obj.Name,
-        hash: obj.Hash,
-        size: node.size
+        callback(err)
       }
-
-      this.push(result)
-      callback(null)
     })
-  }
+
+  streamToValue(outputStream, callback)
 }
 
-module.exports = DAGNodeStream
+module.exports = converter

src/utils/get-dagnode.js

@@ -6,9 +6,9 @@ const concat = require('concat-stream')
 /*
   Concatenate a stream to a single value.
 */
-function streamToValue (res, callback) {
+function streamToValue (response, callback) {
   pump(
-    res,
+    response,
     concat((data) => callback(null, data)),
     (err) => {
       if (err) {