Merge pull request #31 from CfABrigadePhiladelphia/import_stream

import data using createImportStream

Author: CameronSima

.gitignore

@@ -1,2 +1,3 @@
 tmp
 node_modules
+data.jawn

index.js

@@ -1,5 +1,16 @@
-module.exports = Jawn
-function Jawn () {}
-Jawn.prototype.import = function (data) {
-  return 'Hello World'
-}
+var level = require('level')
+var hypercore = require('hypercore')
+var createImportStream = require('./lib/import.js')
+
+module.exports = Jawn
+
+function Jawn (opts) {
+  if (!opts) opts = {}
+  var db = opts.db || level('data.jawn')
+  this.core = opts.core || hypercore(db)
+  this.db = this.core.db
+}
+
+Jawn.prototype.createImportStream = function (opts) {
+  return createImportStream(this, opts)
+}

lib/import.js

@@ -0,0 +1,32 @@
+var pump = require('pump')
+var through = require('through2')
+
+module.exports = importStream
+
+function importStream (jawn, opts) {
+  if (!opts) opts = {}
+  var writeStream = jawn.core.createWriteStream(opts)
+  var stream = pump(parseInputStream(opts), writeStream, function done (err) {
+    stream.id = writeStream.id
+    console.log(err)
+  })
+  return stream
+}
+
+// Transform input CSV, JSON, etc to json objects that will be written as blocks in our hypercore feed
+// TODO: Implement this! See https://github.com/CfABrigadePhiladelphia/jawn/issues/32
+// This is the functionality that parse-input-stream aims to support.
+// If you can get that module to work, feel free to use it!
+function parseInputStream (opts) {
+  var transformStream = through(write, end)
+  return transformStream
+
+  function write (buffer, encoding, next) {
+    // transform input CSV, JSON, etc. here ...
+    next()
+  }
+
+  function end (done) {
+    done()
+  }
+}

package.json

@@ -21,10 +21,18 @@
   },
   "homepage": "https://github.com/CfABrigadePhiladelphia/jawn",
   "dependencies": {
-    "hypercore": "^1.6.2"
+    "fs": "0.0.2",
+    "hypercore": "~1.8.0",
+    "level": "~1.4.0",
+    "parse-input-stream": "^1.0.1",
+    "path": "~0.12.7",
+    "pump": "^1.0.1",
+    "tape": "~4.5.1",
+    "through2": "^2.0.1"
   },
   "devDependencies": {
     "standard": "^6.0.5",
-    "tape": "^4.4.0"
+    "tape": "^4.4.0",
+    "memdb": "^1.3.1"
   }
 }

test/data/sample.csv

@@ -0,0 +1,5 @@
+Type of Experience,Little/No Experience,Some Experience,Very Familiar
+Writing software in any programming language,1,5,4
+Frontend Web Development,4,3,3
+Server-side (“backend”) Web Development,4,4,2
+"Using Git to track changes and share code (add, commit, push, pull)",2,5,3

test/import.js

@@ -0,0 +1,39 @@
+var test = require('tape')
+var Jawn = require('../')
+var memdb = require('memdb')
+
+test('import json to jawn', function (t) {
+  var jawn = freshJawn()
+  var importStream = jawn.createImportStream({'format': 'json'})
+  // Imitate the stream that would come from reading sample.csv
+  // importStream should parse the CSV correctly, identifying the first row as headers
+  // This is the same as doing
+  //    var data = fs.createReadStream('./test/data/sample.csv')
+  //    data.pipe(importStream)
+  // except the writes are being performed synchronously/inline so we can call importStream.end() after writing the contents into it.
+  importStream.write('{foo: "bar", name: "josie"}')
+  importStream.write('{foo: "baz", name: "eloise"}')
+  importStream.write('{foo: "baz", name: "francoise"}')
+
+  // This should be expecting JSON objects, not strings.
+  // temporarily expecting strings in order to hand off the code as-is
+  var expected = [
+    '{foo: "bar", name: "josie"}',
+    '{foo: "baz", name: "eloise"}',
+    '{foo: "baz", name: "francoise"}'
+  ]
+
+  importStream.end(function () {
+    var feedId = importStream.id.toString('hex')
+    var rs = jawn.core.createReadStream(feedId)
+    rs.on('data', function (block) {
+      t.same(block.toString(), expected.shift(), 'block matches imported line')
+    })
+    t.same(jawn.core.get(feedId).blocks, 3, 'correct number of blocks returned')
+    t.end()
+  })
+})
+
+function freshJawn () {
+  return new Jawn({db: memdb()})
+}

test/jawn.js

@@ -1,6 +1,26 @@
 var test = require('tape')
 var Jawn = require('../index')
-test('jawn import', function (t) {
-  t.equal(new Jawn().import('some data'), 'Hello World')
+var memdb = require('memdb')
+var hypercore = require('hypercore')
+
+test('jawn constructor defaults', function (t) {
+  var jawn = new Jawn({db: memdb()})
+  t.true(jawn.core instanceof hypercore, 'core is an instance of Hypercore')
+  // t.true(jawn.db instanceof leveldb, 'db is an instance of leveldb')
+  t.equal(jawn.db, jawn.core.db, 'jawn db matches core db')
+  t.end()
+})
+
+test('jawn constructor set db', function (t) {
+  var sampledb = memdb()
+  var jawn = new Jawn({db: sampledb})
+  t.equal(jawn.core.db, sampledb, 'initializes hypercore with the provided db')
+  t.end()
+})
+
+test('jawn constructor set core', function (t) {
+  var samplecore = hypercore(memdb())
+  var jawn = new Jawn({core: samplecore})
+  t.equal(jawn.core, samplecore, 'uses the provided instance of hypercore')
   t.end()
 })