diff --git a/index.js b/index.js index 256df3fd..5db1cd0e 100644 --- a/index.js +++ b/index.js @@ -12,6 +12,7 @@ const Hypercore = require('hypercore') const { BLOCK_NOT_AVAILABLE, BAD_ARGUMENT } = require('hypercore-errors') const Monitor = require('./lib/monitor') const Download = require('./lib/download') +const header = require('./lib/header') const keyEncoding = new SubEncoder('files', 'utf-8') @@ -565,7 +566,7 @@ module.exports = class Hyperdrive extends ReadyResource { return stream } - createWriteStream(name, { executable = false, metadata = null } = {}) { + createWriteStream(name, { executable = false, metadata = null, dedup = false } = {}) { const self = this let destroyed = false @@ -573,6 +574,8 @@ module.exports = class Hyperdrive extends ReadyResource { let ondrain = null let onfinish = null + const blocks = dedup ? { blocks: [], byteLengths: [] } : null + const stream = new Writable({ open(cb) { self.getBlobs().then(onblobs, cb) @@ -604,6 +607,10 @@ module.exports = class Hyperdrive extends ReadyResource { } }, write(data, cb) { + if (blocks) { + blocks.blocks.push(self.blobs.core.length) + blocks.byteLengths.push(data.byteLength) + } if (ws.write(data) === true) return cb(null) ondrain = cb }, @@ -626,13 +633,24 @@ module.exports = class Hyperdrive extends ReadyResource { onfinish = null if (err) return cb(err) - self.db - .put( - std(name, false), - { executable, linkname: null, blob: ws.id, metadata }, - { keyEncoding } - ) - .then(() => cb(null), cb) + + flush().then(() => cb(null), cb) + } + + async function flush() { + let b = null + + if (blocks) { + const blks = header.encode(blocks) + b = { blockOffset: self.blobs.core.length, blockLength: blks.length } + await self.blobs.core.append(blks) + } + + await self.db.put( + std(name, false), + { executable, linkname: null, blob: ws.id, metadata, blocks: b }, + { keyEncoding } + ) } function callOndrain(err) { diff --git a/lib/header.js b/lib/header.js new file mode 100644 index 00000000..40a1dbde --- /dev/null +++ b/lib/header.js @@ -0,0 +1,48 @@ +const b4a = require('b4a') +const c = require('compact-encoding') + +const SOFT_LIMIT = 128 * 1024 + +const uints = c.array(c.uint) + +exports.encode = function ({ blocks, byteLengths }) { + const result = [] + + const state = { start: 0, end: 0, buffer: null } + + uints.preencode(state, blocks) + uints.preencode(state, byteLengths) + + if (state.end > SOFT_LIMIT) { + const result = [] + + state.start = state.end = 0 + + uints.preencode(state, blocks) + state.buffer = b4a.allocUnsafe(state.end) + uints.encode(state, blocks) + + result.push(state.buffer) + + state.start = state.end = 0 + + uints.preencode(state, byteLengths) + state.buffer = b4a.allocUnsafe(state.end) + uints.encode(state, byteLengths) + + result.push(state.buffer) + return result + } + + state.buffer = b4a.allocUnsafe(state.end) + uints.encode(state, blocks) + uints.encode(state, byteLengths) + + result.push(state.buffer) + + return result +} + +exports.decode = function (buf) { + +}