From 91b704d533ce4f89292856cb935c12ba1d6fcf7e Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 8 Nov 2018 16:00:59 +0000 Subject: [PATCH 1/2] feat: dynamic format loading This backwards compatible PR allows missing IPLD formats to be loaded dynamically. This follows on from the discussion here https://github.com/ipld/js-ipld/pull/164#discussion_r228121092 A new constructor option `loadFormat` allows users to asynchronously load an IPLD format. The IPLD instance will call this function automatically when it is requested to resolve a format that it doesn't currently understand. This can re-enable lazy loading for IPLD modules in IPFS and also means that people who just want to add _more_ resolvers can do so without having to depend directly on the defaults and pass them in as well. License: MIT Signed-off-by: Alan Shaw --- README.md | 20 ++++++ src/index.js | 152 ++++++++++++++++++++--------------------- test/browser.js | 1 + test/format-support.js | 83 ++++++++++++++++++++++ test/node.js | 1 + 5 files changed, 179 insertions(+), 78 deletions(-) create mode 100644 test/format-support.js diff --git a/README.md b/README.md index 114f060..8779a18 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,26 @@ const ipld = new Ipld({ }) ``` +##### `options.loadFormat(codec, callback)` + +| Type | Default | +|------|---------| +| `Function` | `null` | + +Function to dynamically load an [IPLD Format](https://github.com/ipld/interface-ipld-format). It is passed a string `codec`, the multicodec of the IPLD format to load and a callback function to call when the format has been loaded. e.g. + +```js +const ipld = new Ipld({ + loadFormat (codec, callback) { + if (codec === 'git-raw') { + callback(null, require('ipld-git')) + } else { + callback(new Error('unable to load format ' + codec)) + } + } +}) +``` + ### `.put(node, options, callback)` > Store the given node of a recognized IPLD Format. diff --git a/src/index.js b/src/index.js index 9b1ab23..a6bf28e 100644 --- a/src/index.js +++ b/src/index.js @@ -48,6 +48,10 @@ class IPLDResolver { } } + this.support.load = options.loadFormat || ((codec, callback) => { + callback(new Error(`No resolver found for codec "${codec}"`)) + }) + this.support.rm = (multicodec) => { if (this.resolvers[multicodec]) { delete this.resolvers[multicodec] @@ -99,26 +103,24 @@ class IPLDResolver { doUntil( (cb) => { - const r = this.resolvers[cid.codec] - - if (!r) { - return cb(new Error('No resolver found for codec "' + cid.codec + '"')) - } + this._getFormat(cid.codec, (err, format) => { + if (err) return cb(err) - // get block - // use local resolver - // update path value - this.bs.get(cid, (err, block) => { - if (err) { - return cb(err) - } - r.resolver.resolve(block.data, path, (err, result) => { + // get block + // use local resolver + // update path value + this.bs.get(cid, (err, block) => { if (err) { return cb(err) } - value = result.value - path = result.remainderPath - cb() + format.resolver.resolve(block.data, path, (err, result) => { + if (err) { + return cb(err) + } + value = result.value + path = result.remainderPath + cb() + }) }) }) }, @@ -182,17 +184,9 @@ class IPLDResolver { return callback(err) } map(blocks, (block, mapCallback) => { - const resolver = this.resolvers[block.cid.codec] - if (!resolver) { - return mapCallback( - new Error('No resolver found for codec "' + block.cid.codec + '"')) - } - - resolver.util.deserialize(block.data, (err, deserialized) => { - if (err) { - return mapCallback(err) - } - return mapCallback(null, deserialized) + this._getFormat(block.cid.codec, (err, format) => { + if (err) return mapCallback(err) + format.util.deserialize(block.data, mapCallback) }) }, callback) @@ -216,20 +210,20 @@ class IPLDResolver { return this._put(options.cid, node, callback) } - const r = this.resolvers[options.format] - if (!r) { - return callback(new Error('No resolver found for codec "' + options.format + '"')) - } - r.util.cid(node, options, (err, cid) => { - if (err) { - return callback(err) - } + this._getFormat(options.format, (err, format) => { + if (err) return callback(err) - if (options.onlyHash) { - return callback(null, cid) - } + format.util.cid(node, options, (err, cid) => { + if (err) { + return callback(err) + } - this._put(cid, node, callback) + if (options.onlyHash) { + return callback(null, cid) + } + + this._put(cid, node, callback) + }) }) } @@ -245,15 +239,14 @@ class IPLDResolver { if (!options.recursive) { p = pullDeferSource() - const r = this.resolvers[cid.codec] - if (!r) { - p.abort(new Error('No resolver found for codec "' + cid.codec + '"')) - return p - } waterfall([ - (cb) => this.bs.get(cid, cb), - (block, cb) => r.resolver.tree(block.data, cb) + (cb) => this._getFormat(cid.codec, cb), + (format, cb) => this.bs.get(cid, (err, block) => { + if (err) return cb(err) + cb(null, format, block) + }), + (format, block, cb) => format.resolver.tree(block.data, cb) ], (err, paths) => { if (err) { p.abort(err) @@ -280,20 +273,19 @@ class IPLDResolver { const deferred = pullDeferSource() const cid = el.cid - const r = this.resolvers[cid.codec] - if (!r) { - deferred.abort(new Error('No resolver found for codec "' + cid.codec + '"')) - return deferred - } waterfall([ - (cb) => this.bs.get(el.cid, cb), - (block, cb) => r.resolver.tree(block.data, (err, paths) => { + (cb) => this._getFormat(cid.codec, cb), + (format, cb) => this.bs.get(cid, (err, block) => { + if (err) return cb(err) + cb(null, format, block) + }), + (format, block, cb) => format.resolver.tree(block.data, (err, paths) => { if (err) { return cb(err) } map(paths, (p, cb) => { - r.resolver.isLink(block.data, p, (err, link) => { + format.resolver.isLink(block.data, p, (err, link) => { if (err) { return cb(err) } @@ -356,38 +348,42 @@ class IPLDResolver { /* */ _get (cid, callback) { - const r = this.resolvers[cid.codec] - if (!r) { - return callback(new Error('No resolver found for codec "' + cid.codec + '"')) - } - waterfall([ - (cb) => this.bs.get(cid, cb), - (block, cb) => { - if (r) { - r.util.deserialize(block.data, (err, deserialized) => { - if (err) { - return cb(err) - } - cb(null, deserialized) - }) - } else { // multicodec unknown, send back raw data - cb(null, block.data) - } + (cb) => this._getFormat(cid.codec, cb), + (format, cb) => this.bs.get(cid, (err, block) => { + if (err) return cb(err) + cb(null, format, block) + }), + (format, block, cb) => { + format.util.deserialize(block.data, (err, deserialized) => { + if (err) { + return cb(err) + } + cb(null, deserialized) + }) } ], callback) } + _getFormat (codec, callback) { + if (this.resolvers[codec]) { + return callback(null, this.resolvers[codec]) + } + + // If not supported, attempt to dynamically load this format + this.support.load(codec, (err, format) => { + if (err) return callback(err) + this.resolvers[codec] = format + callback(null, format) + }) + } + _put (cid, node, callback) { callback = callback || noop - const r = this.resolvers[cid.codec] - if (!r) { - return callback(new Error('No resolver found for codec "' + cid.codec + '"')) - } - waterfall([ - (cb) => r.util.serialize(node, cb), + (cb) => this._getFormat(cid.codec, cb), + (format, cb) => format.util.serialize(node, cb), (buf, cb) => this.bs.put(new Block(buf, cid), cb) ], (err) => { if (err) { @@ -425,7 +421,7 @@ IPLDResolver.defaultOptions = { } /** - * Create an IPLD resolver with an inmemory blockservice and + * Create an IPLD resolver with an in memory blockservice and * repo. * * @param {function(Error, IPLDResolver)} callback diff --git a/test/browser.js b/test/browser.js index 512f5a4..9d3ce5d 100644 --- a/test/browser.js +++ b/test/browser.js @@ -38,6 +38,7 @@ describe('Browser', () => { }) require('./basics')(repo) + require('./format-support')(repo) require('./ipld-dag-pb')(repo) require('./ipld-dag-cbor')(repo) require('./ipld-git')(repo) diff --git a/test/format-support.js b/test/format-support.js new file mode 100644 index 0000000..7945398 --- /dev/null +++ b/test/format-support.js @@ -0,0 +1,83 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const BlockService = require('ipfs-block-service') +const dagCBOR = require('ipld-dag-cbor') + +const IPLDResolver = require('../src') + +module.exports = (repo) => { + describe('IPLD format support', () => { + let data, cid + + before((done) => { + const bs = new BlockService(repo) + const resolver = new IPLDResolver({ blockService: bs }) + + data = { now: Date.now() } + + dagCBOR.util.cid(data, (err, c) => { + expect(err).to.not.exist() + cid = c + resolver.put(data, { cid }, done) + }) + }) + + describe('Dynamic format loading', () => { + it('should fail to dynamically load format', (done) => { + const bs = new BlockService(repo) + const resolver = new IPLDResolver({ + blockService: bs, + formats: [] + }) + + resolver.get(cid, '/', (err) => { + expect(err).to.exist() + expect(err.message).to.equal('No resolver found for codec "dag-cbor"') + done() + }) + }) + + it('should fail to dynamically load format via loadFormat option', (done) => { + const errMsg = 'BOOM' + Date.now() + const bs = new BlockService(repo) + const resolver = new IPLDResolver({ + blockService: bs, + formats: [], + loadFormat (codec, callback) { + if (codec !== 'dag-cbor') return callback(new Error('unexpected codec')) + setTimeout(() => callback(new Error(errMsg))) + } + }) + + resolver.get(cid, '/', (err) => { + expect(err).to.exist() + expect(err.message).to.equal(errMsg) + done() + }) + }) + + it('should dynamically load missing format', (done) => { + const bs = new BlockService(repo) + const resolver = new IPLDResolver({ + blockService: bs, + formats: [], + loadFormat (codec, callback) { + if (codec !== 'dag-cbor') return callback(new Error('unexpected codec')) + setTimeout(() => callback(null, dagCBOR)) + } + }) + + resolver.get(cid, '/', (err, result) => { + expect(err).to.not.exist() + expect(result.value).to.eql(data) + done() + }) + }) + }) + }) +} diff --git a/test/node.js b/test/node.js index ba5ad35..aeb924f 100644 --- a/test/node.js +++ b/test/node.js @@ -27,6 +27,7 @@ describe('Node.js', () => { }) require('./basics')(repo) + require('./format-support')(repo) require('./ipld-dag-pb')(repo) require('./ipld-dag-cbor')(repo) require('./ipld-git')(repo) From f5fb929b353eccd8041214b88857a42c278f1139 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 9 Nov 2018 10:26:36 +0000 Subject: [PATCH 2/2] test: add test for not dynbamically loading existing format License: MIT Signed-off-by: Alan Shaw --- test/format-support.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/format-support.js b/test/format-support.js index 7945398..be05e1e 100644 --- a/test/format-support.js +++ b/test/format-support.js @@ -78,6 +78,23 @@ module.exports = (repo) => { done() }) }) + + it('should not dynamically load format added statically', (done) => { + const bs = new BlockService(repo) + const resolver = new IPLDResolver({ + blockService: bs, + formats: [dagCBOR], + loadFormat (codec) { + throw new Error(`unexpected load format ${codec}`) + } + }) + + resolver.get(cid, '/', (err, result) => { + expect(err).to.not.exist() + expect(result.value).to.eql(data) + done() + }) + }) }) }) }