feat: enable custom formats for dag put and get

examples/custom-ipld-formats/README.md

@@ -0,0 +1,31 @@
+# Custom IPLD formats
+
+This example shows you how to configure an IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications.
+
+## Before you start
+
+First clone this repo, install dependencies in the project root and build the project.
+
+```console
+$ git clone https://github.com/ipfs/js-ipfs.git
+$ cd js-ipfs
+$ npm install
+$ npm run build
+```
+
+## Running the example
+
+Running this example should result in metrics being logged out to the console every few seconds.
+
+```
+> npm start
+```
+
+## Play with the configuration!
+
+By default, IPFS is only configured to support a few common IPLD formats. Your application may require extra or more esoteric formats, in which case you can configure your node to support them using `options.ipld.formats` passed to the client or an in-process node or even a daemon if you start it with a wrapper.
+
+See the following files for different configuration:
+
+* [./in-process-node.js](./in-process-node.js) for running an in-process node as part of your confiugration
+* [./daemon-node.js](./daemon-node.js) for running a node as a separate daemon process

examples/custom-ipld-formats/daemon-node.js

@@ -0,0 +1,96 @@
+// ordinarily we'd open a PR against the multicodec module to get our
+// codec number added but since we're just testing we shim our new
+// codec into the base-table.json file - this has to be done
+// before requiring other modules as the int table will become read-only
+const codecName = 'dag-test'
+const codecNumber = 392091
+
+const baseTable = require('multicodec/src/base-table.json')
+baseTable[codecName] = codecNumber
+
+// now require modules as usual
+const IPFSDaemon = require('ipfs-cli/src/daemon')
+const multihashing = require('multihashing-async')
+const multihash = multihashing.multihash
+const multicodec = require('multicodec')
+const CID = require('cids')
+const ipfsHttpClient = require('ipfs-http-client')
+const uint8ArrayToString = require('uint8arrays/to-string')
+
+async function main () {
+  // see https://github.com/ipld/interface-ipld-format for the interface definition
+  const format = {
+    codec: codecNumber,
+    defaultHashAlg: multicodec.SHA2_256,
+    util: {
+      serialize (data) {
+        return Buffer.from(JSON.stringify(data))
+      },
+      deserialize (buf) {
+        return JSON.parse(uint8ArrayToString(buf))
+      },
+      async cid (buf) {
+        const multihash = await multihashing(buf, format.defaultHashAlg)
+
+        return new CID(1, format.codec, multihash)
+      }
+    },
+    resolver: {
+      resolve: (buf, path) => {
+        return {
+          value: format.util.deserialize(buf),
+          remainderPath: path
+        }
+      }
+    }
+  }
+
+  // start an IPFS Daemon
+  const daemon = new IPFSDaemon({
+    ipld: {
+      formats: [
+        format
+      ]
+    }
+  })
+  await daemon.start()
+
+  // in another process:
+  const client = ipfsHttpClient({
+    url: `http://localhost:${daemon._httpApi._apiServers[0].info.port}`,
+    ipld: {
+      formats: [
+        format
+      ]
+    }
+  })
+
+  const data = {
+    hello: 'world'
+  }
+
+  const cid = await client.dag.put(data, {
+    format: codecName,
+    hashAlg: multihash.codes[format.defaultHashAlg]
+  })
+
+  console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)
+
+  const {
+    value
+  } = await client.dag.get(cid)
+
+  console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`)
+
+  await daemon.stop()
+}
+
+main()
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(() => {
+    // https://github.com/libp2p/js-libp2p/issues/779
+    process.exit(0)
+  })

examples/custom-ipld-formats/in-process-node.js

@@ -0,0 +1,73 @@
+// ordinarily we'd open a PR against the multicodec module to get our
+// codec number added but since we're just testing we shim our new
+// codec into the base-table.json file - this has to be done
+// before requiring other modules as the int table will become read-only
+const codecName = 'dag-test'
+const codecNumber = 392091
+
+const baseTable = require('multicodec/src/base-table.json')
+baseTable[codecName] = codecNumber
+
+// now require modules as usual
+const IPFS = require('ipfs-core')
+const multihashing = require('multihashing-async')
+const multicodec = require('multicodec')
+const CID = require('cids')
+
+async function main () {
+  // see https://github.com/ipld/interface-ipld-format for the interface definition
+  const format = {
+    codec: codecNumber,
+    defaultHashAlg: multicodec.SHA2_256,
+    util: {
+      serialize (data) {
+        return Buffer.from(JSON.stringify(data))
+      },
+      deserialize (buf) {
+        return JSON.parse(buf.toString('utf8'))
+      },
+      async cid (buf) {
+        const multihash = await multihashing(buf, format.defaultHashAlg)
+
+        return new CID(1, format.codec, multihash)
+      }
+    }
+  }
+
+  const node = await IPFS.create({
+    ipld: {
+      formats: [
+        format
+      ]
+    }
+  })
+
+  const data = {
+    hello: 'world'
+  }
+
+  const cid = await node.dag.put(data, {
+    format: codecName,
+    hashAlg: format.defaultHashAlg
+  })
+
+  console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)
+
+  const {
+    value
+  } = await node.dag.get(cid)
+
+  console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`)
+
+  await node.stop()
+}
+
+main()
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(() => {
+    // https://github.com/libp2p/js-libp2p/issues/779
+    process.exit(0)
+  })

examples/custom-ipld-formats/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "example-custom-ipld-formats",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "test": "test-ipfs-example"
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "execa": "^4.0.3",
+    "test-ipfs-example": "^2.0.3"
+  },
+  "dependencies": {
+    "cids": "^1.0.0",
+    "ipfs-cli": "0.0.1",
+    "ipfs-core": "0.0.1",
+    "ipfs-http-client": "^47.0.0",
+    "multicodec": "^2.0.1",
+    "multihashing-async": "^2.0.1",
+    "uint8arrays": "^1.1.0"
+  }
+}

examples/custom-ipld-formats/test.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const path = require('path')
+const {
+  waitForOutput
+} = require('test-ipfs-example/utils')
+
+const testInProcessNode = async () => {
+  await waitForOutput(
+    'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' +
+    'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'in-process-node.js'))
+}
+
+const testDaemonNode = async () => {
+  await waitForOutput(
+    'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' +
+    'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'daemon-node.js'))
+}
+
+async function test () {
+  console.info('Testing in-process node')
+  await testInProcessNode()
+
+  console.info('Testing daemon node')
+  await testDaemonNode()
+}
+
+module.exports = test

packages/ipfs-cli/src/daemon.js

@@ -28,6 +28,11 @@ class Daemon {
     }
   }
 
+  /**
+   * Starts the IPFS HTTP server
+   *
+   * @returns {Promise<Daemon>}
+   */
   async start () {
     log('starting')
 

packages/ipfs-core/src/components/start.js

@@ -294,6 +294,7 @@ function createApi ({
     id: Components.id({ peerId, libp2p }),
     init: async () => { throw new AlreadyInitializedError() }, // eslint-disable-line require-await
     isOnline,
+    ipld,
     key: {
       export: Components.key.export({ keychain }),
       gen: Components.key.gen({ keychain }),

packages/ipfs-http-client/src/dag/get.js

@@ -1,40 +1,29 @@
 'use strict'
 
-const dagPB = require('ipld-dag-pb')
-const dagCBOR = require('ipld-dag-cbor')
-const raw = require('ipld-raw')
 const configure = require('../lib/configure')
+const multicodec = require('multicodec')
+const loadFormat = require('../lib/ipld-formats')
 
-const resolvers = {
-  'dag-cbor': dagCBOR.resolver,
-  'dag-pb': dagPB.resolver,
-  raw: raw.resolver
-}
-
-module.exports = configure((api, options) => {
-  const getBlock = require('../block/get')(options)
-  const dagResolve = require('./resolve')(options)
+module.exports = configure((api, opts) => {
+  const getBlock = require('../block/get')(opts)
+  const dagResolve = require('./resolve')(opts)
+  const load = loadFormat(opts.ipld)
 
   /**
    * @type {import('..').Implements<import('ipfs-core/src/components/dag/get')>}
    */
   const get = async (cid, options = {}) => {
     const resolved = await dagResolve(cid, options)
     const block = await getBlock(resolved.cid, options)
-    const dagResolver = resolvers[resolved.cid.codec]
 
-    if (!dagResolver) {
-      throw Object.assign(
-        new Error(`Missing IPLD format "${resolved.cid.codec}"`),
-        { missingMulticodec: resolved.cid.codec }
-      )
-    }
+    const codecName = multicodec.getName(resolved.cid.code)
+    const format = await load(codecName)
 
-    if (resolved.cid.codec === 'raw' && !resolved.remainderPath) {
+    if (resolved.cid.code === multicodec.RAW && !resolved.remainderPath) {
       resolved.remainderPath = '/'
     }
 
-    return dagResolver.resolve(block.data, resolved.remainderPath)
+    return format.resolver.resolve(block.data, resolved.remainderPath)
   }
 
   return get

packages/ipfs-http-client/src/dag/put.js

@@ -1,8 +1,5 @@
 'use strict'
 
-const dagCBOR = require('ipld-dag-cbor')
-const dagPB = require('ipld-dag-pb')
-const ipldRaw = require('ipld-raw')
 const CID = require('cids')
 const multihash = require('multihashes')
 const configure = require('../lib/configure')
@@ -11,19 +8,10 @@ const toUrlSearchParams = require('../lib/to-url-search-params')
 const { anySignal } = require('any-signal')
 const AbortController = require('native-abort-controller')
 const multicodec = require('multicodec')
+const loadFormat = require('../lib/ipld-formats')
 
 module.exports = configure((api, opts) => {
-  const formats = {
-    [multicodec.DAG_PB]: dagPB,
-    [multicodec.DAG_CBOR]: dagCBOR,
-    [multicodec.RAW]: ipldRaw
-  }
-
-  const ipldOptions = (opts && opts.ipld) || {}
-  const configuredFormats = (ipldOptions && ipldOptions.formats) || []
-  configuredFormats.forEach(format => {
-    formats[format.codec] = format
-  })
+  const load = loadFormat(opts.ipld)
 
   /**
    * @type {import('..').Implements<import('ipfs-core/src/components/dag/put')>}
@@ -39,7 +27,7 @@ module.exports = configure((api, opts) => {
       const cid = new CID(options.cid)
       options = {
         ...options,
-        format: cid.codec,
+        format: multicodec.getName(cid.code),
         hashAlg: multihash.decode(cid.multihash).name
       }
       delete options.cid
@@ -52,24 +40,7 @@ module.exports = configure((api, opts) => {
       ...options
     }
 
-    const number = multicodec.getNumber(settings.format)
-    let format = formats[number]
-
-    if (!format) {
-      if (opts && opts.ipld && opts.ipld.loadFormat) {
-        // @ts-ignore - loadFormat expect string but it could be a number
-        format = await opts.ipld.loadFormat(settings.format)
-      }
-
-      if (!format) {
-        throw new Error('Format unsupported - please add support using the options.ipld.formats or options.ipld.loadFormat options')
-      }
-    }
-
-    if (!format.util || !format.util.serialize) {
-      throw new Error('Format does not support utils.serialize function')
-    }
-
+    const format = await load(settings.format)
     const serialized = format.util.serialize(dagNode)
 
     // allow aborting requests on body errors