feat: lazy load js-ipfs-http-client and async iterator support (#8)

BREAKING CHANGE: this switches ipfs-provider to the new JS APIs based on
async iterators. More details in: https://blog.ipfs.io/2020-02-01-async-await-refactor

README.md

@@ -5,8 +5,9 @@
 [![Build Status](https://flat.badgen.net/travis/ipfs-shipyard/ipfs-provider)](https://travis-ci.com/ipfs-shipyard/ipfs-provider)
 [![Dependency Status](https://david-dm.org/ipfs-shipyard/ipfs-provider.svg?style=flat-square)](https://david-dm.org/ipfs-shipyard/ipfs-provider)
 
-> This module tries to connect to IPFS via multiple [providers](#providers).  
-> It is a general-purpose replacement for [ipfs-redux-bundle](https://github.com/ipfs-shipyard/ipfs-redux-bundle).
+> Returns IPFS API object by trying multiple [providers](#providers) in a custom fallback order.
+>
+> This is a general-purpose replacement for [ipfs-redux-bundle](https://github.com/ipfs-shipyard/ipfs-redux-bundle).
 
 - [Install](#install)
 - [Usage](#usage)
@@ -30,20 +31,55 @@ $ npm install ipfs-provider
 
 ```js
 const { getIpfs, providers } = require('ipfs-provider')
-const { httpClient, jsIpfs, windowIpfs, webExt } = providers
+const { httpClient, jsIpfs, windowIpfs } = providers
 
-const { ipfs, provider } = await getIpfs({
-  providers: [ // these are the defaults (the order matters)
-    windowIpfs(),
-    httpClient()
-    // disabled by default: jsIpfs(), webExt()
+const { ipfs, provider, apiAddress } = await getIpfs({
+  // when httpClient provider is used multiple times
+  // define its constructor once, at the top level
+  loadHttpClientModule: () => require('ipfs-http-client'),
+
+  // note this is an array, providers are tried in order:
+  providers: [
+
+    // (1) try window.ipfs (experimental, but some browsers expose it),
+    windowIpfs({
+      // request specific permissions upfront (optional)
+      permissions: { commands: ['files.add', 'files.cat'] }
+    }),
+
+    // (2) try various HTTP endpoints (best-effort),
+    httpClient({
+      // (2.1) try multiaddr of a local node
+      apiAddress: '/ip4/127.0.0.1/tcp/5001'
+    }),
+    httpClient(), // (2.2) try "/api/v0/" on the same Origin as the page
+    httpClient({
+      // (2.3) try arbitrary API URL
+      apiAddress: 'https://some.example.com:8080'
+    }),
+
+    // (3) final fallback to spawning embedded js-ipfs running in-page
+    jsIpfs({
+      // js-ipfs package is used only once, as a last resort
+      loadJsIpfsModule: () => require('ipfs'),
+      options: { } // pass config: https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs/docs/MODULE.md#ipfscreateoptions
+    })
   ]
 })
+
+for await (const file of ipfs.add("text")) {
+  if (file && file.cid) {
+    console.log(`successfully stored at ${file.cid}`)
+  } else {
+    console.error('unable to ipfs.add', file)
+  }
+}
 ```
 
-- `ipfs` – returned IPFS API instance
-- `provider` – a string with a name of the first successful provider.  
+- `ipfs` – returned instance of IPFS API (see [SPEC](https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/README.md))
+- `provider` – a string with a name of the first successful provider.
    - built-in names match constants from `providers`: `httpClient`, `jsIpfs`, `windowIpfs` and `webExt`.
+- `apiAddress` – returned only when `httpClient` provider is used, provides information which HTTP endpoint succeded
 
 
 ## Examples
@@ -62,77 +98,81 @@ const { getIpfs, providers } = require('ipfs-provider')
 const { httpClient, windowIpfs } = providers
 
 const { ipfs, provider } = await getIpfs({
-  // These are the defaults:
   providers: [
     httpClient(),
     windowIpfs()
   ]
 })
 ```
 
-#### Global options
-
-There are options that can be passed to each provider and global ones. However, you can always override the global ones by passing the same one for a provider. Here is the list of global options:
+#### Customizing connection test
 
 ```js
 const { ipfs, provider } = await getIpfs({
-  providers: [ /* ... */ ],
-  connectionTest: () => { /* function to test the connection to IPFS */ }
+  providers: [ /* array of providers to try in order */ ],
+  connectionTest: () => { /* boolean function to test the connection to IPFS, default one tries to ipfs.get the CID of an empty directory */ },
 })
 ```
 
-Please keep in mind that all of these have defaults and you **do not** need to specify them.
-
 ### `httpClient`
 
-Tries to connect to HTTP API via [`js-ipfs-http-client`](https://github.com/ipfs/js-ipfs-http-client) with either a user provided `apiAddress`, the current origin, or `defaultApiAddress`.
-
-Value provided in `apiAddress`  can be:
-- a multiaddr (string like `/ip4/127.0.0.1/tcp/5001` or an [object](https://github.com/multiformats/js-multiaddr/))
-- a String with an URL (`https://example.com:8080/`)
-- a configuration object supported by [`js-ipfs-http-client`](https://github.com/ipfs/js-ipfs-http-client#importing-the-module-and-usage)
-
+Tries to connect to HTTP API via [`js-ipfs-http-client`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client):
 
 ```js
 const { ipfs, provider } = await getIpfs({
   providers: [
     httpClient({
-      // defaults
-      defaultApiAddress: '/ip4/127.0.0.1/tcp/5001',
-      apiAddress: null
+      loadHttpClientModule: () => require('ipfs-http-client'),
+      apiAddress: 'https://api.example.com:8080/'
     })
   ]
 })
 ```
 
-To try multiple endpoints, simply use this provider multiple times.  
-See [`examples/browser-browserify/src/index.js`](./examples/browser-browserify/src/index.js) for real world example.
+This provider will attempt to establish connection with (in order):
+1. `apiAddress` (if provided)
+2. `/api/` at the current Origin
+3. the default local API (`/ip4/127.0.0.1/tcp/5001`)
+
+It supports lazy-loading and small bundle sizes. The client library is initialized using constructor (in order):
+1. one returned by `loadHttpClientModule` async function (if provided)
+2. one exposed at `window.IpfsHttpClient` (if present)
+
+Value passed in `apiAddress` can be:
+- a multiaddr (string like `/ip4/127.0.0.1/tcp/5001` or an [object](https://github.com/multiformats/js-multiaddr/))
+- a String with an URL (`https://api.example.com:8080/`)
+- a configuration object supported by the [constructor](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client#importing-the-module-and-usage)
+  (`{ host: '1.1.1.1', port: '80', apiPath: '/ipfs/api/v0' }`)
+
 
+To try multiple endpoints, simply use this provider multiple times.
+See [`examples/browser-browserify/src/index.js`](./examples/browser-browserify/src/index.js) for real world example.
 
 ### `jsIpfs`
 
-Spawns embedded [`js-ipfs`](https://github.com/ipfs/js-ipfs) (full node in JavaScript)
+Spawns embedded [`js-ipfs`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs) (a full IPFS node in JavaScript)
 in the context of the current page using customizable constructor:
 
 ```js
 const { ipfs, provider } = await getIpfs({
   providers: [
     jsIpfs({
-      // defaults
-      getConstructor: () => import('ipfs'),
+      loadJsIpfsModule: () => require('ipfs'),
       options: { /* advanced config */ }
     })
   ]
 })
 ```
 
-- `getConstructor` should be a function that returns a promise that resolves with a `JsIpfs` constructor.  
+- `loadJsIpfsModule` should be a function that returns a promise that resolves to a js-ipfs constructor
+   <!-- TODO confirm below is true, if it is, add example to examples/ and link to it
    This works well with [dynamic `import()`](https://developers.google.com/web/updates/2017/11/dynamic-import), so you can lazily load js-ipfs when it is needed.
+   -->
 - `options` should be an object which specifies [advanced configurations](https://github.com/ipfs/js-ipfs#ipfs-constructor) to the node.
 
 ### `windowIpfs`
 
-[`window.ipfs`](https://github.com/ipfs-shipyard/ipfs-companion/blob/master/docs/window.ipfs.md) is created by [ipfs-companion](https://github.com/ipfs/ipfs-companion) browser extension.  
+[`window.ipfs`](https://github.com/ipfs-shipyard/ipfs-companion/blob/master/docs/window.ipfs.md) is created by [ipfs-companion](https://github.com/ipfs/ipfs-companion) browser extension.
 It supports passing an optional list of permissions to [display a single ACL prompt](https://github.com/ipfs-shipyard/ipfs-companion/blob/master/docs/window.ipfs.md#do-i-need-to-confirm-every-api-call) the first time it is used:
 
 ```js
@@ -148,7 +188,7 @@ const { ipfs, provider } = await getIpfs({
 
 ### `webExt`
 
-`webExt` looks for an instance in the [background page of a WebExtension](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/extension/getBackgroundPage)  
+`webExt` looks for an instance in the [background page of a WebExtension](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/extension/getBackgroundPage)
 (useful only in browser extensions, not regular pages, disabled by default)
 
 ```js

examples/README.md

@@ -14,5 +14,5 @@ Let us know if you find any issue or if you want to contribute and add an exampl
 
 ## See also
 
-- [`js-ipfs-http-client` examples](https://github.com/ipfs/js-ipfs-http-client/tree/master/examples)
+- [`js-ipfs-http-client` examples](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client/examples)
 - [`js-ipfs` examples and tutorials](https://github.com/ipfs/js-ipfs/tree/master/examples)

examples/browser-browserify/README.md

@@ -27,4 +27,4 @@ Let's unpack what happened in the above example:
 2. 🔴 test request to local API (`/ip4/127.0.0.1/tcp/5001`) was blocked due to [CORS protection](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS)
 3. 🔴 `/api/v0/` on the same Origin as the page did not exist
 3. 🔴 explicitly defined remote API was offline (`http://dev.local:8080`)
-4. 💚 final fallback of spawning embedded [js-ipfs](https://github.com/ipfs/js-ipfs) was executed successfully 🚀✨
+4. 💚 final fallback worked: spawning embedded [js-ipfs](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs) was executed successfully 🚀✨

examples/browser-browserify/package.json

@@ -12,14 +12,16 @@
   "keywords": [],
   "license": "MIT",
   "devDependencies": {
-    "browserify": "^16.2.3",
+    "browserify": "^16.5.1",
     "concat-stream": "^2.0.0",
     "execa": "^4.0.0",
-    "http-server": "~0.12.0",
+    "http-server": "~0.12.1",
     "ipfs-provider": "file:../../"
   },
   "dependencies": {
-    "ipfs": "0.40.0"
+    "buffer": "5.6.0",
+    "ipfs": "0.43.0",
+    "ipfs-http-client": "44.0.0"
   },
   "browser": {
     "ipfs": "ipfs/dist"

examples/browser-browserify/public/index.html

@@ -15,11 +15,12 @@
   </head>
   <body>
     <h2>Add data to IPFS from the browser via API found by ipfs-provider</h2>
+    <p>(open the Console to see what is happening)</p>
     <textarea id="source" placeholder="Enter some text here"></textarea>
     <button id="store">Add text</button>
     <div id="output" style="display: none">
-      <p>CID and text read back from IPFS:</p>
-      <div class="content" id="hash">[ipfs hash]</div>
+      <p>CID and the text read back from IPFS:</p>
+      <div class="content" id="cid">[ipfs cid]</div>
       <div class="content" id="content">[ipfs content]</div>
     </div>
   </body>

examples/browser-browserify/src/index.js

@@ -1,10 +1,14 @@
 'use strict'
 
+const { Buffer } = require('buffer')
 const { getIpfs, providers } = require('ipfs-provider')
 const { httpClient, jsIpfs, windowIpfs } = providers
 
 document.addEventListener('DOMContentLoaded', async () => {
-  const { ipfs, provider } = await getIpfs({
+  const { ipfs, provider, apiAddress } = await getIpfs({
+    // HTTP client library can be defined globally to keep code minimal
+    // when httpClient provider is used multiple times
+    loadHttpClientModule: () => require('ipfs-http-client'),
     // try window.ipfs (if present),
     // then http apis (if up),
     // and finally fallback to spawning embedded js-ipfs
@@ -22,28 +26,37 @@ document.addEventListener('DOMContentLoaded', async () => {
         apiAddress: 'http://dev.local:8080'
       }),
       jsIpfs({
-        getConstructor: () => require('ipfs'), // note: 'require' can be used instead of 'import'
-        options: { } // pass config: https://github.com/ipfs/js-ipfs#ipfs-constructor
+        // js-ipfs package is used only once, here
+        loadJsIpfsModule: () => require('ipfs'), // note require instead of
+        options: { } // pass config: https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs/docs/MODULE.md#ipfscreateoptions
       })
     ]
   })
 
   console.log('IPFS API is provided by: ' + provider)
+  if (provider === 'httpClient') {
+    console.log('HTTP API address: ' + apiAddress)
+  }
 
   async function store () {
     const toStore = document.getElementById('source').value
-    const result = await ipfs.add(toStore)
-    for (const file of result) {
-      if (file && file.hash) {
-        console.log('successfully stored', file.hash)
-        await display(file.hash)
+    for await (const file of ipfs.add(toStore)) {
+      if (file && file.cid) {
+        console.log('successfully stored', file)
+        await display(file.cid.toString())
+      } else {
+        console.error('unable to add', file)
       }
     }
   }
 
-  async function display (hash) {
-    const data = await ipfs.cat(hash)
-    document.getElementById('hash').innerText = hash
+  async function display (cid) {
+    const chunks = []
+    for await (const chunk of ipfs.cat(cid)) {
+      chunks.push(chunk)
+    }
+    const data = Buffer.concat(chunks).toString()
+    document.getElementById('cid').innerText = cid
     document.getElementById('content').innerText = data
     document.getElementById('output').setAttribute('style', 'display: block')
   }