refactor: switch tar-stream for it-tar (#1161)

This switches out `tar-stream` for [`it-tar`](https://www.npmjs.com/package/it-tar), which is a fork that uses pure async iterables. This removes `readable-stream` from this branch of the tree (which will allow us to drop it entirely in the future) and allows us to drop the `event-iterator` dependency.

Author: Alan Shaw

package.json

@@ -57,7 +57,6 @@
     "detect-node": "^2.0.4",
     "end-of-stream": "^1.4.1",
     "err-code": "^2.0.0",
-    "event-iterator": "^1.2.0",
     "explain-error": "^1.0.4",
     "flatmap": "0.0.3",
     "form-data": "^3.0.0",
@@ -74,6 +73,7 @@
     "iso-stream-http": "~0.1.2",
     "iso-url": "~0.4.6",
     "it-glob": "0.0.6",
+    "it-tar": "^1.1.0",
     "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "just-kebab-case": "^1.1.0",
@@ -101,7 +101,6 @@
     "qs": "^6.5.2",
     "readable-stream": "^3.1.1",
     "stream-to-pull-stream": "^1.7.2",
-    "tar-stream": "^2.0.1",
     "through2": "^3.0.1"
   },
   "devDependencies": {

src/get.js

@@ -1,8 +1,9 @@
 'use strict'
 
 const configure = require('./lib/configure')
-const tarStreamToObjects = require('./utils/tar-stream-to-objects')
+const Tar = require('it-tar')
 const IsIpfs = require('is-ipfs')
+const toIterable = require('./lib/stream-to-iterable')
 const cleanCID = require('./utils/clean-cid')
 
 module.exports = configure(({ ky }) => {
@@ -43,6 +44,19 @@ module.exports = configure(({ ky }) => {
       searchParams
     })
 
-    yield * tarStreamToObjects(res.body)
+    const extractor = Tar.extract()
+
+    for await (const { header, body } of extractor(toIterable(res.body))) {
+      if (header.type === 'directory') {
+        yield {
+          path: header.name
+        }
+      } else {
+        yield {
+          path: header.name,
+          content: body
+        }
+      }
+    }
   }
 })

src/utils/load-commands.js

@@ -7,6 +7,8 @@ const { concatify, collectify, pullify, streamify } = require('../lib/converters
 const toPullStream = require('async-iterator-to-pull-stream')
 const pull = require('pull-stream/pull')
 const map = require('pull-stream/throughs/map')
+const toStream = require('it-to-stream')
+const BufferList = require('bl/BufferList')
 
 function requireCommands (send, config) {
   const add = require('../add')(config)
@@ -58,21 +60,37 @@ function requireCommands (send, config) {
 
       for await (const entry of get(path, options)) {
         if (entry.content) {
-          entry.content = Buffer.concat(await all(entry.content))
+          entry.content = new BufferList(await all(entry.content)).slice()
         }
 
         output.push(entry)
       }
 
       return output
     }),
-    getReadableStream: streamify.readable(get),
+    getReadableStream: streamify.readable((path, options) => (async function * () {
+      for await (const file of get(path, options)) {
+        if (file.content) {
+          const { content } = file
+          file.content = toStream((async function * () {
+            for await (const chunk of content) {
+              yield chunk.slice() // Convert bl to Buffer
+            }
+          })())
+        }
+
+        yield file
+      }
+    })()),
     getPullStream: (path, options) => {
       return pull(
         toPullStream(get(path, options)),
         map(file => {
           if (file.content) {
-            file.content = toPullStream(file.content)
+            file.content = pull(
+              toPullStream(file.content),
+              map(chunk => chunk.slice()) // Convert bl to Buffer
+            )
           }
 
           return file