chore: declare interface types in .d.ts file (#122)

If we don't declare ts interfaces in a .d.ts file they get duplicated in our generated .d.ts files.

We should pull in types from `ipfs-core-types` for the block api but it's yet to be fully typed so this is draft PR that can be finished after the IPFS top level types have improved.

- Moves all types into `.d.ts` files so they do not get duplicated in generated types
- Switches to named exports
- Loosens mtime definition for unixfs
- Exports utility functions for converting mtimes and modes
- Runs ts check during linting

BREAKING CHANGE: switches to named exports

Author: achingbrain

.travis.yml

@@ -46,7 +46,6 @@ jobs:
     - stage: check
       name: linting
       script:
-        - npm --version
         - npm run lint
 
     - stage: check

package.json

@@ -3,7 +3,6 @@
   "version": "1.0.0",
   "description": "JS implementation of the IPFS UnixFS",
   "scripts": {
-    "postinstall": "rm -rf package-lock.json packages/*/package-lock.json",
     "reset": "lerna run clean && rimraf packages/*/node_modules node_modules",
     "test": "lerna run test",
     "coverage": "lerna run coverage",
@@ -17,7 +16,6 @@
     "update-contributors": "aegir release --lint=false --test=false --bump=false --build=false --changelog=false --commit=false --tag=false --push=false --ghrelease=false --docs=false --publish=false"
   },
   "devDependencies": {
-    "aegir": "^30.3.0",
     "lerna": "^3.22.1"
   },
   "repository": {

packages/ipfs-unixfs-exporter/.aegir.js

@@ -1,16 +1,35 @@
 'use strict'
 
+/** @type {import('aegir').Options["build"]["config"]} */
+const buildConfig = {
+  plugins: [
+    {
+      name: 'node built ins',
+      setup (build) {
+        build.onResolve({ filter: /^stream$/ }, () => {
+          return { path: require.resolve('readable-stream') }
+        })
+        build.onResolve({ filter: /^crypto$/ }, () => {
+          return { path: require.resolve('crypto-browserify') }
+        })
+        build.onResolve({ filter: /^cborg$/ }, () => {
+          return { path: require.resolve('cborg') }
+        })
+      }
+    }
+  ]
+}
+
+/** @type {import('aegir').PartialOptions} */
 module.exports = {
-  karma: {
-    browserNoActivityTimeout: 1000 * 1000,
+  build: {
+    config: buildConfig
   },
-  webpack: {
-    node: {
-      // needed by the cbor module
-      stream: true,
-
-      // needed by the core-util-is module
-      Buffer: true
+  test: {
+    browser: {
+      config: {
+        buildConfig
+      }
     }
   }
 }

packages/ipfs-unixfs-exporter/.npmignore

@@ -26,8 +26,8 @@
       - [Raw entries](#raw-entries)
       - [CBOR entries](#cbor-entries)
       - [`entry.content({ offset, length })`](#entrycontent-offset-length-)
-    - [`exporter.path(cid, ipld)`](#exporterpathcid-ipld)
-    - [`exporter.recursive(cid, ipld)`](#exporterrecursivecid-ipld)
+    - [`walkPath(cid, ipld)`](#walkpathcid-ipld)
+    - [`recursive(cid, ipld)`](#recursivecid-ipld)
   - [Contribute](#contribute)
   - [License](#license)
 
@@ -43,8 +43,8 @@
 
 ```js
 // import a file and export it again
-const importer = require('ipfs-unixfs-importer')
-const exporter = require('ipfs-unixfs-exporter')
+const { importer } = require('ipfs-unixfs-importer')
+const { exporter } = require('ipfs-unixfs-exporter')
 
 const files = []
 
@@ -80,7 +80,7 @@ console.info(bytes) // 0, 1, 2, 3
 #### API
 
 ```js
-const exporter = require('ipfs-unixfs-exporter')
+const { exporter } = require('ipfs-unixfs-exporter')
 ```
 
 ### `exporter(cid, ipld, options)`
@@ -202,28 +202,32 @@ for await (const entry of dir.content({
 // `entries` contains the first 5 files/directories in the directory
 ```
 
-### `exporter.path(cid, ipld)`
+### `walkPath(cid, ipld)`
 
-`exporter.path` will return an async iterator that yields entries for all segments in a path:
+`walkPath` will return an async iterator that yields entries for all segments in a path:
 
 ```javascript
+const { walkPath } = require('ipfs-unixfs-exporter')
+
 const entries = []
 
-for await (const entry of exporter.path('Qmfoo/foo/bar/baz.txt', ipld)) {
+for await (const entry of walkPath('Qmfoo/foo/bar/baz.txt', ipld)) {
   entries.push(entry)
 }
 
 // entries contains 4x `entry` objects
 ```
 
-### `exporter.recursive(cid, ipld)`
+### `recursive(cid, ipld)`
 
-`exporter.recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory.
+`recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory.
 
 ```javascript
+const { recursive } = require('ipfs-unixfs-exporter')
+
 const entries = []
 
-for await (const child of exporter.recursive('Qmfoo/foo/bar', ipld)) {
+for await (const child of recursive('Qmfoo/foo/bar', ipld)) {
   entries.push(entry)
 }
 

packages/ipfs-unixfs-exporter/README.md

@@ -10,10 +10,11 @@
   "scripts": {
     "prepare": "aegir build --no-bundle",
     "test": "aegir test",
+    "build": "aegir build",
     "clean": "rimraf ./dist",
-    "lint": "aegir lint",
+    "lint": "aegir ts --check && aegir lint",
     "coverage": "nyc -s npm run test -t node && nyc report --reporter=html",
-    "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types"
+    "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types -i copy -i util -i crypto-browserify -i events -i readable-stream"
   },
   "repository": {
     "type": "git",
@@ -35,23 +36,29 @@
     "@types/mocha": "^8.2.1",
     "@types/sinon": "^9.0.10",
     "abort-controller": "^3.0.0",
-    "aegir": "^30.3.0",
+    "aegir": "^32.0.0",
+    "copy": "^0.3.2",
+    "crypto-browserify": "^3.12.0",
     "detect-node": "^2.0.4",
-    "ipfs-core-types": "^0.3.0",
+    "events": "^3.3.0",
+    "ipfs-core-types": "^0.3.1",
     "ipfs-unixfs-importer": "^6.0.1",
-    "ipld": "^0.28.0",
-    "ipld-dag-pb": "^0.21.0",
-    "ipld-in-memory": "^7.0.0",
+    "ipld": "^0.29.0",
+    "ipld-block": "^0.11.1",
+    "ipld-dag-pb": "^0.22.1",
+    "ipld-in-memory": "^8.0.0",
     "it-all": "^1.0.5",
     "it-buffer-stream": "^2.0.0",
     "it-first": "^1.0.6",
     "merge-options": "^3.0.4",
-    "multicodec": "^2.1.0",
+    "multicodec": "^3.0.1",
     "native-abort-controller": "^1.0.3",
     "nyc": "^15.0.0",
+    "readable-stream": "^3.6.0",
     "rimraf": "^3.0.2",
     "sinon": "^9.2.4",
-    "uint8arrays": "^2.1.2"
+    "uint8arrays": "^2.1.2",
+    "util": "^0.12.3"
   },
   "dependencies": {
     "cids": "^1.1.5",

packages/ipfs-unixfs-exporter/package.json

@@ -6,66 +6,16 @@ const resolve = require('./resolvers')
 const last = require('it-last')
 
 /**
- * @typedef {import('ipfs-unixfs')} UnixFS
+ * @typedef {import('ipfs-unixfs').UnixFS} UnixFS
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
- * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
- *
- * @typedef {object} UnixFSFile
- * @property {'file'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {UnixFS} unixfs
- * @property {DAGNode} node
- * @property {(options?: ExporterOptions) => AsyncIterable<Uint8Array>} content
- *
- * @typedef {object} UnixFSDirectory
- * @property {'directory'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {UnixFS} unixfs
- * @property {DAGNode} node
- * @property {(options?: ExporterOptions) => AsyncIterable<UnixFSEntry>} content
- *
- * @typedef {object} ObjectNode
- * @property {'object'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {Uint8Array} node
- * @property {(options?: ExporterOptions) => AsyncIterable<any>} content
- *
- * @typedef {object} RawNode
- * @property {'raw'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {Uint8Array} node
- * @property {(options?: ExporterOptions) => AsyncIterable<Uint8Array>} content
- *
- * @typedef {object} IdentityNode
- * @property {'identity'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {Uint8Array} node
- * @property {(options?: ExporterOptions) => AsyncIterable<Uint8Array>} content
- *
- * @typedef {UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode} UnixFSEntry
- */
-
-/**
- * @typedef {object} ExporterOptions
- * @property {number} [offset=0]
- * @property {number} [length]
- * @property {AbortSignal} [signal]
- * @property {number} [timeout]
+ * @typedef {import('ipld')} IPLD
+ * @typedef {import('./types').ExporterOptions} ExporterOptions
+ * @typedef {import('./types').UnixFSFile} UnixFSFile
+ * @typedef {import('./types').UnixFSDirectory} UnixFSDirectory
+ * @typedef {import('./types').ObjectNode} ObjectNode
+ * @typedef {import('./types').RawNode} RawNode
+ * @typedef {import('./types').IdentityNode} IdentityNode
+ * @typedef {import('./types').UnixFSEntry} UnixFSEntry
  */
 
 const toPathComponents = (path = '') => {
@@ -115,7 +65,7 @@ const cidAndRest = (path) => {
  * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
-const walkPath = async function * (path, ipld, options = {}) {
+async function * walkPath (path, ipld, options = {}) {
   let {
     cid,
     toResolve
@@ -152,7 +102,7 @@ const walkPath = async function * (path, ipld, options = {}) {
  * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
-const exporter = async (path, ipld, options = {}) => {
+async function exporter (path, ipld, options = {}) {
   const result = await last(walkPath(path, ipld, options))
 
   if (!result) {
@@ -167,7 +117,7 @@ const exporter = async (path, ipld, options = {}) => {
  * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
-const recursive = async function * (path, ipld, options = {}) {
+async function * recursive (path, ipld, options = {}) {
   const node = await exporter(path, ipld, options)
 
   if (!node) {
@@ -202,7 +152,8 @@ const recursive = async function * (path, ipld, options = {}) {
   }
 }
 
-exporter.path = walkPath
-exporter.recursive = recursive
-
-module.exports = exporter
+module.exports = {
+  exporter,
+  walkPath,
+  recursive
+}

packages/ipfs-unixfs-exporter/src/index.js

@@ -4,7 +4,11 @@ const CID = require('cids')
 const errCode = require('err-code')
 
 /**
- * @type {import('./').Resolver}
+ * @typedef {import('../types').Resolver} Resolver
+ */
+
+/**
+ * @type {Resolver}
  */
 const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
   const object = await ipld.get(cid, options)
@@ -29,6 +33,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
             cid,
             node: block,
             depth,
+            size: block.length,
             content: async function * () {
               yield object
             }
@@ -57,6 +62,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       cid,
       node: block,
       depth,
+      size: block.length,
       content: async function * () {
         yield object
       }

packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

@@ -6,7 +6,8 @@ const validateOffsetAndLength = require('../utils/validate-offset-and-length')
 const mh = require('multihashing-async').multihash
 
 /**
- * @typedef {import('../').ExporterOptions} ExporterOptions
+ * @typedef {import('../types').ExporterOptions} ExporterOptions
+ * @typedef {import('../types').Resolver} Resolver
  */
 
 /**
@@ -29,7 +30,7 @@ const rawContent = (node) => {
 }
 
 /**
- * @type {import('./').Resolver}
+ * @type {Resolver}
  */
 const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
   if (toResolve.length) {
@@ -46,6 +47,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       cid,
       content: rawContent(buf.digest),
       depth,
+      size: buf.length,
       node: buf.digest
     }
   }