perf: do not list directory contents when statting files

Restrict how deeply directory trees are listed

License: MIT
Signed-off-by: achingbrain <[email protected]>

Author: achingbrain

src/cli/read.js

@@ -2,7 +2,7 @@
 
 const pull = require('pull-stream/pull')
 const through = require('pull-stream/throughs/through')
-const collect = require('pull-stream/sinks/collect')
+const onEnd = require('pull-stream/sinks/on-end')
 const {
   print
 } = require('./utils')
@@ -43,7 +43,7 @@ module.exports = {
           through(buffer => {
             print(buffer, false)
           }),
-          collect((error) => {
+          onEnd((error) => {
             if (error) {
               return reject(error)
             }

src/core/stat.js

@@ -10,7 +10,6 @@ const waterfall = require('async/waterfall')
 const pull = require('pull-stream/pull')
 const collect = require('pull-stream/sinks/collect')
 const asyncMap = require('pull-stream/throughs/async-map')
-const filter = require('pull-stream/throughs/filter')
 const exporter = require('ipfs-unixfs-exporter')
 const log = require('debug')('ipfs:mfs:stat')
 
@@ -37,12 +36,9 @@ module.exports = (context) => {
       ({ mfsPath, depth }, cb) => {
         pull(
           exporter(mfsPath, context.ipld, {
-            maxDepth: depth + 1,
-            fullPath: true
+            maxDepth: depth
           }),
-          filter(node => node.depth === depth),
 
-          // load DAGNodes for each file
           asyncMap((file, cb) => {
             loadNode(context, {
               cid: file.hash

src/core/utils/add-link.js

@@ -72,7 +72,7 @@ const addLink = (context, options, callback) => {
     return addToShardedDirectory(context, options, callback)
   }
 
-  if (options.parent.links.length === options.shardSplitThreshold) {
+  if (options.parent.links.length >= options.shardSplitThreshold) {
     log('Converting directory to sharded directory')
 
     return convertToShardedDirectory(context, options, callback)
@@ -151,8 +151,21 @@ const addToShardedDirectory = async (context, options, callback) => {
     })
   }
 
+  const existingFile = options.parent.links
+    .filter(link => link.name.substring(2) === options.name)
+    .pop()
+
+  if (existingFile) {
+    log(`Updating file ${existingFile.name}`)
+
+    return addToDirectory(context, {
+      ...options,
+      name: existingFile.name
+    }, callback)
+  }
+
   const existingUnshardedFile = options.parent.links
-    .filter(link => link.name.substring(0, 2) === prefix || link.name.substring(2) === options.name)
+    .filter(link => link.name.substring(0, 2) === prefix)
     .pop()
 
   if (existingUnshardedFile) {
@@ -194,7 +207,7 @@ const addToShardedDirectory = async (context, options, callback) => {
     })
   }
 
-  log(`Updating or appending ${prefix + options.name} to shard`)
+  log(`Appending ${prefix + options.name} to shard`)
 
   return addToDirectory(context, {
     ...options,

src/core/utils/constants.js

@@ -4,7 +4,8 @@ const Key = require('interface-datastore').Key
 
 const FILE_TYPES = {
   file: 0,
-  directory: 1
+  directory: 1,
+  'hamt-sharded-directory': 1
 }
 
 module.exports = {