From 51ba1e2de961619a5f9eafbba62bb867549215d0 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Thu, 6 May 2021 19:49:14 -0300
Subject: [PATCH 1/9] feat: switch to HAMT based on size

---
 io/directory.go      | 129 +++++++++++++++++++++++++++++++------------
 io/directory_test.go | 104 +++++++++++++++++++++++++++++++---
 unixfs.go            |   5 ++
 3 files changed, 196 insertions(+), 42 deletions(-)

diff --git a/io/directory.go b/io/directory.go
index 03b84b98f..e5fbeeaac 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -14,9 +14,13 @@ import (
 	ipld "github.com/ipfs/go-ipld-format"
 )
 
-// UseHAMTSharding is a global flag that signifies whether or not to use the
-// HAMT sharding scheme for directory creation
-var UseHAMTSharding = false
+// UseHAMTSharding is a global option that allows switching to a HAMTDirectory
+// when the BasicDirectory grows above the size (in bytes) signalled by this
+// flag. The default size of 0 disables the option.
+// The size is not the *exact* block size of the encoded BasicDirectory but just
+// the estimated size based byte length of links name and CID (BasicDirectory's
+// ProtoNode doesn't use the Data field so this estimate is pretty accurate).
+var HAMTShardingSize = 0
 
 // DefaultShardWidth is the default value used for hamt sharding width.
 var DefaultShardWidth = 256
@@ -72,6 +76,12 @@ type Directory interface {
 type BasicDirectory struct {
 	node  *mdag.ProtoNode
 	dserv ipld.DAGService
+
+	// Internal variable used to cache the estimated size used for the
+	// HAMTShardingSize option. We maintain this value even if the
+	// HAMTShardingSize is off since potentially the option could be activated
+	// on the fly.
+	estimatedSize int
 }
 
 // HAMTDirectory is the HAMT implementation of `Directory`.
@@ -81,26 +91,29 @@ type HAMTDirectory struct {
 	dserv ipld.DAGService
 }
 
+func NewEmptyBasicDirectory(dserv ipld.DAGService) *BasicDirectory {
+	return NewBasicDirectoryFromNode(dserv, format.EmptyDirNode())
+}
+
+func NewBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *BasicDirectory {
+	basicDir := new(BasicDirectory)
+	basicDir.node = node
+	basicDir.dserv = dserv
+
+	// Scan node links (if any) to restore estimated size.
+	basicDir.ForEachLink(nil, func(l *ipld.Link) error {
+		basicDir.addToEstimatedSize(l.Name, l.Cid)
+		return nil
+	})
+	return basicDir
+}
+
 // NewDirectory returns a Directory that can either be a HAMTDirectory if the
 // UseHAMTSharding is set, or otherwise an UpgradeableDirectory containing a
 // BasicDirectory that can be converted to a HAMTDirectory if the option is
 // set in the future.
 func NewDirectory(dserv ipld.DAGService) Directory {
-	if UseHAMTSharding {
-		dir := new(HAMTDirectory)
-		s, err := hamt.NewShard(dserv, DefaultShardWidth)
-		if err != nil {
-			panic(err) // will only panic if DefaultShardWidth is a bad value
-		}
-		dir.shard = s
-		dir.dserv = dserv
-		return dir
-	}
-
-	basicDir := new(BasicDirectory)
-	basicDir.node = format.EmptyDirNode()
-	basicDir.dserv = dserv
-	return &UpgradeableDirectory{basicDir}
+	return &UpgradeableDirectory{NewEmptyBasicDirectory(dserv)}
 }
 
 // ErrNotADir implies that the given node was not a unixfs directory
@@ -121,10 +134,7 @@ func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, err
 
 	switch fsNode.Type() {
 	case format.TDirectory:
-		return &BasicDirectory{
-			dserv: dserv,
-			node:  protoBufNode.Copy().(*mdag.ProtoNode),
-		}, nil
+		return NewBasicDirectoryFromNode(dserv, protoBufNode.Copy().(*mdag.ProtoNode)), nil
 	case format.THAMTShard:
 		shard, err := hamt.NewHamtFromDag(dserv, node)
 		if err != nil {
@@ -139,6 +149,19 @@ func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, err
 	return nil, ErrNotADir
 }
 
+func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
+	d.estimatedSize += len(name) + len(linkCid.Bytes())
+	// FIXME: Ideally we may want to track the Link size as well but it is
+	//  minor in comparison with the other two.
+}
+
+func (d *BasicDirectory) removeFromEstimatedSize(name string, linkCid cid.Cid) {
+	d.estimatedSize -= len(name) + len(linkCid.Bytes())
+	if d.estimatedSize < 0 {
+		panic("BasicDirectory's estimatedSize went below 0")
+	}
+}
+
 // SetCidBuilder implements the `Directory` interface.
 func (d *BasicDirectory) SetCidBuilder(builder cid.Builder) {
 	d.node.SetCidBuilder(builder)
@@ -147,10 +170,15 @@ func (d *BasicDirectory) SetCidBuilder(builder cid.Builder) {
 // AddChild implements the `Directory` interface. It adds (or replaces)
 // a link to the given `node` under `name`.
 func (d *BasicDirectory) AddChild(ctx context.Context, name string, node ipld.Node) error {
-	d.node.RemoveNodeLink(name)
 	// Remove old link (if it existed), don't check a potential `ErrNotFound`.
+	d.RemoveChild(ctx, name)
 
-	return d.node.AddNodeLink(name, node)
+	err := d.node.AddNodeLink(name, node)
+	if err != nil {
+		return err
+	}
+	d.addToEstimatedSize(name, node.Cid())
+	return nil
 }
 
 // EnumLinksAsync returns a channel which will receive Links in the directory
@@ -203,11 +231,26 @@ func (d *BasicDirectory) Find(ctx context.Context, name string) (ipld.Node, erro
 
 // RemoveChild implements the `Directory` interface.
 func (d *BasicDirectory) RemoveChild(ctx context.Context, name string) error {
-	err := d.node.RemoveNodeLink(name)
+	// We need to *retrieve* the link before removing it to update the estimated
+	// size.
+	// FIXME: If this is too much of a potential penalty we could leave a fixed
+	//  CID size estimation based on the most common one used (normally SHA-256).
+	//  Alternatively we could add a GetAndRemoveLink method in `merkledag` to
+	//  iterate node links slice only once.
+	link, err := d.node.GetNodeLink(name)
 	if err == mdag.ErrLinkNotFound {
-		err = os.ErrNotExist
+		return os.ErrNotExist
 	}
-	return err
+	if err != nil {
+		return err // at the moment there is no other error besides ErrLinkNotFound
+	}
+
+	// The name actually existed so we should update the estimated size.
+	d.removeFromEstimatedSize(link.Name, link.Cid)
+
+	return d.node.RemoveNodeLink(name)
+	// GetNodeLink didn't return ErrLinkNotFound so this won't fail with that
+	// and we don't need to convert the error again.
 }
 
 // GetNode implements the `Directory` interface.
@@ -309,15 +352,31 @@ var _ Directory = (*UpgradeableDirectory)(nil)
 // AddChild implements the `Directory` interface. We check when adding new entries
 // if we should switch to HAMTDirectory according to global option(s).
 func (d *UpgradeableDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error {
-	if UseHAMTSharding {
-		if basicDir, ok := d.Directory.(*BasicDirectory); ok {
-			hamtDir, err := basicDir.SwitchToSharding(ctx)
-			if err != nil {
-				return err
-			}
-			d.Directory = hamtDir
+	err := d.Directory.AddChild(ctx, name, nd)
+	if err != nil {
+		return err
+	}
+
+	// Evaluate possible HAMT upgrade.
+	if HAMTShardingSize == 0 {
+		return nil
+	}
+	basicDir, ok := d.Directory.(*BasicDirectory)
+	if !ok {
+		return nil
+	}
+	if basicDir.estimatedSize >= HAMTShardingSize {
+		// FIXME: Ideally to minimize performance we should check if this last
+		//  `AddChild` call would bring the directory size over the threshold
+		//  *before* executing it since we would end up switching anyway and
+		//  that call would be "wasted". This is a minimal performance impact
+		//  and we prioritize a simple code base.
+		hamtDir, err := basicDir.SwitchToSharding(ctx)
+		if err != nil {
+			return err
 		}
+		d.Directory = hamtDir
 	}
 
-	return d.Directory.AddChild(ctx, name, nd)
+	return nil
 }
diff --git a/io/directory_test.go b/io/directory_test.go
index 2b6e6afa2..9501eb500 100644
--- a/io/directory_test.go
+++ b/io/directory_test.go
@@ -3,6 +3,8 @@ package io
 import (
 	"context"
 	"fmt"
+	mdag "github.com/ipfs/go-merkledag"
+	"math"
 	"testing"
 
 	ipld "github.com/ipfs/go-ipld-format"
@@ -98,27 +100,115 @@ func TestDuplicateAddDir(t *testing.T) {
 	}
 }
 
+// FIXME: Nothing blocking but nice to have:
+//  * Check estimated size against link enumeration (indirectly done in the
+//    restored node check from NewDirectoryFromNode).
+//  * Check estimated size against encoded node (the difference should only be
+//    a small percentage for a directory with 10s of entries).
+func TestBasicDirectory_estimatedSize(t *testing.T) {
+	ds := mdtest.Mock()
+	ctx := context.Background()
+	child := ft.EmptyFileNode()
+	err := ds.Add(ctx, child)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	basicDir := NewEmptyBasicDirectory(ds)
+
+	// Several overwrites should not corrupt the size estimation.
+	basicDir.AddChild(ctx, "child", child)
+	basicDir.AddChild(ctx, "child", child)
+	basicDir.AddChild(ctx, "child", child)
+	basicDir.RemoveChild(ctx, "child")
+	basicDir.AddChild(ctx, "child", child)
+	basicDir.RemoveChild(ctx, "child")
+	// FIXME: Check errors above (abstract adds/removals in iteration).
+	if basicDir.estimatedSize != 0 {
+		t.Fatal("estimated size is not zero after removing all entries")
+	}
+
+	for i := 0; i < 100; i++ {
+		basicDir.AddChild(ctx, fmt.Sprintf("child-%03d", i), child) // e.g., "child-045"
+	}
+	// Estimated entry size: name (9) + CID (32 from hash and 2 extra for header)
+	entrySize := 9 + 32 + 2
+	expectedSize := 100 * entrySize
+	if basicDir.estimatedSize != expectedSize {
+		t.Fatalf("estimated size (%d) inaccurate after adding many entries (expected %d)",
+			basicDir.estimatedSize, expectedSize)
+	}
+
+	basicDir.RemoveChild(ctx, "child-045") // just random values
+	basicDir.RemoveChild(ctx, "child-063")
+	basicDir.RemoveChild(ctx, "child-011")
+	basicDir.RemoveChild(ctx, "child-000")
+	basicDir.RemoveChild(ctx, "child-099")
+
+	basicDir.RemoveChild(ctx, "child-045")        // already removed, won't impact size
+	basicDir.RemoveChild(ctx, "nonexistent-name") // also doesn't count
+	basicDir.RemoveChild(ctx, "child-100")        // same
+	expectedSize -= 5 * entrySize
+	if basicDir.estimatedSize != expectedSize {
+		t.Fatalf("estimated size (%d) inaccurate after removing some entries (expected %d)",
+			basicDir.estimatedSize, expectedSize)
+	}
+
+	// Restore a directory from original's node and check estimated size consistency.
+	basicDirSingleNode, _ := basicDir.GetNode() // no possible error
+	restoredBasicDir := NewBasicDirectoryFromNode(ds, basicDirSingleNode.(*mdag.ProtoNode))
+	if basicDir.estimatedSize != restoredBasicDir.estimatedSize {
+		t.Fatalf("restored basic directory size (%d) doesn't match original estimate (%d)",
+			basicDir.estimatedSize, restoredBasicDir.estimatedSize)
+	}
+}
+
+// Basic test on extreme threshold to trigger switch. More fine-grained sizes
+// are checked in TestBasicDirectory_estimatedSize (without the swtich itself
+// but focusing on the size computation).
+// FIXME: Ideally, instead of checking size computation on one test and directory
+//  upgrade on another a better structured test should test both dimensions
+//  simultaneously.
 func TestUpgradeableDirectory(t *testing.T) {
-	oldHamtOption := UseHAMTSharding
-	defer func() { UseHAMTSharding = oldHamtOption }()
+	oldHamtOption := HAMTShardingSize
+	defer func() { HAMTShardingSize = oldHamtOption }()
 
 	ds := mdtest.Mock()
-	UseHAMTSharding = false // Create a BasicDirectory.
 	dir := NewDirectory(ds)
+	ctx := context.Background()
+	child := ft.EmptyDirNode()
+	err := ds.Add(ctx, child)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	HAMTShardingSize = 0 // Create a BasicDirectory.
 	if _, ok := dir.(*UpgradeableDirectory).Directory.(*BasicDirectory); !ok {
 		t.Fatal("UpgradeableDirectory doesn't contain BasicDirectory")
 	}
 
-	// Any new directory entry will trigger the upgrade to HAMTDirectory
-	UseHAMTSharding = true
+	// Set a threshold so big a new entry won't trigger the change.
+	HAMTShardingSize = math.MaxInt32
+
+	err = dir.AddChild(ctx, "test", child)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := dir.(*UpgradeableDirectory).Directory.(*HAMTDirectory); ok {
+		t.Fatal("UpgradeableDirectory was upgraded to HAMTDirectory for a large threshold")
+	}
+
+	// Now set it so low to make sure any new entry will trigger the upgrade.
+	HAMTShardingSize = 1
 
-	err := dir.AddChild(context.Background(), "test", ft.EmptyDirNode())
+	err = dir.AddChild(ctx, "test", child) // overwriting an entry should also trigger the switch
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	if _, ok := dir.(*UpgradeableDirectory).Directory.(*HAMTDirectory); !ok {
-		t.Fatal("UpgradeableDirectory wasn't upgraded to HAMTDirectory")
+		t.Fatal("UpgradeableDirectory wasn't upgraded to HAMTDirectory for a low threshold")
 	}
 }
 
diff --git a/unixfs.go b/unixfs.go
index 555d24efc..1b5a4edd9 100644
--- a/unixfs.go
+++ b/unixfs.go
@@ -361,6 +361,11 @@ func EmptyDirNode() *dag.ProtoNode {
 	return dag.NodeWithData(FolderPBData())
 }
 
+// EmptyDirNode creates an empty folder Protonode.
+func EmptyFileNode() *dag.ProtoNode {
+	return dag.NodeWithData(FilePBData(nil, 0))
+}
+
 // ReadUnixFSNodeData extracts the UnixFS data from an IPLD node.
 // Raw nodes are (also) processed because they are used as leaf
 // nodes containing (only) UnixFS data.

From 7237e575783a86680bed3f0d07573107a4a75a59 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Thu, 6 May 2021 20:00:15 -0300
Subject: [PATCH 2/9] fix doc

---
 io/directory.go | 8 +++-----
 unixfs.go       | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/io/directory.go b/io/directory.go
index e5fbeeaac..565232954 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -14,7 +14,7 @@ import (
 	ipld "github.com/ipfs/go-ipld-format"
 )
 
-// UseHAMTSharding is a global option that allows switching to a HAMTDirectory
+// HAMTShardingSize is a global option that allows switching to a HAMTDirectory
 // when the BasicDirectory grows above the size (in bytes) signalled by this
 // flag. The default size of 0 disables the option.
 // The size is not the *exact* block size of the encoded BasicDirectory but just
@@ -108,10 +108,8 @@ func NewBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *Bas
 	return basicDir
 }
 
-// NewDirectory returns a Directory that can either be a HAMTDirectory if the
-// UseHAMTSharding is set, or otherwise an UpgradeableDirectory containing a
-// BasicDirectory that can be converted to a HAMTDirectory if the option is
-// set in the future.
+// NewDirectory returns a Directory implemented by UpgradeableDirectory
+// containing a BasicDirectory that can be converted to a HAMTDirectory.
 func NewDirectory(dserv ipld.DAGService) Directory {
 	return &UpgradeableDirectory{NewEmptyBasicDirectory(dserv)}
 }
diff --git a/unixfs.go b/unixfs.go
index 1b5a4edd9..026b8bb3f 100644
--- a/unixfs.go
+++ b/unixfs.go
@@ -361,7 +361,7 @@ func EmptyDirNode() *dag.ProtoNode {
 	return dag.NodeWithData(FolderPBData())
 }
 
-// EmptyDirNode creates an empty folder Protonode.
+// EmptyFileNode creates an empty file Protonode.
 func EmptyFileNode() *dag.ProtoNode {
 	return dag.NodeWithData(FilePBData(nil, 0))
 }

From 0d6f81a016ee30d290ccf9553d4570f43d27a99b Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 12:39:24 -0300
Subject: [PATCH 3/9] fix: make basic dir constructors private

---
 io/directory.go      | 10 +++++-----
 io/directory_test.go |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/io/directory.go b/io/directory.go
index 565232954..86199be01 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -91,11 +91,11 @@ type HAMTDirectory struct {
 	dserv ipld.DAGService
 }
 
-func NewEmptyBasicDirectory(dserv ipld.DAGService) *BasicDirectory {
-	return NewBasicDirectoryFromNode(dserv, format.EmptyDirNode())
+func newEmptyBasicDirectory(dserv ipld.DAGService) *BasicDirectory {
+	return newBasicDirectoryFromNode(dserv, format.EmptyDirNode())
 }
 
-func NewBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *BasicDirectory {
+func newBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *BasicDirectory {
 	basicDir := new(BasicDirectory)
 	basicDir.node = node
 	basicDir.dserv = dserv
@@ -111,7 +111,7 @@ func NewBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *Bas
 // NewDirectory returns a Directory implemented by UpgradeableDirectory
 // containing a BasicDirectory that can be converted to a HAMTDirectory.
 func NewDirectory(dserv ipld.DAGService) Directory {
-	return &UpgradeableDirectory{NewEmptyBasicDirectory(dserv)}
+	return &UpgradeableDirectory{newEmptyBasicDirectory(dserv)}
 }
 
 // ErrNotADir implies that the given node was not a unixfs directory
@@ -132,7 +132,7 @@ func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, err
 
 	switch fsNode.Type() {
 	case format.TDirectory:
-		return NewBasicDirectoryFromNode(dserv, protoBufNode.Copy().(*mdag.ProtoNode)), nil
+		return newBasicDirectoryFromNode(dserv, protoBufNode.Copy().(*mdag.ProtoNode)), nil
 	case format.THAMTShard:
 		shard, err := hamt.NewHamtFromDag(dserv, node)
 		if err != nil {
diff --git a/io/directory_test.go b/io/directory_test.go
index 9501eb500..6fb1270b7 100644
--- a/io/directory_test.go
+++ b/io/directory_test.go
@@ -114,7 +114,7 @@ func TestBasicDirectory_estimatedSize(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	basicDir := NewEmptyBasicDirectory(ds)
+	basicDir := newEmptyBasicDirectory(ds)
 
 	// Several overwrites should not corrupt the size estimation.
 	basicDir.AddChild(ctx, "child", child)
@@ -156,7 +156,7 @@ func TestBasicDirectory_estimatedSize(t *testing.T) {
 
 	// Restore a directory from original's node and check estimated size consistency.
 	basicDirSingleNode, _ := basicDir.GetNode() // no possible error
-	restoredBasicDir := NewBasicDirectoryFromNode(ds, basicDirSingleNode.(*mdag.ProtoNode))
+	restoredBasicDir := newBasicDirectoryFromNode(ds, basicDirSingleNode.(*mdag.ProtoNode))
 	if basicDir.estimatedSize != restoredBasicDir.estimatedSize {
 		t.Fatalf("restored basic directory size (%d) doesn't match original estimate (%d)",
 			basicDir.estimatedSize, restoredBasicDir.estimatedSize)

From 2f8cfa75fa46a18f999c62f088a828fb066c3051 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 12:52:53 -0300
Subject: [PATCH 4/9] fix: if underflow log instead of panic

---
 go.mod          |  3 ++-
 go.sum          | 54 +++++++++++++++++++++++++++++++++++++++++++++++++
 io/directory.go | 21 ++++++++++++++-----
 3 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index da2e60b18..ea45ba145 100644
--- a/go.mod
+++ b/go.mod
@@ -2,7 +2,7 @@ module github.com/ipfs/go-unixfs
 
 require (
 	github.com/Stebalien/go-bitfield v0.0.1
-	github.com/gogo/protobuf v1.2.1
+	github.com/gogo/protobuf v1.3.2
 	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
 	github.com/ipfs/go-bitswap v0.1.2 // indirect
 	github.com/ipfs/go-cid v0.0.2
@@ -11,6 +11,7 @@ require (
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
 	github.com/ipfs/go-ipfs-util v0.0.1
 	github.com/ipfs/go-ipld-format v0.0.2
+	github.com/ipfs/go-log v1.0.5
 	github.com/ipfs/go-merkledag v0.2.3
 	github.com/multiformats/go-multihash v0.0.5
 	github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 // indirect
diff --git a/go.sum b/go.sum
index 46d1f2d26..e4bd326d7 100644
--- a/go.sum
+++ b/go.sum
@@ -1,4 +1,5 @@
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo=
 github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s=
@@ -35,12 +36,15 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -114,6 +118,10 @@ github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2
 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
 github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8=
+github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo=
+github.com/ipfs/go-log/v2 v2.1.3 h1:1iS3IU7aXRlbgUpN8yTTpJ53NXYjAe37vcI5+5nYrzk=
+github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
 github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
@@ -143,6 +151,7 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ=
@@ -288,6 +297,8 @@ github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -296,6 +307,7 @@ github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGg
 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 h1:2m16U/rLwVaRdz7ANkHtHTodP3zTP3N451MADg64x5k=
 github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
@@ -313,6 +325,7 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
@@ -330,16 +343,35 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX
 github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds=
 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
+go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
@@ -350,8 +382,14 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgP
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -363,17 +401,32 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -382,3 +435,4 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
diff --git a/io/directory.go b/io/directory.go
index 86199be01..4f9796235 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -12,8 +12,11 @@ import (
 
 	cid "github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
+	logging "github.com/ipfs/go-log"
 )
 
+var log = logging.Logger("unixfs")
+
 // HAMTShardingSize is a global option that allows switching to a HAMTDirectory
 // when the BasicDirectory grows above the size (in bytes) signalled by this
 // flag. The default size of 0 disables the option.
@@ -101,10 +104,8 @@ func newBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *Bas
 	basicDir.dserv = dserv
 
 	// Scan node links (if any) to restore estimated size.
-	basicDir.ForEachLink(nil, func(l *ipld.Link) error {
-		basicDir.addToEstimatedSize(l.Name, l.Cid)
-		return nil
-	})
+	basicDir.computeEstimatedSize()
+
 	return basicDir
 }
 
@@ -147,6 +148,13 @@ func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, err
 	return nil, ErrNotADir
 }
 
+func (d *BasicDirectory) computeEstimatedSize() {
+	d.ForEachLink(nil, func(l *ipld.Link) error {
+		d.addToEstimatedSize(l.Name, l.Cid)
+		return nil
+	})
+}
+
 func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
 	d.estimatedSize += len(name) + len(linkCid.Bytes())
 	// FIXME: Ideally we may want to track the Link size as well but it is
@@ -156,7 +164,10 @@ func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
 func (d *BasicDirectory) removeFromEstimatedSize(name string, linkCid cid.Cid) {
 	d.estimatedSize -= len(name) + len(linkCid.Bytes())
 	if d.estimatedSize < 0 {
-		panic("BasicDirectory's estimatedSize went below 0")
+		// Something has gone very wrong. Log an error and recompute the
+		// size from scratch.
+		log.Error("BasicDirectory's estimatedSize went below 0")
+		d.computeEstimatedSize()
 	}
 }
 

From 13a7f619b84fc84ef358d58d3daa59a5c0fae502 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 12:56:57 -0300
Subject: [PATCH 5/9] fix: don't ignore RemoveChild error

---
 io/directory.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/io/directory.go b/io/directory.go
index 4f9796235..17693310a 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -179,10 +179,13 @@ func (d *BasicDirectory) SetCidBuilder(builder cid.Builder) {
 // AddChild implements the `Directory` interface. It adds (or replaces)
 // a link to the given `node` under `name`.
 func (d *BasicDirectory) AddChild(ctx context.Context, name string, node ipld.Node) error {
-	// Remove old link (if it existed), don't check a potential `ErrNotFound`.
-	d.RemoveChild(ctx, name)
+	// Remove old link (if it existed; ignore `ErrNotExist` otherwise).
+	err := d.RemoveChild(ctx, name)
+	if err != nil && err != os.ErrNotExist {
+		return err
+	}
 
-	err := d.node.AddNodeLink(name, node)
+	err = d.node.AddNodeLink(name, node)
 	if err != nil {
 		return err
 	}

From 53f2df15adccdacd44f8fdf940508135b9edda38 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 13:00:17 -0300
Subject: [PATCH 6/9] fix: resolve fixmes

---
 io/directory.go | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/io/directory.go b/io/directory.go
index 17693310a..484d3899d 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -244,11 +244,9 @@ func (d *BasicDirectory) Find(ctx context.Context, name string) (ipld.Node, erro
 // RemoveChild implements the `Directory` interface.
 func (d *BasicDirectory) RemoveChild(ctx context.Context, name string) error {
 	// We need to *retrieve* the link before removing it to update the estimated
-	// size.
-	// FIXME: If this is too much of a potential penalty we could leave a fixed
-	//  CID size estimation based on the most common one used (normally SHA-256).
-	//  Alternatively we could add a GetAndRemoveLink method in `merkledag` to
-	//  iterate node links slice only once.
+	// size. This means we may iterate the links slice twice: if traversing this
+	// becomes a problem, a factor of 2 isn't going to make much of a difference.
+	// We'd likely need to cache a link resolution map in that case.
 	link, err := d.node.GetNodeLink(name)
 	if err == mdag.ErrLinkNotFound {
 		return os.ErrNotExist
@@ -378,11 +376,11 @@ func (d *UpgradeableDirectory) AddChild(ctx context.Context, name string, nd ipl
 		return nil
 	}
 	if basicDir.estimatedSize >= HAMTShardingSize {
-		// FIXME: Ideally to minimize performance we should check if this last
-		//  `AddChild` call would bring the directory size over the threshold
-		//  *before* executing it since we would end up switching anyway and
-		//  that call would be "wasted". This is a minimal performance impact
-		//  and we prioritize a simple code base.
+		// Ideally to minimize performance we should check if this last
+		// `AddChild` call would bring the directory size over the threshold
+		// *before* executing it since we would end up switching anyway and
+		// that call would be "wasted". This is a minimal performance impact
+		// and we prioritize a simple code base.
 		hamtDir, err := basicDir.SwitchToSharding(ctx)
 		if err != nil {
 			return err

From 06ef3e866066812516d7e0199be5f233cc36e1ef Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 13:01:27 -0300
Subject: [PATCH 7/9] fix: sort imports

---
 io/directory_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/io/directory_test.go b/io/directory_test.go
index 6fb1270b7..8c5d8e109 100644
--- a/io/directory_test.go
+++ b/io/directory_test.go
@@ -3,11 +3,11 @@ package io
 import (
 	"context"
 	"fmt"
-	mdag "github.com/ipfs/go-merkledag"
 	"math"
 	"testing"
 
 	ipld "github.com/ipfs/go-ipld-format"
+	mdag "github.com/ipfs/go-merkledag"
 	mdtest "github.com/ipfs/go-merkledag/test"
 
 	ft "github.com/ipfs/go-unixfs"

From 6a50780a8ace0be98add1e0aa1b428d831beb62e Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 13:19:42 -0300
Subject: [PATCH 8/9] fix: use (Cid).ByteLen()

---
 go.mod          |  4 ++--
 go.sum          | 14 ++++++++++++++
 io/directory.go |  8 ++++++--
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index ea45ba145..3adce8acb 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
 	github.com/ipfs/go-bitswap v0.1.2 // indirect
-	github.com/ipfs/go-cid v0.0.2
+	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-ipfs-chunker v0.0.1
 	github.com/ipfs/go-ipfs-files v0.0.3
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
@@ -13,7 +13,7 @@ require (
 	github.com/ipfs/go-ipld-format v0.0.2
 	github.com/ipfs/go-log v1.0.5
 	github.com/ipfs/go-merkledag v0.2.3
-	github.com/multiformats/go-multihash v0.0.5
+	github.com/multiformats/go-multihash v0.0.13
 	github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 // indirect
 	github.com/smartystreets/assertions v1.0.0 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
diff --git a/go.sum b/go.sum
index e4bd326d7..1b2b7dbcc 100644
--- a/go.sum
+++ b/go.sum
@@ -77,6 +77,8 @@ github.com/ipfs/go-cid v0.0.1 h1:GBjWPktLnNyX0JiQCNFpUuUSoMw5KMyqrsejHYlILBE=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
+github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
 github.com/ipfs/go-datastore v0.0.1 h1:AW/KZCScnBWlSb5JbnEnLKFWXL224LBEh/9KXXOrUms=
 github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6NriQo=
@@ -257,13 +259,19 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+
 github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM=
 github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78=
 github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
+github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
 github.com/multiformats/go-multiaddr v0.0.1 h1:/QUV3VBMDI6pi6xfiw7lr6xhDWWvQKn9udPn68kLSdY=
 github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
 github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
@@ -278,12 +286,18 @@ github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi
 github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
 github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA=
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
 github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5 h1:1wxmCvTXAifAepIMyF39vZinRw5sbqjPs/UIi93+uik=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
+github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
+github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ=
 github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
+github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
+github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
diff --git a/io/directory.go b/io/directory.go
index 484d3899d..f8bfcf06c 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -155,14 +155,18 @@ func (d *BasicDirectory) computeEstimatedSize() {
 	})
 }
 
+func estimatedLinkSize(linkName string, linkCid cid.Cid) int {
+	return len(linkName) + linkCid.ByteLen()
+}
+
 func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
-	d.estimatedSize += len(name) + len(linkCid.Bytes())
+	d.estimatedSize += estimatedLinkSize(name, linkCid)
 	// FIXME: Ideally we may want to track the Link size as well but it is
 	//  minor in comparison with the other two.
 }
 
 func (d *BasicDirectory) removeFromEstimatedSize(name string, linkCid cid.Cid) {
-	d.estimatedSize -= len(name) + len(linkCid.Bytes())
+	d.estimatedSize -= estimatedLinkSize(name, linkCid)
 	if d.estimatedSize < 0 {
 		// Something has gone very wrong. Log an error and recompute the
 		// size from scratch.

From 527eae40fff3761a3b0967e7de802555d1955bc0 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 16:41:51 -0300
Subject: [PATCH 9/9] fix: estimatedSize comment

---
 io/directory.go | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/io/directory.go b/io/directory.go
index f8bfcf06c..b0c4549aa 100644
--- a/io/directory.go
+++ b/io/directory.go
@@ -80,10 +80,11 @@ type BasicDirectory struct {
 	node  *mdag.ProtoNode
 	dserv ipld.DAGService
 
-	// Internal variable used to cache the estimated size used for the
-	// HAMTShardingSize option. We maintain this value even if the
-	// HAMTShardingSize is off since potentially the option could be activated
-	// on the fly.
+	// Internal variable used to cache the estimated size of the basic directory:
+	// for each link, aggregate link name + link CID. DO NOT CHANGE THIS
+	// as it will affect the HAMT transition behavior in HAMTShardingSize.
+	// (We maintain this value up to date even if the HAMTShardingSize is off
+	// since potentially the option could be activated on the fly.)
 	estimatedSize int
 }
 
@@ -161,8 +162,6 @@ func estimatedLinkSize(linkName string, linkCid cid.Cid) int {
 
 func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
 	d.estimatedSize += estimatedLinkSize(name, linkCid)
-	// FIXME: Ideally we may want to track the Link size as well but it is
-	//  minor in comparison with the other two.
 }
 
 func (d *BasicDirectory) removeFromEstimatedSize(name string, linkCid cid.Cid) {