Merge pull request #2960 from ipfs/feat/mdag-caching

cache encoded data when reading dag nodes from disk

core/commands/add.go

@@ -12,6 +12,7 @@ import (
 	core "github.com/ipfs/go-ipfs/core"
 	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
 	mfs "github.com/ipfs/go-ipfs/mfs"
+	ft "github.com/ipfs/go-ipfs/unixfs"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
 
@@ -160,7 +161,7 @@ You can now refer to the added file in a gateway, like so:
 
 		if hash {
 			md := dagtest.Mock()
-			mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil)
+			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
 			if err != nil {
 				res.SetError(err, cmds.ErrNormal)
 				return

core/commands/files/files.go

@@ -163,7 +163,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
 		return nil, err
 	}
 
-	d, err := ft.FromBytes(nd.Data)
+	d, err := ft.FromBytes(nd.Data())
 	if err != nil {
 		return nil, err
 	}
@@ -871,7 +871,7 @@ func getFileHandle(r *mfs.Root, path string, create bool) (*mfs.File, error) {
 			return nil, fmt.Errorf("%s was not a directory", dirname)
 		}
 
-		nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
+		nd := dag.NodeWithData(ft.FilePBData(nil, 0))
 		err = pdir.AddChild(fname, nd)
 		if err != nil {
 			return nil, err

core/commands/ls.go

@@ -110,7 +110,7 @@ format:
 					}
 				}
 				if linkNode != nil {
-					d, err := unixfs.FromBytes(linkNode.Data)
+					d, err := unixfs.FromBytes(linkNode.Data())
 					if err != nil {
 						res.SetError(err, cmds.ErrNormal)
 						return

core/commands/object/object.go

@@ -93,7 +93,7 @@ is the raw data of the object.
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		res.SetOutput(bytes.NewReader(node.Data))
+		res.SetOutput(bytes.NewReader(node.Data()))
 	},
 }
 
@@ -198,7 +198,7 @@ This command outputs data in the following encodings:
 
 		node := &Node{
 			Links: make([]Link, len(object.Links)),
-			Data:  string(object.Data),
+			Data:  string(object.Data()),
 		}
 
 		for i, link := range object.Links {
@@ -438,9 +438,7 @@ Available templates:
 func nodeFromTemplate(template string) (*dag.Node, error) {
 	switch template {
 	case "unixfs-dir":
-		nd := new(dag.Node)
-		nd.Data = ft.FolderPBData()
-		return nd, nil
+		return ft.EmptyDirNode(), nil
 	default:
 		return nil, fmt.Errorf("template '%s' not found", template)
 	}
@@ -566,9 +564,10 @@ func deserializeNode(node *Node, dataFieldEncoding string) (*dag.Node, error) {
 	dagnode := new(dag.Node)
 	switch dataFieldEncoding {
 	case "text":
-		dagnode.Data = []byte(node.Data)
+		dagnode.SetData([]byte(node.Data))
 	case "base64":
-		dagnode.Data, _ = base64.StdEncoding.DecodeString(node.Data)
+		data, _ := base64.StdEncoding.DecodeString(node.Data)
+		dagnode.SetData(data)
 	default:
 		return nil, fmt.Errorf("Unkown data field encoding")
 	}

core/commands/object/patch.go

@@ -91,7 +91,7 @@ the limit will not be respected by the network.
 			return
 		}
 
-		rootnd.Data = append(rootnd.Data, data...)
+		rootnd.SetData(append(rootnd.Data(), data...))
 
 		newkey, err := nd.DAG.Add(rootnd)
 		if err != nil {
@@ -153,7 +153,7 @@ Example:
 			return
 		}
 
-		root.Data = data
+		root.SetData(data)
 
 		newkey, err := nd.DAG.Add(root)
 		if err != nil {
@@ -287,9 +287,7 @@ to a file containing 'bar', and returns the hash of the new object.
 
 		var createfunc func() *dag.Node
 		if create {
-			createfunc = func() *dag.Node {
-				return &dag.Node{Data: ft.FolderPBData()}
-			}
+			createfunc = ft.EmptyDirNode
 		}
 
 		e := dagutils.NewDagEditor(root, nd.DAG)

core/commands/unixfs/ls.go

@@ -101,7 +101,7 @@ Example:
 				continue
 			}
 
-			unixFSNode, err := unixfs.FromBytes(merkleNode.Data)
+			unixFSNode, err := unixfs.FromBytes(merkleNode.Data())
 			if err != nil {
 				res.SetError(err, cmds.ErrNormal)
 				return
@@ -128,7 +128,7 @@ Example:
 						res.SetError(err, cmds.ErrNormal)
 						return
 					}
-					d, err := unixfs.FromBytes(linkNode.Data)
+					d, err := unixfs.FromBytes(linkNode.Data())
 					if err != nil {
 						res.SetError(err, cmds.ErrNormal)
 						return

core/corehttp/gateway_handler.go

@@ -391,7 +391,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
 
 	case nil:
 		// object set-data case
-		rnode.Data = newnode.Data
+		rnode.SetData(newnode.Data())
 
 		newkey, err = i.node.DAG.Add(rnode)
 		if err != nil {

core/coreunix/add.go

@@ -29,8 +29,6 @@ import (
 
 var log = logging.Logger("coreunix")
 
-var folderData = unixfs.FolderPBData()
-
 // how many bytes of progress to wait before sending a progress update message
 const progressReaderIncrement = 1024 * 256
 
@@ -67,7 +65,7 @@ type AddedObject struct {
 }
 
 func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
-	mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil)
+	mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -392,7 +390,7 @@ func (adder *Adder) addFile(file files.File) error {
 			return err
 		}
 
-		dagnode := &dag.Node{Data: sdata}
+		dagnode := dag.NodeWithData(sdata)
 		_, err = adder.dagService.Add(dagnode)
 		if err != nil {
 			return err
@@ -488,11 +486,6 @@ func NewMemoryDagService() dag.DAGService {
 	return dag.NewDAGService(bsrv)
 }
 
-// TODO: generalize this to more than unix-fs nodes.
-func NewDirNode() *dag.Node {
-	return &dag.Node{Data: unixfs.FolderPBData()}
-}
-
 // from core/commands/object.go
 func getOutput(dagnode *dag.Node) (*Object, error) {
 	key, err := dagnode.Key()

core/coreunix/metadata.go

@@ -21,7 +21,7 @@ func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error
 		return "", err
 	}
 
-	mdnode.Data = mdata
+	mdnode.SetData(mdata)
 	if err := mdnode.AddNodeLinkClean("file", nd); err != nil {
 		return "", err
 	}
@@ -42,5 +42,5 @@ func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) {
 		return nil, err
 	}
 
-	return ft.MetadataFromBytes(nd.Data)
+	return ft.MetadataFromBytes(nd.Data())
 }

fuse/ipns/common.go

@@ -4,7 +4,6 @@ import (
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
 
 	"github.com/ipfs/go-ipfs/core"
-	mdag "github.com/ipfs/go-ipfs/merkledag"
 	nsys "github.com/ipfs/go-ipfs/namesys"
 	path "github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
@@ -14,7 +13,7 @@ import (
 // InitializeKeyspace sets the ipns record for the given key to
 // point to an empty directory.
 func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
-	emptyDir := &mdag.Node{Data: ft.FolderPBData()}
+	emptyDir := ft.EmptyDirNode()
 	nodek, err := n.DAG.Add(emptyDir)
 	if err != nil {
 		return err

fuse/ipns/ipns_unix.go

@@ -481,7 +481,7 @@ func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 
 func (dir *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
 	// New 'empty' file
-	nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
+	nd := dag.NodeWithData(ft.FilePBData(nil, 0))
 	err := dir.dir.AddChild(req.Name, nd)
 	if err != nil {
 		return nil, nil, err

fuse/readonly/readonly_unix.go

@@ -84,7 +84,7 @@ type Node struct {
 
 func (s *Node) loadData() error {
 	s.cached = new(ftpb.Data)
-	return proto.Unmarshal(s.Nd.Data, s.cached)
+	return proto.Unmarshal(s.Nd.Data(), s.cached)
 }
 
 // Attr returns the attributes of a given node.

importer/helpers/helpers.go

@@ -59,7 +59,7 @@ func NewUnixfsBlock() *UnixfsNode {
 
 // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
 func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) {
-	mb, err := ft.FSNodeFromBytes(nd.Data)
+	mb, err := ft.FSNodeFromBytes(nd.Data())
 	if err != nil {
 		return nil, err
 	}
@@ -126,6 +126,6 @@ func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
 	if err != nil {
 		return nil, err
 	}
-	n.node.Data = data
+	n.node.SetData(data)
 	return n.node, nil
 }

importer/trickle/trickle_test.go

@@ -524,7 +524,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
 	data := []byte("AB")
 
 	nd := new(merkledag.Node)
-	nd.Data = ft.FilePBData(nil, 0)
+	nd.SetData(ft.FilePBData(nil, 0))
 
 	dbp := &h.DagBuilderParams{
 		Dagserv:  ds,
@@ -562,7 +562,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
 }
 
 func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) {
-	pbd, err := ft.FromBytes(nd.Data)
+	pbd, err := ft.FromBytes(nd.Data())
 	if err != nil {
 		panic(err)
 	}

importer/trickle/trickledag.go

@@ -241,7 +241,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
 			return errors.New("expected direct block")
 		}
 
-		pbn, err := ft.FromBytes(nd.Data)
+		pbn, err := ft.FromBytes(nd.Data())
 		if err != nil {
 			return err
 		}
@@ -253,7 +253,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
 	}
 
 	// Verify this is a branch node
-	pbn, err := ft.FromBytes(nd.Data)
+	pbn, err := ft.FromBytes(nd.Data())
 	if err != nil {
 		return err
 	}

merkledag/coding.go

@@ -33,7 +33,8 @@ func (n *Node) unmarshal(encoded []byte) error {
 	}
 	sort.Stable(LinkSlice(n.Links)) // keep links sorted
 
-	n.Data = pbn.GetData()
+	n.data = pbn.GetData()
+	n.encoded = encoded
 	return nil
 }
 
@@ -62,8 +63,8 @@ func (n *Node) getPBNode() *pb.PBNode {
 		pbn.Links[i].Hash = []byte(l.Hash)
 	}
 
-	if len(n.Data) > 0 {
-		pbn.Data = n.Data
+	if len(n.data) > 0 {
+		pbn.Data = n.data
 	}
 	return pbn
 }
@@ -73,11 +74,15 @@ func (n *Node) getPBNode() *pb.PBNode {
 func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
 	sort.Stable(LinkSlice(n.Links)) // keep links sorted
 	if n.encoded == nil || force {
+		n.cached = nil
 		var err error
 		n.encoded, err = n.Marshal()
 		if err != nil {
 			return nil, err
 		}
+	}
+
+	if n.cached == nil {
 		n.cached = u.Hash(n.encoded)
 	}
 

merkledag/merkledag.go

@@ -93,6 +93,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
 		}
 		return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
 	}
+
+	res.cached = k.ToMultihash()
+
 	return res, nil
 }
 
@@ -147,6 +150,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
 					out <- &NodeOption{Err: err}
 					return
 				}
+				nd.cached = b.Key().ToMultihash()
 
 				// buffered, no need to select
 				out <- &NodeOption{Node: nd}

merkledag/merkledag_test.go

@@ -46,9 +46,9 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
 
 func TestNode(t *testing.T) {
 
-	n1 := &Node{Data: []byte("beep")}
-	n2 := &Node{Data: []byte("boop")}
-	n3 := &Node{Data: []byte("beep boop")}
+	n1 := NodeWithData([]byte("beep"))
+	n2 := NodeWithData([]byte("boop"))
+	n3 := NodeWithData([]byte("beep boop"))
 	if err := n3.AddNodeLink("beep-link", n1); err != nil {
 		t.Error(err)
 	}
@@ -58,7 +58,7 @@ func TestNode(t *testing.T) {
 
 	printn := func(name string, n *Node) {
 		fmt.Println(">", name)
-		fmt.Println("data:", string(n.Data))
+		fmt.Println("data:", string(n.Data()))
 
 		fmt.Println("links:")
 		for _, l := range n.Links {
@@ -118,8 +118,8 @@ func SubtestNodeStat(t *testing.T, n *Node) {
 	expected := NodeStat{
 		NumLinks:       len(n.Links),
 		BlockSize:      len(enc),
-		LinksSize:      len(enc) - len(n.Data), // includes framing.
-		DataSize:       len(n.Data),
+		LinksSize:      len(enc) - len(n.Data()), // includes framing.
+		DataSize:       len(n.Data()),
 		CumulativeSize: int(cumSize),
 		Hash:           k.B58String(),
 	}
@@ -255,7 +255,7 @@ func TestEmptyKey(t *testing.T) {
 
 func TestCantGet(t *testing.T) {
 	dsp := getDagservAndPinner(t)
-	a := &Node{Data: []byte("A")}
+	a := NodeWithData([]byte("A"))
 
 	k, err := a.Key()
 	if err != nil {
@@ -339,7 +339,7 @@ func TestFetchFailure(t *testing.T) {
 
 	top := new(Node)
 	for i := 0; i < 10; i++ {
-		nd := &Node{Data: []byte{byte('a' + i)}}
+		nd := NodeWithData([]byte{byte('a' + i)})
 		_, err := ds.Add(nd)
 		if err != nil {
 			t.Fatal(err)
@@ -352,7 +352,7 @@ func TestFetchFailure(t *testing.T) {
 	}
 
 	for i := 0; i < 10; i++ {
-		nd := &Node{Data: []byte{'f', 'a' + byte(i)}}
+		nd := NodeWithData([]byte{'f', 'a' + byte(i)})
 		_, err := ds_bad.Add(nd)
 		if err != nil {
 			t.Fatal(err)