Merge pull request #3743 from ipfs/kevina/cidv1-add

Add support for using CidV1 in 'ipfs add'

core/commands/add.go

@@ -38,8 +38,11 @@ const (
 	rawLeavesOptionName   = "raw-leaves"
 	noCopyOptionName      = "nocopy"
 	fstoreCacheOptionName = "fscache"
+	cidVersionOptionName  = "cid-version"
 )
 
+const adderOutChanSize = 8
+
 var AddCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Add a file or directory to ipfs.",
@@ -86,6 +89,7 @@ You can now refer to the added file in a gateway, like so:
 		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
+		cmds.IntOption(cidVersionOptionName, "Cid version. Non-zero value will change default of 'raw-leaves' to true. (experimental)").Default(0),
 	},
 	PreRun: func(req cmds.Request) error {
 		quiet, _, _ := req.Option(quietOptionName).Bool()
@@ -159,6 +163,7 @@ You can now refer to the added file in a gateway, like so:
 		rawblks, rbset, _ := req.Option(rawLeavesOptionName).Bool()
 		nocopy, _, _ := req.Option(noCopyOptionName).Bool()
 		fscache, _, _ := req.Option(fstoreCacheOptionName).Bool()
+		cidVer, _, _ := req.Option(cidVersionOptionName).Int()
 
 		if nocopy && !cfg.Experimental.FilestoreEnabled {
 			res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"),
@@ -175,6 +180,16 @@ You can now refer to the added file in a gateway, like so:
 			return
 		}
 
+		if cidVer >= 1 && !rbset {
+			rawblks = true
+		}
+
+		prefix, err := dag.PrefixForCidVersion(cidVer)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
 		if hash {
 			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
 				//TODO: need this to be true or all files
@@ -202,15 +217,15 @@ You can now refer to the added file in a gateway, like so:
 		bserv := blockservice.New(addblockstore, exch)
 		dserv := dag.NewDAGService(bserv)
 
-		outChan := make(chan interface{}, 8)
-		res.SetOutput((<-chan interface{})(outChan))
-
 		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
 
+		outChan := make(chan interface{}, adderOutChanSize)
+		res.SetOutput((<-chan interface{})(outChan))
+
 		fileAdder.Out = outChan
 		fileAdder.Chunker = chunker
 		fileAdder.Progress = progress
@@ -221,6 +236,7 @@ You can now refer to the added file in a gateway, like so:
 		fileAdder.Silent = silent
 		fileAdder.RawLeaves = rawblks
 		fileAdder.NoCopy = nocopy
+		fileAdder.Prefix = &prefix
 
 		if hash {
 			md := dagtest.Mock()

core/coreunix/add.go

@@ -69,13 +69,7 @@ type AddedObject struct {
 }
 
 func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
-	mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil)
-	if err != nil {
-		return nil, err
-	}
-
 	return &Adder{
-		mr:         mr,
 		ctx:        ctx,
 		pinning:    p,
 		blockstore: bs,
@@ -87,7 +81,6 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.
 		Wrap:       false,
 		Chunker:    "",
 	}, nil
-
 }
 
 // Adder holds the switches passed to the `add` command.
@@ -107,13 +100,29 @@ type Adder struct {
 	NoCopy     bool
 	Chunker    string
 	root       node.Node
-	mr         *mfs.Root
+	mroot      *mfs.Root
 	unlocker   bs.Unlocker
 	tempRoot   *cid.Cid
+	Prefix     *cid.Prefix
+}
+
+func (adder *Adder) mfsRoot() (*mfs.Root, error) {
+	if adder.mroot != nil {
+		return adder.mroot, nil
+	}
+	rnode := unixfs.EmptyDirNode()
+	rnode.SetPrefix(adder.Prefix)
+	mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil)
+	mr.Prefix = adder.Prefix
+	if err != nil {
+		return nil, err
+	}
+	adder.mroot = mr
+	return adder.mroot, nil
 }
 
 func (adder *Adder) SetMfsRoot(r *mfs.Root) {
-	adder.mr = r
+	adder.mroot = r
 }
 
 // Constructs a node from reader's data, and adds it. Doesn't pin.
@@ -122,11 +131,13 @@ func (adder Adder) add(reader io.Reader) (node.Node, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	params := ihelper.DagBuilderParams{
 		Dagserv:   adder.dagService,
 		RawLeaves: adder.RawLeaves,
 		Maxlinks:  ihelper.DefaultLinksPerBlock,
 		NoCopy:    adder.NoCopy,
+		Prefix:    adder.Prefix,
 	}
 
 	if adder.Trickle {
@@ -142,7 +153,11 @@ func (adder *Adder) RootNode() (node.Node, error) {
 		return adder.root, nil
 	}
 
-	root, err := adder.mr.GetValue().GetNode()
+	mr, err := adder.mfsRoot()
+	if err != nil {
+		return nil, err
+	}
+	root, err := mr.GetValue().GetNode()
 	if err != nil {
 		return nil, err
 	}
@@ -188,9 +203,13 @@ func (adder *Adder) PinRoot() error {
 }
 
 func (adder *Adder) Finalize() (node.Node, error) {
-	root := adder.mr.GetValue()
+	mr, err := adder.mfsRoot()
+	if err != nil {
+		return nil, err
+	}
+	root := mr.GetValue()
 
-	err := root.Flush()
+	err = root.Flush()
 	if err != nil {
 		return nil, err
 	}
@@ -203,7 +222,12 @@ func (adder *Adder) Finalize() (node.Node, error) {
 		}
 		name = children[0]
 
-		dir, ok := adder.mr.GetValue().(*mfs.Directory)
+		mr, err := adder.mfsRoot()
+		if err != nil {
+			return nil, err
+		}
+
+		dir, ok := mr.GetValue().(*mfs.Directory)
 		if !ok {
 			return nil, fmt.Errorf("root is not a directory")
 		}
@@ -219,7 +243,7 @@ func (adder *Adder) Finalize() (node.Node, error) {
 		return nil, err
 	}
 
-	err = adder.mr.Close()
+	err = mr.Close()
 	if err != nil {
 		return nil, err
 	}
@@ -357,14 +381,18 @@ func (adder *Adder) addNode(node node.Node, path string) error {
 		node = pi.Node
 	}
 
+	mr, err := adder.mfsRoot()
+	if err != nil {
+		return err
+	}
 	dir := gopath.Dir(path)
 	if dir != "." {
-		if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil {
+		if err := mfs.Mkdir(mr, dir, true, false); err != nil {
 			return err
 		}
 	}
 
-	if err := mfs.PutNode(adder.mr, path, node); err != nil {
+	if err := mfs.PutNode(mr, path, node); err != nil {
 		return err
 	}
 
@@ -406,6 +434,7 @@ func (adder *Adder) addFile(file files.File) error {
 		}
 
 		dagnode := dag.NodeWithData(sdata)
+		dagnode.SetPrefix(adder.Prefix)
 		_, err = adder.dagService.Add(dagnode)
 		if err != nil {
 			return err
@@ -439,7 +468,11 @@ func (adder *Adder) addFile(file files.File) error {
 func (adder *Adder) addDir(dir files.File) error {
 	log.Infof("adding directory: %s", dir.FileName())
 
-	err := mfs.Mkdir(adder.mr, dir.FileName(), true, false)
+	mr, err := adder.mfsRoot()
+	if err != nil {
+		return err
+	}
+	err = mfs.Mkdir(mr, dir.FileName(), true, false)
 	if err != nil {
 		return err
 	}

importer/balanced/builder.go

@@ -13,7 +13,7 @@ func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) {
 	var root *h.UnixfsNode
 	for level := 0; !db.Done(); level++ {
 
-		nroot := h.NewUnixfsNode()
+		nroot := db.NewUnixfsNode()
 		db.SetPosInfo(nroot, 0)
 
 		// add our old root as a child of the new root.
@@ -33,7 +33,7 @@ func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) {
 
 	}
 	if root == nil {
-		root = h.NewUnixfsNode()
+		root = db.NewUnixfsNode()
 	}
 
 	out, err := db.Add(root)
@@ -72,7 +72,7 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.UnixfsNode, depth int, offset u
 
 	// while we have room AND we're not done
 	for node.NumChildren() < db.Maxlinks() && !db.Done() {
-		child := h.NewUnixfsNode()
+		child := db.NewUnixfsNode()
 		db.SetPosInfo(child, offset)
 
 		err := fillNodeRec(db, child, depth-1, offset)

importer/helpers/dagbuilder.go

@@ -7,7 +7,9 @@ import (
 	"github.com/ipfs/go-ipfs/commands/files"
 	"github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
+	ft "github.com/ipfs/go-ipfs/unixfs"
 
+	cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid"
 	node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format"
 )
 
@@ -23,6 +25,7 @@ type DagBuilderHelper struct {
 	batch     *dag.Batch
 	fullPath  string
 	stat      os.FileInfo
+	prefix    *cid.Prefix
 }
 
 type DagBuilderParams struct {
@@ -33,6 +36,9 @@ type DagBuilderParams struct {
 	// instead of using the unixfs TRaw type
 	RawLeaves bool
 
+	// CID Prefix to use if set
+	Prefix *cid.Prefix
+
 	// DAGService to write blocks to (required)
 	Dagserv dag.DAGService
 
@@ -48,6 +54,7 @@ func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper {
 		dserv:     dbp.Dagserv,
 		spl:       spl,
 		rawLeaves: dbp.RawLeaves,
+		prefix:    dbp.Prefix,
 		maxlinks:  dbp.Maxlinks,
 		batch:     dbp.Dagserv.Batch(),
 	}
@@ -103,6 +110,26 @@ func (db *DagBuilderHelper) GetDagServ() dag.DAGService {
 	return db.dserv
 }
 
+// NewUnixfsNode creates a new Unixfs node to represent a file.
+func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode {
+	n := &UnixfsNode{
+		node: new(dag.ProtoNode),
+		ufmt: &ft.FSNode{Type: ft.TFile},
+	}
+	n.SetPrefix(db.prefix)
+	return n
+}
+
+// NewUnixfsBlock creates a new Unixfs node to represent a raw data block
+func (db *DagBuilderHelper) NewUnixfsBlock() *UnixfsNode {
+	n := &UnixfsNode{
+		node: new(dag.ProtoNode),
+		ufmt: &ft.FSNode{Type: ft.TRaw},
+	}
+	n.SetPrefix(db.prefix)
+	return n
+}
+
 // FillNodeLayer will add datanodes as children to the give node until
 // at most db.indirSize ndoes are added
 //
@@ -143,7 +170,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
 			raw:     true,
 		}, nil
 	} else {
-		blk := NewUnixfsBlock()
+		blk := db.NewUnixfsBlock()
 		blk.SetData(data)
 		return blk, nil
 	}

importer/helpers/helpers.go

@@ -10,6 +10,7 @@ import (
 	pi "github.com/ipfs/go-ipfs/thirdparty/posinfo"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
+	cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid"
 	node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format"
 )
 
@@ -48,22 +49,6 @@ type UnixfsNode struct {
 	posInfo *pi.PosInfo
 }
 
-// NewUnixfsNode creates a new Unixfs node to represent a file
-func NewUnixfsNode() *UnixfsNode {
-	return &UnixfsNode{
-		node: new(dag.ProtoNode),
-		ufmt: &ft.FSNode{Type: ft.TFile},
-	}
-}
-
-// NewUnixfsBlock creates a new Unixfs node to represent a raw data block
-func NewUnixfsBlock() *UnixfsNode {
-	return &UnixfsNode{
-		node: new(dag.ProtoNode),
-		ufmt: &ft.FSNode{Type: ft.TRaw},
-	}
-}
-
 // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
 func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) {
 	mb, err := ft.FSNodeFromBytes(nd.Data())
@@ -77,6 +62,11 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) {
 	}, nil
 }
 
+// SetPrefix sets the CID Prefix
+func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) {
+	n.node.SetPrefix(prefix)
+}
+
 func (n *UnixfsNode) NumChildren() int {
 	return n.ufmt.NumChildren()
 }