From 22a5b795085dae7415442974af3af48a78d1f103 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 2 Jun 2016 12:12:43 -0700 Subject: [PATCH] rework add-mfs to not use caching License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 13 ++++++++ core/coreunix/add.go | 51 ++++++++++++++++---------------- mfs/dir.go | 14 +-------- mfs/file.go | 14 +++++++++ test/sharness/t0250-files-api.sh | 23 ++++++++++++++ unixfs/format.go | 1 + 6 files changed, 78 insertions(+), 38 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 48ca23a6455..2198dedcaad 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -10,6 +10,8 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core" + dagtest "github.com/ipfs/go-ipfs/merkledag/test" + mfs "github.com/ipfs/go-ipfs/mfs" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) @@ -156,6 +158,17 @@ You can now refer to the added file in a gateway, like so: fileAdder.Pin = dopin fileAdder.Silent = silent + if hash { + md := dagtest.Mock() + mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + fileAdder.SetMfsRoot(mr) + } + addAllAndPin := func(f files.File) error { // Iterate over each top-level file and add individually. Otherwise the // single files.File f is treated as a directory, affecting hidden file diff --git a/core/coreunix/add.go b/core/coreunix/add.go index b815e42c513..e050c0f7d3c 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -1,7 +1,6 @@ package coreunix import ( - "bytes" "fmt" "io" "io/ioutil" @@ -68,7 +67,7 @@ type AddedObject struct { } func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) { - mr, err := mfs.NewRoot(ctx, ds, newDirNode(), nil) + mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil) if err != nil { return nil, err } @@ -109,6 +108,10 @@ type Adder struct { tempRoot key.Key } +func (adder *Adder) SetMfsRoot(r *mfs.Root) { + adder.mr = r +} + // Perform the actual add & pin locally, outputting results to reader func (adder Adder) add(reader io.Reader) (*dag.Node, error) { chnk, err := chunk.FromString(reader, adder.Chunker) @@ -214,34 +217,32 @@ func (adder *Adder) Finalize() (*dag.Node, error) { return root.GetNode() } -func (adder *Adder) outputDirs(path string, fs mfs.FSNode) error { - nd, err := fs.GetNode() - if err != nil { - return err - } - - if !bytes.Equal(nd.Data, folderData) || fs.Type() != mfs.TDir { +func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error { + switch fsn := fsn.(type) { + case *mfs.File: return nil - } - - dir, ok := fs.(*mfs.Directory) - if !ok { - return fmt.Errorf("received FSNode of type TDir that was not a Directory") - } - - for _, name := range dir.ListNames() { - child, err := dir.Child(name) - if err != nil { - return err + case *mfs.Directory: + for _, name := range fsn.ListNames() { + child, err := fsn.Child(name) + if err != nil { + return err + } + + childpath := gopath.Join(path, name) + err = adder.outputDirs(childpath, child) + if err != nil { + return err + } } - - err = adder.outputDirs(gopath.Join(path, name), child) + nd, err := fsn.GetNode() if err != nil { return err } - } - return outputDagnode(adder.Out, path, nd) + return outputDagnode(adder.Out, path, nd) + default: + return fmt.Errorf("unrecognized fsn type: %#v", fsn) + } } // Add builds a merkledag from the a reader, pinning all objects to the local @@ -488,7 +489,7 @@ func NewMemoryDagService() dag.DAGService { } // TODO: generalize this to more than unix-fs nodes. -func newDirNode() *dag.Node { +func NewDirNode() *dag.Node { return &dag.Node{Data: unixfs.FolderPBData()} } diff --git a/mfs/dir.go b/mfs/dir.go index ba14464ae76..fba61ea4c49 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -131,7 +131,7 @@ func (d *Directory) cacheNode(name string, nd *dag.Node) (FSNode, error) { ndir := NewDirectory(d.ctx, name, nd, d, d.dserv) d.childDirs[name] = ndir return ndir, nil - case ufspb.Data_File, ufspb.Data_Raw: + case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink: nfi, err := NewFile(name, nd, d, d.dserv) if err != nil { return nil, err @@ -338,18 +338,6 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error { } d.modTime = time.Now() - - if len(nd.Links) == 0 { - nfi, err := NewFile(name, nd, d, d.dserv) - if err != nil { - return err - } - d.files[name] = nfi - } else { - ndir := NewDirectory(d.ctx, name, nd, d, d.dserv) - d.childDirs[name] = ndir - } - return nil } diff --git a/mfs/file.go b/mfs/file.go index 578da98f6dd..216bdfa7516 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -45,6 +45,20 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { node := fi.node fi.nodelk.Unlock() + fsn, err := ft.FSNodeFromBytes(node.Data) + if err != nil { + return nil, err + } + + switch fsn.Type { + default: + return nil, fmt.Errorf("unsupported fsnode type for 'file'") + case ft.TSymlink: + return nil, fmt.Errorf("symlinks not yet supported") + case ft.TFile, ft.TRaw: + // OK case + } + switch flags { case OpenReadOnly: fi.desclock.RLock() diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index 167ef3f9c4d..31e91c04e96 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -437,6 +437,29 @@ test_files_api() { test_expect_success "child dir looks right" ' verify_dir_contents / ' + + # test for https://github.com/ipfs/go-ipfs/issues/2654 + test_expect_success "create and remove dir" ' + ipfs files mkdir /test_dir && + ipfs files rm -r "/test_dir" + ' + + test_expect_success "create test file" ' + echo "content" | ipfs files write -e "/test_file" + ' + + test_expect_success "copy test file onto test dir" ' + ipfs files cp "/test_file" "/test_dir" + ' + + test_expect_success "test /test_dir" ' + ipfs files stat "/test_dir" | grep -q "^Type: file" + ' + + test_expect_success "clean up /test_dir and /test_file" ' + ipfs files rm -r /test_dir && + ipfs files rm -r /test_file + ' } # test offline and online diff --git a/unixfs/format.go b/unixfs/format.go index 6acb41050c2..f279a8843b3 100644 --- a/unixfs/format.go +++ b/unixfs/format.go @@ -15,6 +15,7 @@ const ( TFile = pb.Data_File TDirectory = pb.Data_Directory TMetadata = pb.Data_Metadata + TSymlink = pb.Data_Symlink ) var ErrMalformedFileFormat = errors.New("malformed data in file format")