feat: Support UnixFS mode and modification times in ipld dag and mfs (#…

…658)

* feat: Support UnixFS mode and modification times in ipld dag and mfs

Adds support for storing and retrieving file mode and last modification time.

Support added to:
- Files
- LinkFiles
- Webfiles
- Directories

Tar archives are supported by the parent branch.

ipld/unixfs/file/unixfile.go

@@ -23,7 +23,8 @@ type ufsDirectory struct {
 	dserv ipld.DAGService
 	dir   uio.Directory
 	size  int64
-	stat  os.FileInfo
+	mode  os.FileMode
+	mtime time.Time
 }
 
 type ufsIterator struct {
@@ -122,17 +123,11 @@ func (d *ufsDirectory) Entries() files.DirIterator {
 }
 
 func (d *ufsDirectory) Mode() os.FileMode {
-	if d.stat == nil {
-		return 0
-	}
-	return d.stat.Mode()
+	return d.mode
 }
 
 func (d *ufsDirectory) ModTime() time.Time {
-	if d.stat == nil {
-		return time.Time{}
-	}
-	return d.stat.ModTime()
+	return d.mtime
 }
 
 func (d *ufsDirectory) Size() (int64, error) {
@@ -141,28 +136,21 @@ func (d *ufsDirectory) Size() (int64, error) {
 
 type ufsFile struct {
 	uio.DagReader
-	stat os.FileInfo
 }
 
 func (f *ufsFile) Mode() os.FileMode {
-	if f.stat == nil {
-		return 0
-	}
-	return f.stat.Mode()
+	return f.DagReader.Mode()
 }
 
 func (f *ufsFile) ModTime() time.Time {
-	if f.stat == nil {
-		return time.Time{}
-	}
-	return f.stat.ModTime()
+	return f.DagReader.ModTime()
 }
 
 func (f *ufsFile) Size() (int64, error) {
 	return int64(f.DagReader.Size()), nil
 }
 
-func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, stat os.FileInfo) (files.Directory, error) {
+func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) {
 	dir, err := uio.NewDirectoryFromNode(dserv, nd)
 	if err != nil {
 		return nil, err
@@ -173,32 +161,35 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode,
 		return nil, err
 	}
 
+	fsn, err := ft.FSNodeFromBytes(nd.Data())
+	if err != nil {
+		return nil, err
+	}
+
 	return &ufsDirectory{
 		ctx:   ctx,
 		dserv: dserv,
 
-		dir:  dir,
-		size: int64(size),
-		stat: stat,
+		dir:   dir,
+		size:  int64(size),
+		mode:  fsn.Mode(),
+		mtime: fsn.ModTime(),
 	}, nil
 }
 
 func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) {
-	return NewUnixfsFileWithStat(ctx, dserv, nd, nil)
-}
-
-func NewUnixfsFileWithStat(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, stat os.FileInfo) (files.Node, error) {
 	switch dn := nd.(type) {
 	case *dag.ProtoNode:
 		fsn, err := ft.FSNodeFromBytes(dn.Data())
 		if err != nil {
 			return nil, err
 		}
+
 		if fsn.IsDir() {
-			return newUnixfsDir(ctx, dserv, dn, stat)
+			return newUnixfsDir(ctx, dserv, dn)
 		}
 		if fsn.Type() == ft.TSymlink {
-			return files.NewLinkFile(string(fsn.Data()), stat), nil
+			return files.NewSymlinkFile(string(fsn.Data()), fsn.ModTime()), nil
 		}
 
 	case *dag.RawNode:

ipld/unixfs/importer/balanced/balanced_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	mrand "math/rand"
 	"testing"
+	"time"
 
 	h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
 	uio "github.com/ipfs/boxo/ipld/unixfs/io"
@@ -26,6 +27,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, err
 		Maxlinks: h.DefaultLinksPerBlock,
 	}
 
+	return buildTestDagWithParams(spl, dbp)
+}
+
+func buildTestDagWithParams(spl chunker.Splitter, dbp h.DagBuilderParams) (*dag.ProtoNode, error) {
 	db, err := dbp.New(spl)
 	if err != nil {
 		return nil, err
@@ -335,3 +340,46 @@ func TestSeekingConsistency(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestMetadataNoData(t *testing.T) {
+	testMetadata(t, new(bytes.Buffer))
+}
+
+func TestMetadata(t *testing.T) {
+	nbytes := 3 * chunker.DefaultBlockSize
+	buf := new(bytes.Buffer)
+	_, err := io.CopyN(buf, random.NewRand(), nbytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testMetadata(t, buf)
+}
+
+func testMetadata(t *testing.T, buf *bytes.Buffer) {
+	dagserv := mdtest.Mock()
+	dbp := h.DagBuilderParams{
+		Dagserv:     dagserv,
+		Maxlinks:    h.DefaultLinksPerBlock,
+		FileMode:    0522,
+		FileModTime: time.Unix(1638111600, 76552),
+	}
+
+	nd, err := buildTestDagWithParams(chunker.DefaultSplitter(buf), dbp)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dr, err := uio.NewDagReader(context.Background(), nd, dagserv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !dr.ModTime().Equal(dbp.FileModTime) {
+		t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime)
+	}
+
+	if dr.Mode() != dbp.FileMode {
+		t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode)
+	}
+}

ipld/unixfs/importer/balanced/builder.go

@@ -130,18 +130,33 @@ import (
 //	  | Chunk 1 |   | Chunk 2 |   | Chunk 3 |
 //	  +=========+   +=========+   + - - - - +
 func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
+	var root ipld.Node
+	var err error
+
 	if db.Done() {
-		// No data, return just an empty node.
-		root, err := db.NewLeafNode(nil, ft.TFile)
-		if err != nil {
-			return nil, err
-		}
+		// No data, just create an empty node.
+		root, err = db.NewLeafNode(nil, ft.TFile)
 		// This works without Filestore support (`ProcessFileStore`).
 		// TODO: Why? Is there a test case missing?
+	} else {
+		root, err = layoutData(db)
+	}
+
+	if err != nil {
+		return nil, err
+	}
 
-		return root, db.Add(root)
+	if db.HasFileAttributes() {
+		err = db.SetFileAttributes(root)
+		if err != nil {
+			return nil, err
+		}
 	}
 
+	return root, db.Add(root)
+}
+
+func layoutData(db *h.DagBuilderHelper) (ipld.Node, error) {
 	// The first `root` will be a single leaf node with data
 	// (corner case), after that subsequent `root` nodes will
 	// always be internal nodes (with a depth > 0) that can
@@ -172,7 +187,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
 		}
 	}
 
-	return root, db.Add(root)
+	return root, nil
 }
 
 // fillNodeRec will "fill" the given internal (non-leaf) `node` with data by

ipld/unixfs/importer/helpers/dagbuilder.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"os"
+	"time"
 
 	dag "github.com/ipfs/boxo/ipld/merkledag"
 
@@ -23,13 +24,15 @@ var ErrMissingFsRef = errors.New("missing file path or URL, can't create filesto
 // DagBuilderHelper wraps together a bunch of objects needed to
 // efficiently create unixfs dag trees
 type DagBuilderHelper struct {
-	dserv      ipld.DAGService
-	spl        chunker.Splitter
-	recvdErr   error
-	rawLeaves  bool
-	nextData   []byte // the next item to return.
-	maxlinks   int
-	cidBuilder cid.Builder
+	dserv       ipld.DAGService
+	spl         chunker.Splitter
+	recvdErr    error
+	rawLeaves   bool
+	nextData    []byte // the next item to return.
+	maxlinks    int
+	cidBuilder  cid.Builder
+	fileMode    os.FileMode
+	fileModTime time.Time
 
 	// Filestore support variables.
 	// ----------------------------
@@ -62,6 +65,12 @@ type DagBuilderParams struct {
 	// DAGService to write blocks to (required)
 	Dagserv ipld.DAGService
 
+	// The unixfs file mode
+	FileMode os.FileMode
+
+	// The unixfs last modified time
+	FileModTime time.Time
+
 	// NoCopy signals to the chunker that it should track fileinfo for
 	// filestore adds
 	NoCopy bool
@@ -71,11 +80,13 @@ type DagBuilderParams struct {
 // chunker.Splitter as data source.
 func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) {
 	db := &DagBuilderHelper{
-		dserv:      dbp.Dagserv,
-		spl:        spl,
-		rawLeaves:  dbp.RawLeaves,
-		cidBuilder: dbp.CidBuilder,
-		maxlinks:   dbp.Maxlinks,
+		dserv:       dbp.Dagserv,
+		spl:         spl,
+		rawLeaves:   dbp.RawLeaves,
+		cidBuilder:  dbp.CidBuilder,
+		maxlinks:    dbp.Maxlinks,
+		fileMode:    dbp.FileMode,
+		fileModTime: dbp.FileModTime,
 	}
 	if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok {
 		db.fullPath = fi.AbsPath()
@@ -138,9 +149,9 @@ func (db *DagBuilderHelper) GetCidBuilder() cid.Builder {
 	return db.cidBuilder
 }
 
-// NewLeafNode creates a leaf node filled with data.  If rawLeaves is
-// defined then a raw leaf will be returned.  Otherwise, it will create
-// and return `FSNodeOverDag` with `fsNodeType`.
+// NewLeafNode creates a leaf node filled with data. If rawLeaves is defined
+// then a raw leaf will be returned. Otherwise, it will create and return
+// `FSNodeOverDag` with `fsNodeType`.
 func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) {
 	if len(data) > BlockSizeLimit {
 		return nil, ErrSizeLimitExceeded
@@ -161,6 +172,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType
 	// Encapsulate the data in UnixFS node (instead of a raw node).
 	fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType)
 	fsNodeOverDag.SetFileData(data)
+
 	node, err := fsNodeOverDag.Commit()
 	if err != nil {
 		return nil, err
@@ -172,9 +184,10 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType
 	return node, nil
 }
 
-// FillNodeLayer will add datanodes as children to the give node until
+// FillNodeLayer will add data-nodes as children to the given node until
 // it is full in this layer or no more data.
-// NOTE: This function creates raw data nodes so it only works
+//
+// NOTE: This function creates raw data nodes, so it only works
 // for the `trickle.Layout`.
 func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error {
 	// while we have room AND we're not done
@@ -265,6 +278,34 @@ func (db *DagBuilderHelper) Maxlinks() int {
 	return db.maxlinks
 }
 
+// HasFileAttributes will return false if Filestore is being used,
+// otherwise returns true if a file mode or last modification time is set.
+func (db *DagBuilderHelper) HasFileAttributes() bool {
+	return db.fullPath == "" && (db.fileMode != 0 || !db.fileModTime.IsZero())
+}
+
+// SetFileAttributes stores file attributes present in the `DagBuilderHelper`
+// into the associated `ft.FSNode`.
+func (db *DagBuilderHelper) SetFileAttributes(n ipld.Node) error {
+	if pn, ok := n.(*dag.ProtoNode); ok {
+		fsn, err := ft.FSNodeFromBytes(pn.Data())
+		if err != nil {
+			return err
+		}
+		fsn.SetModTime(db.fileModTime)
+		fsn.SetMode(db.fileMode)
+
+		d, err := fsn.GetBytes()
+		if err != nil {
+			return err
+		}
+
+		pn.SetData(d)
+	}
+
+	return nil
+}
+
 // FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a
 // `dag.ProtoNode`. Instead of just having a single `ipld.Node` that
 // would need to be constantly (un)packed to access and modify its
@@ -288,7 +329,7 @@ type FSNodeOverDag struct {
 }
 
 // NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode`
-// decoupled from one onther (and will continue in that way until
+// decoupled from one anonther (and will continue in that way until
 // `Commit` is called), with `fsNodeType` specifying the type of
 // the UnixFS layer node (either `File` or `Raw`).
 func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag {
@@ -374,6 +415,26 @@ func (n *FSNodeOverDag) SetFileData(fileData []byte) {
 	n.file.SetData(fileData)
 }
 
+// SetMode sets the file mode of the associated `ft.FSNode`.
+func (n *FSNodeOverDag) SetMode(mode os.FileMode) {
+	n.file.SetMode(mode)
+}
+
+// SetModTime sets the file modification time of the associated `ft.FSNode`.
+func (n *FSNodeOverDag) SetModTime(ts time.Time) {
+	n.file.SetModTime(ts)
+}
+
+// Mode returns the file mode of the associated `ft.FSNode`
+func (n *FSNodeOverDag) Mode() os.FileMode {
+	return n.file.Mode()
+}
+
+// ModTime returns the last modification time of the associated `ft.FSNode`
+func (n *FSNodeOverDag) ModTime() time.Time {
+	return n.file.ModTime()
+}
+
 // GetDagNode fills out the proper formatting for the FSNodeOverDag node
 // inside of a DAG node and returns the dag node.
 // TODO: Check if we have committed (passed the UnixFS information