From db078943a5a1c634dedc87167ea922631a7420b8 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 21:23:22 -0700 Subject: [PATCH] feat: Support storing UnixFS 1.5 Mode and ModTime Replaces #7754 written by @kstuart - https://github.com/ipfs/boxo/pull/653 - https://github.com/ipfs/boxo/pull/658 - [X] Can `ipfs add` with preserved mode and/or last modification time - [X] on files - [X] on directories - [X] Can `ipfs add` with custom mode and/or last modification time - [X] on files - [X] on directories - [X] Can `ipfs get` restoring mode and/or last modification time - [X] on files - [X] on directories - [X] in archives - [X] Can `ipfs files chmod` to change mode - [X] on files - [X] on directories - [X] Can `ipfs files touch` to change last modification time - [X] on files - [X] on directories - [X] Automatically update the last modification time when file data is changed or truncated (e.g. `ipfs files write`) - [X] Can add files and directories with mode and/or modification time using multipart-form data - [X] `ipfs files stat` reports mode and last modification time **Note:** - [X] Adds support to `kubo/core/rpc` (may require additional tests). - ~ipfs/interface-go-ipfs-core/pull/66~ replace by this PR - ~ipfs/go-unixfs/pull/85~ replaced by: https://github.com/ipfs/boxo/pull/658 - ~ipfs/go-mfs/pull/93~ replaced by: https://github.com/ipfs/boxo/pull/658 - ~ipfs/go-ipfs-files/pull/31~ replaced by: https://github.com/ipfs/boxo/pull/653 - ~ipfs/tar-utils/pull/11~ replaced by: https://github.com/ipfs/boxo/pull/653 - When adding files and directories without opting to store a mode or modification time the same CIDs are generated that would have been created before this feature was implemented (opt-in). - The Go runtime currently has no native support for restoring file mode and modification time on symbolic-links, support for restoring the last modification time has been added for Linux distributions and the following BSDs: freebsd, netbsd, openbsd, dragonflybsd. - Automatically updating a modification time will only occur if a modification time was previously stored. - When creating an archive, for compatibility, time resolution is to the second; Nanoseconds are not supported. The `ipfs add` options `--preserve-mode` and `--preserve-mtime` are used to store the original mode and last modified time of the file being added, the options `--mode`, `--mtime` and `--mtime-nsecs` are used to store custom values, a custom value of 0 is a no-op as is providing `--mtime-nsecs` without `--mtime`. The preserve flags and custom options are mutually exclusive, if both are provided the custom options take precedence. --- Closes #6920 --- client/rpc/apifile.go | 58 +++- client/rpc/unixfs.go | 18 +- core/commands/add.go | 93 +++++- core/commands/commands_test.go | 2 + core/commands/files.go | 204 +++++++++++-- core/commands/get.go | 9 +- core/coreapi/unixfs.go | 4 + core/coreiface/options/unixfs.go | 55 ++++ core/coreiface/unixfs.go | 12 +- core/coreunix/add.go | 52 +++- core/node/storage.go | 2 +- go.mod | 2 +- go.sum | 4 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 6 +- test/sharness/t0047-add-mode-mtime.sh | 422 ++++++++++++++++++++++++++ test/sharness/t0250-files-api.sh | 4 + 17 files changed, 884 insertions(+), 65 deletions(-) create mode 100644 test/sharness/t0047-add-mode-mtime.sh diff --git a/client/rpc/apifile.go b/client/rpc/apifile.go index 7a54995b1819..80f8bea0a436 100644 --- a/client/rpc/apifile.go +++ b/client/rpc/apifile.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "io" + "os" + "time" "github.com/ipfs/boxo/files" unixfs "github.com/ipfs/boxo/ipld/unixfs" @@ -24,9 +26,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) } var stat struct { - Hash string - Type string - Size int64 // unixfs size + Hash string + Type string + Size int64 // unixfs size + Mode os.FileMode + Mtime int64 + MtimeNsecs int } err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { @@ -35,9 +40,9 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) switch stat.Type { case "file": - return api.getFile(ctx, p, stat.Size) + return api.getFile(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs) case "directory": - return api.getDir(ctx, p, stat.Size) + return api.getDir(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs) default: return nil, fmt.Errorf("unsupported file type '%s'", stat.Type) } @@ -49,6 +54,9 @@ type apiFile struct { size int64 path path.Path + mode os.FileMode + mtime time.Time + r *Response at int64 } @@ -128,17 +136,31 @@ func (f *apiFile) Close() error { return nil } +func (f *apiFile) Mode() os.FileMode { + return f.mode +} + +func (f *apiFile) ModTime() time.Time { + return f.mtime +} + func (f *apiFile) Size() (int64, error) { return f.size, nil } -func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime int64, mtimeNsecs int) (files.Node, error) { f := &apiFile{ ctx: ctx, core: api.core(), size: size, path: p, } + if mode != 0 { + f.mode = os.FileMode(mode) + } + if mtime != 0 { + f.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC() + } return f, f.reset() } @@ -195,13 +217,13 @@ func (it *apiIter) Next() bool { switch it.cur.Type { case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: - it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size)) + it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs) if err != nil { it.err = err return false } case unixfs.TFile: - it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size)) + it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs) if err != nil { it.err = err return false @@ -223,6 +245,9 @@ type apiDir struct { size int64 path path.Path + mode os.FileMode + mtime time.Time + dec *json.Decoder } @@ -230,6 +255,14 @@ func (d *apiDir) Close() error { return nil } +func (d *apiDir) Mode() os.FileMode { + return d.mode +} + +func (d *apiDir) ModTime() time.Time { + return d.mtime +} + func (d *apiDir) Size() (int64, error) { return d.size, nil } @@ -242,7 +275,7 @@ func (d *apiDir) Entries() files.DirIterator { } } -func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime int64, mtimeNsecs int) (files.Node, error) { resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) @@ -262,6 +295,13 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (file dec: json.NewDecoder(resp.Output), } + if mode != 0 { + d.mode = os.FileMode(mode) + } + if mtime != 0 { + d.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC() + } + return d, nil } diff --git a/client/rpc/unixfs.go b/client/rpc/unixfs.go index 501e8d02511e..a585a27b8013 100644 --- a/client/rpc/unixfs.go +++ b/client/rpc/unixfs.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "os" "github.com/ipfs/boxo/files" unixfs "github.com/ipfs/boxo/ipld/unixfs" @@ -22,6 +23,10 @@ type addEvent struct { Hash string `json:",omitempty"` Bytes int64 `json:",omitempty"` Size string `json:",omitempty"` + + Mode os.FileMode `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` } type UnixfsAPI HttpApi @@ -94,9 +99,12 @@ loop: if options.Events != nil { ifevt := &iface.AddEvent{ - Name: out.Name, - Size: out.Size, - Bytes: out.Bytes, + Name: out.Name, + Size: out.Size, + Bytes: out.Bytes, + Mode: out.Mode, + Mtime: out.Mtime, + MtimeNsecs: out.MtimeNsecs, } if out.Hash != "" { @@ -129,6 +137,10 @@ type lsLink struct { Size uint64 Type unixfs_pb.Data_DataType Target string + + Mode os.FileMode + Mtime int64 + MtimeNsecs int } type lsObject struct { diff --git a/core/commands/add.go b/core/commands/add.go index 94a5a0f51f5d..8d90cc0e0641 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -6,7 +6,9 @@ import ( "io" "os" gopath "path" + "strconv" "strings" + "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" @@ -25,11 +27,31 @@ import ( // ErrDepthLimitExceeded indicates that the max depth has been exceeded. var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") +type TimeParts struct { + t *time.Time +} + +func (t TimeParts) MarshalJSON() ([]byte, error) { + return t.t.MarshalJSON() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// The time is expected to be a quoted string in RFC 3339 format. +func (t *TimeParts) UnmarshalJSON(data []byte) (err error) { + // Fractional seconds are handled implicitly by Parse. + tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data)) + *t = TimeParts{&tt} + return +} + type AddEvent struct { - Name string - Hash string `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` + Mode string `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` } const ( @@ -50,6 +72,12 @@ const ( inlineOptionName = "inline" inlineLimitOptionName = "inline-limit" toFilesOptionName = "to-files" + + preserveModeOptionName = "preserve-mode" + preserveMtimeOptionName = "preserve-mtime" + modeOptionName = "mode" + mtimeOptionName = "mtime" + mtimeNsecsOptionName = "mtime-nsecs" ) const adderOutChanSize = 8 @@ -166,6 +194,12 @@ See 'dag export' and 'dag import' for more information. cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."), + + cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries"), + cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries"), + cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries"), + cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch)"), + cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -217,6 +251,11 @@ See 'dag export' and 'dag import' for more information. inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) + preserveMode, _ := req.Options[preserveModeOptionName].(bool) + preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool) + mode, _ := req.Options[modeOptionName].(uint) + mtime, _ := req.Options[mtimeOptionName].(int64) + mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) if chunker == "" { chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) @@ -272,6 +311,19 @@ See 'dag export' and 'dag import' for more information. options.Unixfs.Progress(progress), options.Unixfs.Silent(silent), + + options.Unixfs.PreserveMode(preserveMode), + options.Unixfs.PreserveMtime(preserveMtime), + } + + if mode != 0 { + opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) + } + + if mtime != 0 { + opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) + } else if mtimeNsecs != 0 { + fmt.Println("option", mtimeNsecsOptionName, "ignored as no valid", mtimeOptionName, "value provided") } if cidVerSet { @@ -383,12 +435,33 @@ See 'dag export' and 'dag import' for more information. output.Name = gopath.Join(addit.Name(), output.Name) } - if err := res.Emit(&AddEvent{ - Name: output.Name, - Hash: h, - Bytes: output.Bytes, - Size: output.Size, - }); err != nil { + output.Mode = addit.Node().Mode() + if ts := addit.Node().ModTime(); !ts.IsZero() { + output.Mtime = addit.Node().ModTime().Unix() + output.MtimeNsecs = addit.Node().ModTime().Nanosecond() + } + + addEvent := AddEvent{ + Name: output.Name, + Hash: h, + Bytes: output.Bytes, + Size: output.Size, + Mtime: output.Mtime, + MtimeNsecs: output.MtimeNsecs, + } + + if output.Mode != 0 { + addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8) + } + + if output.Mtime > 0 { + addEvent.Mtime = output.Mtime + if output.MtimeNsecs > 0 { + addEvent.MtimeNsecs = output.MtimeNsecs + } + } + + if err := res.Emit(&addEvent); err != nil { return err } } diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 018b6734e79f..b04a5459b50a 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -89,6 +89,8 @@ func TestCommands(t *testing.T) { "/files/rm", "/files/stat", "/files/write", + "/files/chmod", + "/files/touch", "/filestore", "/filestore/dups", "/filestore/ls", diff --git a/core/commands/files.go b/core/commands/files.go index 12891a7301d9..a8d6f1397c66 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -2,13 +2,16 @@ package commands import ( "context" + "encoding/json" "errors" "fmt" "io" "os" gopath "path" "sort" + "strconv" "strings" + "time" humanize "github.com/dustin/go-humanize" "github.com/ipfs/kubo/config" @@ -81,6 +84,8 @@ operations. "rm": filesRmCmd, "flush": filesFlushCmd, "chcid": filesChcidCmd, + "chmod": filesChmodCmd, + "touch": filesTouchCmd, }, } @@ -105,6 +110,43 @@ type statOutput struct { WithLocality bool `json:",omitempty"` Local bool `json:",omitempty"` SizeLocal uint64 `json:",omitempty"` + Mode uint32 `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` +} + +func (s *statOutput) MarshalJSON() ([]byte, error) { + type so statOutput + out := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if s.Mode != 0 { + out.Mode = fmt.Sprintf("%04o", s.Mode) + } + return json.Marshal(out) +} + +func (s *statOutput) UnmarshalJSON(data []byte) error { + var err error + type so statOutput + tmp := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + if tmp.Mode != "" { + mode, err := strconv.ParseUint(tmp.Mode, 8, 32) + if err == nil { + s.Mode = uint32(mode) + } + } + return err } const ( @@ -112,7 +154,9 @@ const ( Size: CumulativeSize: ChildBlocks: -Type: ` +Type: +Mode: () +Mtime: ` filesFormatOptionName = "format" filesSizeOptionName = "size" filesWithLocalOptionName = "with-local" @@ -199,12 +243,26 @@ var filesStatCmd = &cmds.Command{ }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *statOutput) error { + var mode os.FileMode + if out.Mode != 0 { + mode = os.FileMode(out.Mode) + } + var mtime string + if out.Mtime > 0 { + mtime = time.Unix(out.Mtime, int64(out.MtimeNsecs)).UTC().Format("2 Jan 2006, 15:04:05 MST") + } + s, _ := statGetFormatOptions(req) s = strings.Replace(s, "", out.Hash, -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Size), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) s = strings.Replace(s, "", out.Type, -1) + s = strings.Replace(s, "", strings.ToLower(mode.String()), -1) + s = strings.Replace(s, "", mtime, -1) + s = strings.Replace(s, "", strconv.FormatInt(out.Mtime, 10), -1) + s = strings.Replace(s, "", strconv.Itoa(out.MtimeNsecs), -1) + s = strings.Replace(s, "", "0"+strconv.FormatInt(int64(out.Mode&0x1FF), 8), -1) fmt.Fprintln(w, s) @@ -254,28 +312,7 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { switch n := nd.(type) { case *dag.ProtoNode: - d, err := ft.FSNodeFromBytes(n.Data()) - if err != nil { - return nil, err - } - - var ndtype string - switch d.Type() { - case ft.TDirectory, ft.THAMTShard: - ndtype = "directory" - case ft.TFile, ft.TMetadata, ft.TRaw: - ndtype = "file" - default: - return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) - } - - return &statOutput{ - Hash: enc.Encode(c), - Blocks: len(nd.Links()), - Size: d.FileSize(), - CumulativeSize: cumulsize, - Type: ndtype, - }, nil + return statProtoNode(n, enc, c, cumulsize) case *dag.RawNode: return &statOutput{ Hash: enc.Encode(c), @@ -289,6 +326,44 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { } } +func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize uint64) (*statOutput, error) { + d, err := ft.FSNodeFromBytes(n.Data()) + if err != nil { + return nil, err + } + + stat := statOutput{ + Hash: enc.Encode(cid), + Blocks: len(n.Links()), + Size: d.FileSize(), + CumulativeSize: cumulsize, + } + + switch d.Type() { + case ft.TDirectory, ft.THAMTShard: + stat.Type = "directory" + case ft.TFile, ft.TSymlink, ft.TMetadata, ft.TRaw: + stat.Type = "file" + default: + return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) + } + + if mode := d.Mode(); mode != 0 { + stat.Mode = uint32(mode) + } else if d.Type() == ft.TSymlink { + stat.Mode = uint32(os.ModeSymlink | 0x1FF) + } + + if mt := d.ModTime(); !mt.IsZero() { + stat.Mtime = mt.Unix() + if ns := mt.Nanosecond(); ns > 0 { + stat.MtimeNsecs = ns + } + } + + return &stat, nil +} + func walkBlock(ctx context.Context, dagserv ipld.DAGService, nd ipld.Node) (bool, uint64, error) { // Start with the block data size sizeLocal := uint64(len(nd.RawData())) @@ -341,7 +416,7 @@ $ ipfs add --quieter --pin=false $ ipfs files cp /ipfs/ /your/desired/mfs/path If you wish to fully copy content from a different IPFS peer into MFS, do not -forget to force IPFS to fetch to full DAG after doing the "cp" operation. i.e: +forget to force IPFS to fetch the full DAG after doing a "cp" operation. i.e: $ ipfs files cp /ipfs/ /your/desired/mfs/path $ ipfs pin add @@ -1313,3 +1388,84 @@ func getParentDir(root *mfs.Root, dir string) (*mfs.Directory, error) { } return pdir, nil } + +var filesChmodCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Change optional POSIX mode permissions", + ShortDescription: ` +The mode argument must be specified in Unix numeric notation. + + $ ipfs files chmod 0644 /foo + $ ipfs files stat /foo + ... + Type: file + Mode: -rw-r--r-- + ... +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("mode", true, false, "mode to apply to node"), + cmds.StringArg("path", true, false, "Path to target node"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + path, err := checkPath(req.Arguments[1]) + if err != nil { + return err + } + + mode, err := strconv.ParseInt(req.Arguments[0], 8, 32) + if err != nil { + return err + } + + return mfs.Chmod(nd.FilesRoot, path, os.FileMode(mode)) + }, +} + +var filesTouchCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Change optional POSIX modification times.", + ShortDescription: ` +Examples: + # set modification time to now. + $ ipfs files touch /foo + # set a custom modification time. + $ ipfs files touch --mtime=1630937926 /foo +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("path", true, false, "Path of target to update."), + }, + Options: []cmds.Option{ + cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or since the Unix Epoch to apply to created UnixFS entries."), + cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + path, err := checkPath(req.Arguments[0]) + if err != nil { + return err + } + + mtime, _ := req.Options[mtimeOptionName].(int64) + nsecs, _ := req.Options[mtimeNsecsOptionName].(uint) + + var ts time.Time + if mtime != 0 { + ts = time.Unix(mtime, int64(nsecs)).UTC() + } else { + ts = time.Now().UTC() + } + + return mfs.Touch(nd.FilesRoot, path, ts) + }, +} diff --git a/core/commands/get.go b/core/commands/get.go index 5b64c281bbd2..12a3ea8ca26d 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -1,6 +1,7 @@ package commands import ( + gotar "archive/tar" "bufio" "compress/gzip" "errors" @@ -331,7 +332,8 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R closeGzwAndPipe() // everything seems to be ok }() } else { - // the case for 1. archive, and 2. not archived and not compressed, in which tar is used anyway as a transport format + // the case for 1. archive, and 2. not archived and not compressed, in + // which tar is used anyway as a transport format // construct the tar writer w, err := files.NewTarWriter(maybeGzw) @@ -339,6 +341,11 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R return nil, err } + // if not creating an archive set the format to PAX in order to preserve nanoseconds + if !archive { + w.SetFormat(gotar.FormatPAX) + } + go func() { // write all the nodes recursively if err := w.WriteFile(f, filename); checkErrAndClosePipe(err) { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 860574945d71..ddc8308ebe12 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -130,6 +130,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.RawLeaves = settings.RawLeaves fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix + fileAdder.PreserveMode = settings.PreserveMode + fileAdder.PreserveMtime = settings.PreserveMtime + fileAdder.FileMode = settings.Mode + fileAdder.FileMtime = settings.Mtime switch settings.Layout { case options.BalancedLayout: diff --git a/core/coreiface/options/unixfs.go b/core/coreiface/options/unixfs.go index f00fffb87b07..c837ec1b2db0 100644 --- a/core/coreiface/options/unixfs.go +++ b/core/coreiface/options/unixfs.go @@ -3,6 +3,8 @@ package options import ( "errors" "fmt" + "os" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" cid "github.com/ipfs/go-cid" @@ -36,6 +38,11 @@ type UnixfsAddSettings struct { Events chan<- interface{} Silent bool Progress bool + + PreserveMode bool + PreserveMtime bool + Mode os.FileMode + Mtime time.Time } type UnixfsLsSettings struct { @@ -69,6 +76,11 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Events: nil, Silent: false, Progress: false, + + PreserveMode: false, + PreserveMtime: false, + Mode: 0, + Mtime: time.Time{}, } for _, opt := range opts { @@ -106,6 +118,14 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, } } + if !options.Mtime.IsZero() && options.PreserveMtime { + options.PreserveMtime = false + } + + if options.Mode != 0 && options.PreserveMode { + options.PreserveMode = false + } + // cidV1 -> raw blocks (by default) if options.CidVersion > 0 && !options.RawLeavesSet { options.RawLeaves = true @@ -293,3 +313,38 @@ func (unixfsOpts) UseCumulativeSize(use bool) UnixfsLsOption { return nil } } + +// PreserveMode tells the adder to store the file permissions +func (unixfsOpts) PreserveMode(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.PreserveMode = enable + return nil + } +} + +// PreserveMtime tells the adder to store the file modification time +func (unixfsOpts) PreserveMtime(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.PreserveMtime = enable + return nil + } +} + +// Mode represents a unix file mode +func (unixfsOpts) Mode(mode os.FileMode) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Mode = mode + return nil + } +} + +// Mtime represents a unix file mtime +func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + if nsecs > 999999999 { + return errors.New("mtime nanoseconds must be in range [1, 999999999]") + } + settings.Mtime = time.Unix(seconds, int64(nsecs)) + return nil + } +} diff --git a/core/coreiface/unixfs.go b/core/coreiface/unixfs.go index d0dc4d8ce095..e6f1fa6b8f65 100644 --- a/core/coreiface/unixfs.go +++ b/core/coreiface/unixfs.go @@ -2,6 +2,7 @@ package iface import ( "context" + "os" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" @@ -10,10 +11,13 @@ import ( ) type AddEvent struct { - Name string - Path path.ImmutablePath `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` + Name string + Path path.ImmutablePath `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` + Mode os.FileMode `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` } // FileType is an enum of possible UnixFS file types. diff --git a/core/coreunix/add.go b/core/coreunix/add.go index a8d7e5982f0c..67f4216065e5 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "io" + "os" gopath "path" "strconv" + "time" bstore "github.com/ipfs/boxo/blockstore" chunker "github.com/ipfs/boxo/chunker" @@ -81,6 +83,11 @@ type Adder struct { tempRoot cid.Cid CidBuilder cid.Builder liveNodes uint64 + + PreserveMode bool + PreserveMtime bool + FileMode os.FileMode + FileMtime time.Time } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -113,11 +120,13 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { } params := ihelper.DagBuilderParams{ - Dagserv: adder.bufferedDS, - RawLeaves: adder.RawLeaves, - Maxlinks: ihelper.DefaultLinksPerBlock, - NoCopy: adder.NoCopy, - CidBuilder: adder.CidBuilder, + Dagserv: adder.bufferedDS, + RawLeaves: adder.RawLeaves, + Maxlinks: ihelper.DefaultLinksPerBlock, + NoCopy: adder.NoCopy, + CidBuilder: adder.CidBuilder, + FileMode: adder.FileMode, + FileModTime: adder.FileMtime, } db, err := params.New(chnk) @@ -359,6 +368,14 @@ func (adder *Adder) addFileNode(ctx context.Context, path string, file files.Nod return err } + if adder.PreserveMtime { + adder.FileMtime = file.ModTime() + } + + if adder.PreserveMode { + adder.FileMode = file.Mode() + } + if adder.liveNodes >= liveCacheSize { // TODO: A smarter cache that uses some sort of lru cache with an eviction handler mr, err := adder.mfsRoot() @@ -391,6 +408,18 @@ func (adder *Adder) addSymlink(path string, l *files.Symlink) error { return err } + if !adder.FileMtime.IsZero() { + fsn, err := unixfs.FSNodeFromBytes(sdata) + if err != nil { + return err + } + + fsn.SetModTime(adder.FileMtime) + if sdata, err = fsn.GetBytes(); err != nil { + return err + } + } + dagnode := dag.NodeWithData(sdata) err = dagnode.SetCidBuilder(adder.CidBuilder) if err != nil { @@ -429,6 +458,17 @@ func (adder *Adder) addFile(path string, file files.File) error { func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error { log.Infof("adding directory: %s", path) + // if we need to store mode or modification time then create a new root which includes that data + if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { + nd := unixfs.EmptyDirNodeWithStat(adder.FileMode, adder.FileMtime) + nd.SetCidBuilder(adder.CidBuilder) + mr, err := mfs.NewRoot(ctx, adder.dagService, nd, nil) + if err != nil { + return err + } + adder.SetMfsRoot(mr) + } + if !(toplevel && path == "") { mr, err := adder.mfsRoot() if err != nil { @@ -438,6 +478,8 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory Mkparents: true, Flush: false, CidBuilder: adder.CidBuilder, + Mode: adder.FileMode, + ModTime: adder.FileMtime, }) if err != nil { return err diff --git a/core/node/storage.go b/core/node/storage.go index a303ddc23a34..aedf0ee6a1db 100644 --- a/core/node/storage.go +++ b/core/node/storage.go @@ -56,7 +56,7 @@ func GcBlockstoreCtor(bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockst return } -// GcBlockstoreCtor wraps GcBlockstore and adds Filestore support +// FilestoreBlockstoreCtor wraps GcBlockstore and adds Filestore support func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) { gclocker = blockstore.NewGCLocker() diff --git a/go.mod b/go.mod index 233bf5976c02..d1f932d6aaf5 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c - github.com/ipfs/boxo v0.22.0 + github.com/ipfs/boxo v0.22.1-0.20240813034811-1b5bb03a0b96 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index d3296bd39ee6..2eae1e222d93 100644 --- a/go.sum +++ b/go.sum @@ -330,8 +330,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78= -github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw= +github.com/ipfs/boxo v0.22.1-0.20240813034811-1b5bb03a0b96 h1:cnIzVgFxh18/8yQdsN7WUkK8Mejbtg5jf7DW5sVPPZc= +github.com/ipfs/boxo v0.22.1-0.20240813034811-1b5bb03a0b96/go.mod h1:0KqWXIrHOCXfraT2sUdduz29UM0W7x342mZJe9XIKjs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index dd32ac5e805e..8c46e11a105a 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -113,7 +113,7 @@ require ( github.com/hexops/gotextdiff v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.22.0 // indirect + github.com/ipfs/boxo v0.22.1-0.20240813034811-1b5bb03a0b96 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index d4730e09e2a5..c82b4639fc44 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -280,8 +280,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78= -github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw= +github.com/ipfs/boxo v0.22.1-0.20240813034811-1b5bb03a0b96 h1:cnIzVgFxh18/8yQdsN7WUkK8Mejbtg5jf7DW5sVPPZc= +github.com/ipfs/boxo v0.22.1-0.20240813034811-1b5bb03a0b96/go.mod h1:0KqWXIrHOCXfraT2sUdduz29UM0W7x342mZJe9XIKjs= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -292,8 +292,6 @@ github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0M github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= -github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh new file mode 100644 index 000000000000..535cafc94121 --- /dev/null +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -0,0 +1,422 @@ +#!/usr/bin/env bash + +test_description="Test storing and retrieving mode and mtime" + +. lib/test-lib.sh + +test_init_ipfs + +HASH_NO_PRESERVE=QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH + +PRESERVE_MTIME=1604320482 +PRESERVE_MODE="0640" +HASH_PRESERVE_MODE=QmQLgxypSNGNFTuUPGCecq6dDEjb6hNB5xSyVmP3cEuNtq +HASH_PRESERVE_MTIME=QmQ6kErEW8kztQFV8vbwNU8E4dmtGsYpRiboiLxUEwibvj +HASH_PRESERVE_LINK_MTIME=QmbJwotgtr84JxcnjpwJ86uZiyMoxbZuNH4YrdJMypkYaB +HASH_PRESERVE_MODE_AND_MTIME=QmYkvboLsvLFcSYmqVJRxvBdYRQLroLv9kELf3LRiCqBri + +CUSTOM_MTIME=1603539720 +CUSTOM_MTIME_NSECS=54321 +CUSTOM_MODE="0764" +HASH_CUSTOM_MODE=QmchD3BN8TQ3RW6jPLxSaNkqvfuj7syKhzTRmL4EpyY1Nz +HASH_CUSTOM_MTIME=QmT3aY4avDcYXCWpU8CJzqUkW7YEuEsx36S8cTNoLcuK1B +HASH_CUSTOM_MTIME_NSECS=QmaKH8H5rXBUBCX4vdxi7ktGQEL7wejV7L9rX2qpZjwncz +HASH_CUSTOM_MODE_AND_MTIME=QmUkxrtBA8tPjwCYz1HrsoRfDz6NgKut3asVeHVQNH4C8L +HASH_CUSTOM_LINK_MTIME=QmV1Uot2gy4bhY9yvYiZxhhchhyYC6MKKoGV1XtWNmpCLe +HASH_CUSTOM_LINK_MTIME_NSECS=QmPHYCxYvvHj6VxiPNJ3kXxcPsnJLDYUJqsDJWjvytmrmY + +mk_name() { + tr -dc '[:alnum:]'> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && + echo "Mode: ----------" >> file1stat_expect && + echo "Mtime: " >> file1stat_expect && test_cmp file1stat_expect file1stat_actual ' @@ -243,6 +245,8 @@ test_files_api() { echo "Size: 4" >> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && + echo "Mode: ----------" >> file1stat_expect && + echo "Mtime: " >> file1stat_expect && test_cmp file1stat_expect file1stat_actual '