diff --git a/cmd/fsck.go b/cmd/fsck.go index 63c8bd8eea3d..9fe3d8fec5ef 100644 --- a/cmd/fsck.go +++ b/cmd/fsck.go @@ -43,18 +43,41 @@ It scans all objects in data storage and slices in metadata, comparing them to s lost object or broken file. Examples: -$ juicefs fsck redis://localhost`, +$ juicefs fsck redis://localhost + +# Repair broken directories +$ juicefs fsck redis://localhost --path /d1/d2 --repair`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "path", + Usage: "absolute path within JuiceFS to check", + }, + &cli.BoolFlag{ + Name: "repair", + Usage: "repair specified path if it's broken", + }, + }, } } func fsck(ctx *cli.Context) error { setup(ctx, 1) + if ctx.Bool("repair") && ctx.String("path") == "" { + logger.Fatalf("Please provide the path to repair with `--path` option") + } removePassword(ctx.Args().Get(0)) m := meta.NewClient(ctx.Args().Get(0), &meta.Config{Retries: 10, Strict: true}) format, err := m.Load(true) if err != nil { logger.Fatalf("load setting: %s", err) } + var c = meta.NewContext(0, 0, []uint32{0}) + if p := ctx.String("path"); p != "" { + if !strings.HasPrefix(p, "/") { + logger.Fatalf("File path should be the absolute path within JuiceFS") + } + return m.Check(c, p, ctx.Bool("repair")) + } chunkConf := chunk.Config{ BlockSize: format.BlockSize * 1024, @@ -106,7 +129,6 @@ func fsck(ctx *cli.Context) error { // List all slices in metadata engine sliceCSpin := progress.AddCountSpinner("Listed slices") - var c = meta.NewContext(0, 0, []uint32{0}) slices := make(map[meta.Ino][]meta.Slice) r := m.ListSlices(c, slices, false, sliceCSpin.Increment) if r != 0 { diff --git a/cmd/fsck_test.go b/cmd/fsck_test.go index b0b19a7984a0..3f97481dc666 100644 --- a/cmd/fsck_test.go +++ b/cmd/fsck_test.go @@ -35,4 +35,7 @@ func TestFsck(t *testing.T) { if err := Main([]string{"", "fsck", testMeta}); err != nil { t.Fatalf("fsck failed: %s", err) } + if err := Main([]string{"", "fsck", testMeta, "--path", "/f3.txt"}); err != nil { + t.Fatalf("fsck failed: %s", err) + } } diff --git a/cmd/main.go b/cmd/main.go index 4a5c1904bd96..7487d29f3d1a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,6 +23,7 @@ import ( "os" "strconv" "strings" + "syscall" "github.com/erikdubbelboer/gspt" "github.com/google/gops/agent" @@ -82,7 +83,11 @@ func Main(args []string) error { args = []string{"mount", "--help"} } } - return app.Run(reorderOptions(app, args)) + err := app.Run(reorderOptions(app, args)) + if errno, ok := err.(syscall.Errno); ok && errno == 0 { + err = nil + } + return err } func calledViaMount(args []string) bool { diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 5d89d835595f..f48603b6e589 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -19,6 +19,7 @@ package meta import ( "encoding/json" "fmt" + "path" "runtime" "sort" "strings" @@ -73,6 +74,7 @@ type engine interface { doSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno doRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno doGetParents(ctx Context, inode Ino) map[Ino]int + doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno GetSession(sid uint64, detail bool) (*Session, error) } @@ -951,6 +953,52 @@ func (m *baseMeta) GetParents(ctx Context, inode Ino) map[Ino]int { } } +func (m *baseMeta) Check(ctx Context, fpath string, repair bool) (st syscall.Errno) { + var attr Attr + var inode Ino + var parent Ino = 1 + ps := strings.Split(fpath, "/") + for i, name := range ps { + if len(name) == 0 { + continue + } + if st = m.Lookup(ctx, parent, name, &inode, &attr); st != 0 { + logger.Errorf("Lookup parent %d name %s: %s", parent, name, st) + return + } + if attr.Full { + parent = inode + continue + } + + // missing attribute + p := "/" + path.Join(ps[:i+1]...) + if attr.Typ != TypeDirectory { // TODO: determine file size? + logger.Errorf("Path %s (inode %d type %d) cannot be auto-repaired, you have to repair it manually or remove it", p, inode, attr.Typ) + } else if !repair { + logger.Warnf("Path %s (inode %d) can be repaired, please re-check with 'repair' enabled", p, inode) + } else { // repair directory inode + now := time.Now().Unix() + attr.Mode = 0644 + attr.Uid = ctx.Uid() + attr.Gid = ctx.Gid() + attr.Atime = now + attr.Mtime = now + attr.Ctime = now + attr.Length = 4 << 10 + attr.Parent = parent + if st = m.en.doRepair(ctx, inode, &attr); st == 0 { + logger.Infof("Path %s (inode %d) is successfully repaired", p, inode) + } else { + logger.Errorf("Repair path %s inode %d: %s", p, inode, st) + } + } + return // handle one missing inode at a time + } + logger.Infof("Path %s inode %d is valid", fpath, parent) + return +} + func (m *baseMeta) fileDeleted(opened bool, inode Ino, length uint64) { if opened { m.Lock() diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index d38ae6fc7e12..7cab97e717b4 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -349,6 +349,8 @@ type Meta interface { ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno // Remove all files and directories recursively. Remove(ctx Context, parent Ino, name string, count *uint64) syscall.Errno + // Check integrity of an absolute path and repair it if asked + Check(ctx Context, fpath string, repair bool) syscall.Errno // OnMsg add a callback for the given message type. OnMsg(mtype uint32, cb MsgCallback) diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index fcaa9057b4ee..1af00c20e4ca 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -2824,6 +2824,23 @@ func (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, return errno(err) } +func (m *redisMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno { + return errno(m.txn(ctx, func(tx *redis.Tx) error { + attr.Nlink = 2 + vals, err := tx.HGetAll(ctx, m.entryKey(inode)).Result() + if err != nil { + return err + } + for _, v := range vals { + typ, _ := m.parseEntry([]byte(v)) + if typ == TypeDirectory { + attr.Nlink++ + } + } + return tx.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0).Err() + }, m.entryKey(inode), m.inodeKey(inode))) +} + func (m *redisMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno { defer m.timeit(time.Now()) inode = m.checkRoot(inode) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 1553a0d153bd..1009ed9abe20 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2590,6 +2590,34 @@ func (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, sh return errno(err) } +func (m *dbMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno { + n := &node{ + Inode: inode, + Type: attr.Typ, + Mode: attr.Mode, + Uid: attr.Uid, + Gid: attr.Gid, + Atime: attr.Atime * 1e6, + Mtime: attr.Mtime * 1e6, + Ctime: attr.Ctime * 1e6, + Length: 4 << 10, + Parent: attr.Parent, + } + return errno(m.txn(func(s *xorm.Session) error { + n.Nlink = 2 + var rows []edge + if err := s.Find(&rows, &edge{Parent: inode}); err != nil { + return err + } + for _, row := range rows { + if row.Type == TypeDirectory { + n.Nlink++ + } + } + return mustInsert(s, n) + }, inode)) +} + func (m *dbMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno { defer m.timeit(time.Now()) inode = m.checkRoot(inode) diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 906906bdeac0..b31b29849f6d 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -2221,6 +2221,21 @@ func (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, sh return 0 } +func (m *kvMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno { + return errno(m.txn(func(tx kvTxn) error { + attr.Nlink = 2 + _ = tx.scanValues(m.entryKey(inode, ""), 0, func(k, v []byte) bool { + typ, _ := m.parseEntry(v) + if typ == TypeDirectory { + attr.Nlink++ + } + return false + }) + tx.set(m.inodeKey(inode), m.marshal(attr)) + return nil + }, inode)) +} + func (m *kvMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno { defer m.timeit(time.Now()) inode = m.checkRoot(inode)