cmd/fsck: support the nlink of the check and repair directory

cmd/fsck.go

@@ -46,7 +46,10 @@ Examples:
 $ juicefs fsck redis://localhost
 
 # Repair broken directories
-$ juicefs fsck redis://localhost --path /d1/d2 --repair`,
+$ juicefs fsck redis://localhost --path /d1/d2 --repair
+
+# recursively check
+$ juicefs fsck redis://localhost --path /d1/d2 --recursive`,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:  "path",
@@ -56,6 +59,11 @@ $ juicefs fsck redis://localhost --path /d1/d2 --repair`,
 				Name:  "repair",
 				Usage: "repair specified path if it's broken",
 			},
+			&cli.BoolFlag{
+				Name:    "recursive",
+				Aliases: []string{"r"},
+				Usage:   "recursively check or repair",
+			},
 		},
 	}
 }
@@ -76,7 +84,7 @@ func fsck(ctx *cli.Context) error {
 		if !strings.HasPrefix(p, "/") {
 			logger.Fatalf("File path should be the absolute path within JuiceFS")
 		}
-		return m.Check(c, p, ctx.Bool("repair"))
+		return m.Check(c, p, ctx.Bool("repair"), ctx.Bool("recursive"))
 	}
 
 	chunkConf := chunk.Config{

pkg/meta/base.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path"
+	"path/filepath"
 	"runtime"
 	"sort"
 	"strings"
@@ -1044,32 +1045,114 @@ func (m *baseMeta) GetPaths(ctx Context, inode Ino) []string {
 	}
 	return paths
 }
+func (m *baseMeta) countDirNlink(ctx Context, inode Ino) (uint32, syscall.Errno) {
+	var entries []*Entry
+	if st := m.en.doReaddir(ctx, inode, 0, &entries, -1); st != 0 {
+		logger.Errorf("readdir inode %d: %s", inode, st)
+		return 0, st
+	}
+	var dirCounter uint32 = 2
+	for _, e := range entries {
+		if e.Attr.Typ == TypeDirectory {
+			dirCounter++
+		}
+	}
+	return dirCounter, 0
+}
+
+type metaWalkFunc func(ctx Context, inode Ino, path string, attr *Attr, st syscall.Errno) syscall.Errno
+
+const skipDir = syscall.Errno(10000)
+
+func (m *baseMeta) walkHelpFunc(ctx Context, inode Ino, path string, attr *Attr, walkFn metaWalkFunc) syscall.Errno {
+	if attr.Typ != TypeDirectory {
+		return walkFn(ctx, inode, path, attr, 0)
+	}
+	var entries []*Entry
+	st := m.en.doReaddir(ctx, inode, 0, &entries, -1)
+	st1 := walkFn(ctx, inode, path, attr, st)
+	if st != 0 || st1 != 0 {
+		return st1
+	}
+
+	for _, entry := range entries {
+		filename := filepath.Join(path, string(entry.Name))
+		if entry.Attr.Typ == TypeDirectory {
+			if st := m.walkHelpFunc(ctx, entry.Inode, filename, entry.Attr, walkFn); st != 0 && st != skipDir {
+				return st
+			}
+		} else {
+			if st := walkFn(ctx, entry.Inode, filename, entry.Attr, 0); st != 0 && st != skipDir {
+				return st
+			}
+		}
+	}
+	return 0
+}
+
+func (m *baseMeta) walk(ctx Context, inode Ino, path string, attr *Attr, walkFn metaWalkFunc) syscall.Errno {
+	st := m.walkHelpFunc(ctx, inode, path, attr, walkFn)
+	if st == skipDir {
+		return 0
+	}
+	return st
+}
 
-func (m *baseMeta) Check(ctx Context, fpath string, repair bool) (st syscall.Errno) {
+func (m *baseMeta) Check(ctx Context, fpath string, repair bool, recursive bool) (st syscall.Errno) {
 	var attr Attr
-	var inode Ino
-	var parent Ino = 1
-	ps := strings.Split(fpath, "/")
+	var inode = RootInode
+	var parent = RootInode
+	attr.Typ = TypeDirectory
+	ps := strings.FieldsFunc(fpath, func(r rune) bool {
+		return r == '/'
+	})
 	for i, name := range ps {
-		if len(name) == 0 {
-			continue
-		}
 		if st = m.Lookup(ctx, parent, name, &inode, &attr); st != 0 {
 			logger.Errorf("Lookup parent %d name %s: %s", parent, name, st)
 			return
 		}
 		if attr.Full {
+			if i >= len(ps)-1 {
+				break
+			}
 			parent = inode
 			continue
 		}
 
 		// missing attribute
 		p := "/" + path.Join(ps[:i+1]...)
 		if attr.Typ != TypeDirectory { // TODO: determine file size?
-			logger.Errorf("Path %s (inode %d type %d) cannot be auto-repaired, you have to repair it manually or remove it", p, inode, attr.Typ)
-		} else if !repair {
-			logger.Warnf("Path %s (inode %d) can be repaired, please re-check with 'repair' enabled", p, inode)
-		} else { // repair directory inode
+			logger.Warnf("Path %s (inode %d type %d) attribute is missing and cannot be auto-repaired, you have to repair it manually or remove it", p, inode, attr.Typ)
+		} else {
+			logger.Warnf("Path %s (inode %d) attribute is missing but can be repaired, please re-run with '--path %s --repair' to fix it", p, inode, p)
+		}
+	}
+
+	if st = m.walk(ctx, inode, fpath, &attr, func(ctx Context, inode1 Ino, path string, attr *Attr, st1 syscall.Errno) syscall.Errno {
+		if st1 != 0 {
+			logger.Errorf("Walk path %s inode %d: %s", path, inode1, st1)
+			return st1
+		}
+		if attr.Typ != TypeDirectory {
+			logger.Debugf("Path %s (inode %d) is a file, skip the nlink check", path, inode1)
+			return 0
+		}
+		if st1 := m.GetAttr(ctx, inode1, attr); st1 != 0 {
+			logger.Errorf("GetAttr inode %d: %s", inode1, st1)
+			return st1
+		}
+		nlink, st1 := m.countDirNlink(ctx, inode1)
+		if st1 != 0 {
+			logger.Errorf("Count nlink for inode %d: %s", inode1, st1)
+			return st1
+		}
+		if attr.Full && attr.Nlink == nlink {
+			if !recursive && inode1 == inode {
+				return skipDir
+			}
+			return 0
+		}
+		if !attr.Full {
 			now := time.Now().Unix()
 			attr.Mode = 0644
 			attr.Uid = ctx.Uid()
@@ -1079,13 +1162,26 @@ func (m *baseMeta) Check(ctx Context, fpath string, repair bool) (st syscall.Err
 			attr.Ctime = now
 			attr.Length = 4 << 10
 			attr.Parent = parent
-			if st = m.en.doRepair(ctx, inode, &attr); st == 0 {
-				logger.Infof("Path %s (inode %d) is successfully repaired", p, inode)
+		}
+		attr.Nlink = nlink
+		if repair {
+			if st1 = m.en.doRepair(ctx, inode1, attr); st1 == 0 {
+				logger.Infof("Path %s (inode %d) is successfully repaired", path, inode1)
+				if !recursive && inode1 == inode {
+					return skipDir
+				}
 			} else {
-				logger.Errorf("Repair path %s inode %d: %s", p, inode, st)
+				logger.Errorf("Repair path %s inode %d: %s", path, inode1, st1)
 			}
+			return st1
+		}
+		logger.Warnf("Path %s (inode %d) can be repaired, please re-check with 'repair' enabled", path, inode1)
+		if !recursive && inode1 == inode {
+			return skipDir
 		}
-		return // handle one missing inode at a time
+		return 0
+	}); st != 0 {
+		return
 	}
 	logger.Infof("Path %s inode %d is valid", fpath, parent)
 	return

pkg/meta/base_test.go

@@ -30,7 +30,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-redis/redis/v8"
 	"github.com/juicedata/juicefs/pkg/utils"
+	"xorm.io/xorm"
 )
 
 func TestRedisClient(t *testing.T) {
@@ -76,6 +78,7 @@ func testMeta(t *testing.T, m Meta) {
 	testOpenCache(t, m)
 	base.conf.CaseInsensi = true
 	testCaseIncensi(t, m)
+	testCheckAndRepair(t, m)
 	base.conf.ReadOnly = true
 	testReadOnly(t, m)
 }
@@ -1584,3 +1587,154 @@ func testAttrFlags(t *testing.T, m Meta) {
 		t.Fatalf("copy_file_range f: %s", st)
 	}
 }
+
+func setAttr(t *testing.T, m Meta, inode Ino, attr *Attr) {
+	var err error
+	switch m := m.(type) {
+	case *redisMeta:
+		err = m.txn(Background, func(tx *redis.Tx) error {
+			return tx.Set(Background, m.inodeKey(inode), m.marshal(attr), 0).Err()
+		}, m.inodeKey(inode))
+	case *dbMeta:
+		err = m.txn(func(s *xorm.Session) error {
+			_, err = s.ID(inode).AllCols().Update(&node{
+				Inode:  inode,
+				Type:   attr.Typ,
+				Flags:  attr.Flags,
+				Mode:   attr.Mode,
+				Uid:    attr.Uid,
+				Gid:    attr.Gid,
+				Mtime:  attr.Mtime * 1e6,
+				Ctime:  attr.Ctime * 1e6,
+				Atime:  attr.Atime * 1e6,
+				Nlink:  attr.Nlink,
+				Length: attr.Length,
+				Rdev:   attr.Rdev,
+				Parent: attr.Parent,
+			})
+			return err
+		})
+	case *kvMeta:
+		err = m.txn(func(tx kvTxn) error {
+			tx.set(m.inodeKey(inode), m.marshal(attr))
+			return nil
+		})
+	}
+	if err != nil {
+		t.Fatalf("setAttr: %v", err)
+	}
+}
+
+func testCheckAndRepair(t *testing.T, m Meta) {
+	var checkInode, d1Inode, d2Inode, d3Inode, d4Inode Ino
+	dirAttr := &Attr{Mode: 0644, Full: true, Typ: TypeDirectory, Nlink: 3}
+	dirAttr.Parent = RootInode
+	if st := m.Mkdir(Background, RootInode, "check", 0640, 022, 0, &checkInode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	dirAttr.Parent = checkInode
+	if st := m.Mkdir(Background, checkInode, "d1", 0640, 022, 0, &d1Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	dirAttr.Parent = d1Inode
+	if st := m.Mkdir(Background, d1Inode, "d2", 0640, 022, 0, &d2Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	dirAttr.Parent = d2Inode
+	if st := m.Mkdir(Background, d2Inode, "d3", 0640, 022, 0, &d3Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	dirAttr.Parent = d3Inode
+	dirAttr.Nlink = 2
+	if st := m.Mkdir(Background, d3Inode, "d4", 0640, 022, 0, &d4Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+
+	if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, checkInode, dirAttr)
+
+	if st := m.GetAttr(Background, d1Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, d1Inode, dirAttr)
+
+	if st := m.GetAttr(Background, d2Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, d2Inode, dirAttr)
+
+	if st := m.GetAttr(Background, d3Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, d3Inode, dirAttr)
+
+	if st := m.GetAttr(Background, d4Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Full = false
+	dirAttr.Nlink = 0
+	setAttr(t, m, d4Inode, dirAttr)
+
+	if st := m.Check(Background, "/check", false, false); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 0 {
+		t.Fatalf("checkInode nlink should is 0 now: %d", dirAttr.Nlink)
+	}
+
+	if st := m.Check(Background, "/check", true, false); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 3 {
+		t.Fatalf("checkInode nlink should is 3 now: %d", dirAttr.Nlink)
+	}
+
+	if st := m.Check(Background, "/check/d1/d2", true, false); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	if st := m.GetAttr(Background, d2Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 3 {
+		t.Fatalf("d2Inode nlink should is 3 now: %d", dirAttr.Nlink)
+	}
+	if st := m.GetAttr(Background, d1Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 0 {
+		t.Fatalf("d1Inode nlink should is  now: %d", dirAttr.Nlink)
+	}
+
+	if st := m.Check(Background, "/", true, true); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	for _, ino := range []Ino{checkInode, d1Inode, d2Inode, d3Inode} {
+		if st := m.GetAttr(Background, ino, dirAttr); st != 0 {
+			t.Fatalf("getattr: %s", st)
+		}
+		if dirAttr.Nlink != 3 {
+			t.Fatalf("nlink should is 3 now: %d", dirAttr.Nlink)
+		}
+	}
+	if st := m.GetAttr(Background, d4Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if !dirAttr.Full {
+		t.Fatalf("d4Inode attr should not be missing: %v", dirAttr.Full)
+	}
+	if dirAttr.Nlink != 2 {
+		t.Fatalf("d4Inode nlink should is 2 now: %d", dirAttr.Nlink)
+	}
+}

pkg/meta/interface.go

@@ -351,7 +351,7 @@ type Meta interface {
 	// GetPaths returns all paths of an inode
 	GetPaths(ctx Context, inode Ino) []string
 	// Check integrity of an absolute path and repair it if asked
-	Check(ctx Context, fpath string, repair bool) syscall.Errno
+	Check(ctx Context, fpath string, repair bool, recursive bool) syscall.Errno
 
 	// OnMsg add a callback for the given message type.
 	OnMsg(mtype uint32, cb MsgCallback)