cmd/fsck: support the nlink of the check and repair directory

cmd/fsck.go

@@ -46,7 +46,10 @@ Examples:
 $ juicefs fsck redis://localhost
 
 # Repair broken directories
-$ juicefs fsck redis://localhost --path /d1/d2 --repair`,
+$ juicefs fsck redis://localhost --path /d1/d2 --repair
+
+# recursively check
+$ juicefs fsck redis://localhost --path /d1/d2 --recursive`,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:  "path",
@@ -56,6 +59,11 @@ $ juicefs fsck redis://localhost --path /d1/d2 --repair`,
 				Name:  "repair",
 				Usage: "repair specified path if it's broken",
 			},
+			&cli.BoolFlag{
+				Name:    "recursive",
+				Aliases: []string{"r"},
+				Usage:   "recursively check or repair",
+			},
 		},
 	}
 }
@@ -76,7 +84,7 @@ func fsck(ctx *cli.Context) error {
 		if !strings.HasPrefix(p, "/") {
 			logger.Fatalf("File path should be the absolute path within JuiceFS")
 		}
-		return m.Check(c, p, ctx.Bool("repair"))
+		return m.Check(c, p, ctx.Bool("repair"), ctx.Bool("recursive"))
 	}
 
 	chunkConf := chunk.Config{

pkg/meta/base.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path"
+	"path/filepath"
 	"runtime"
 	"sort"
 	"strings"
@@ -1045,49 +1046,137 @@ func (m *baseMeta) GetPaths(ctx Context, inode Ino) []string {
 	return paths
 }
 
-func (m *baseMeta) Check(ctx Context, fpath string, repair bool) (st syscall.Errno) {
+func (m *baseMeta) countDirNlink(ctx Context, inode Ino) (uint32, syscall.Errno) {
+	var entries []*Entry
+	if st := m.en.doReaddir(ctx, inode, 0, &entries, -1); st != 0 {
+		logger.Errorf("readdir inode %d: %s", inode, st)
+		return 0, st
+	}
+	var dirCounter uint32 = 2
+	for _, e := range entries {
+		if e.Attr.Typ == TypeDirectory {
+			dirCounter++
+		}
+	}
+	return dirCounter, 0
+}
+
+type metaWalkFunc func(ctx Context, inode Ino, path string, attr *Attr)
+
+func (m *baseMeta) walk(ctx Context, inode Ino, path string, attr *Attr, walkFn metaWalkFunc) syscall.Errno {
+	walkFn(ctx, inode, path, attr)
+	var entries []*Entry
+	st := m.en.doReaddir(ctx, inode, 1, &entries, -1)
+	if st != 0 {
+		logger.Errorf("list %s: %s", path, st)
+		return st
+	}
+	for _, entry := range entries {
+		if !entry.Attr.Full {
+			entry.Attr.Parent = inode
+		}
+		if st := m.walk(ctx, entry.Inode, filepath.Join(path, string(entry.Name)), entry.Attr, walkFn); st != 0 {
+			return st
+		}
+	}
+	return 0
+}
+
+func (m *baseMeta) Check(ctx Context, fpath string, repair bool, recursive bool) (st syscall.Errno) {
 	var attr Attr
-	var inode Ino
-	var parent Ino = 1
-	ps := strings.Split(fpath, "/")
+	var inode = RootInode
+	var parent = RootInode
+	attr.Typ = TypeDirectory
+	ps := strings.FieldsFunc(fpath, func(r rune) bool {
+		return r == '/'
+	})
 	for i, name := range ps {
-		if len(name) == 0 {
-			continue
-		}
+		parent = inode
 		if st = m.Lookup(ctx, parent, name, &inode, &attr); st != 0 {
 			logger.Errorf("Lookup parent %d name %s: %s", parent, name, st)
 			return
 		}
-		if attr.Full {
-			parent = inode
-			continue
-		}
-
-		// missing attribute
-		p := "/" + path.Join(ps[:i+1]...)
-		if attr.Typ != TypeDirectory { // TODO: determine file size?
-			logger.Errorf("Path %s (inode %d type %d) cannot be auto-repaired, you have to repair it manually or remove it", p, inode, attr.Typ)
-		} else if !repair {
-			logger.Warnf("Path %s (inode %d) can be repaired, please re-check with 'repair' enabled", p, inode)
-		} else { // repair directory inode
-			now := time.Now().Unix()
-			attr.Mode = 0644
-			attr.Uid = ctx.Uid()
-			attr.Gid = ctx.Gid()
-			attr.Atime = now
-			attr.Mtime = now
-			attr.Ctime = now
-			attr.Length = 4 << 10
-			attr.Parent = parent
-			if st = m.en.doRepair(ctx, inode, &attr); st == 0 {
-				logger.Infof("Path %s (inode %d) is successfully repaired", p, inode)
+		if !attr.Full && i < len(ps)-1 {
+			// missing attribute
+			p := "/" + path.Join(ps[:i+1]...)
+			if attr.Typ != TypeDirectory { // TODO: determine file size?
+				logger.Warnf("Attribute of %s (inode %d type %d) is missing and cannot be auto-repaired, please repair it manually or remove it", p, inode, attr.Typ)
 			} else {
-				logger.Errorf("Repair path %s inode %d: %s", p, inode, st)
+				logger.Warnf("Attribute of %s (inode %d) is missing, please re-run with '--path %s --repair' to fix it", p, inode, p)
 			}
 		}
-		return // handle one missing inode at a time
 	}
-	logger.Infof("Path %s inode %d is valid", fpath, parent)
+	if !attr.Full {
+		attr.Parent = parent
+	}
+
+	type node struct {
+		inode Ino
+		path  string
+		attr  *Attr
+	}
+	nodes := make(chan *node, 1000)
+	go func() {
+		defer close(nodes)
+		if recursive {
+			st = m.walk(ctx, inode, fpath, &attr, func(ctx Context, inode Ino, path string, attr *Attr) {
+				nodes <- &node{inode, path, attr}
+			})
+		} else {
+			nodes <- &node{inode, fpath, &attr}
+		}
+	}()
+
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for e := range nodes {
+				inode := e.inode
+				path := e.path
+				attr := e.attr
+				if attr.Typ != TypeDirectory {
+					// TODO
+					continue
+				}
+				if attr.Full {
+					nlink, st := m.countDirNlink(ctx, inode)
+					if st != 0 {
+						logger.Errorf("Count nlink for inode %d: %s", inode, st)
+						continue
+					}
+					if attr.Nlink == nlink {
+						continue
+					}
+					logger.Warnf("nlink of %s should be %d, but got %d", path, nlink, attr.Nlink)
+				} else {
+					logger.Warnf("attribute of %s is missing", path)
+				}
+
+				if repair {
+					if !attr.Full {
+						now := time.Now().Unix()
+						attr.Mode = 0644
+						attr.Uid = ctx.Uid()
+						attr.Gid = ctx.Gid()
+						attr.Atime = now
+						attr.Mtime = now
+						attr.Ctime = now
+						attr.Length = 4 << 10
+					}
+					if st1 := m.en.doRepair(ctx, inode, attr); st1 == 0 {
+						logger.Infof("Path %s (inode %d) is successfully repaired", path, inode)
+					} else {
+						logger.Errorf("Repair path %s inode %d: %s", path, inode, st1)
+					}
+				} else {
+					logger.Warnf("Path %s (inode %d) can be repaired, please re-run with '--path %s --repair' to fix it", path, inode, path)
+				}
+			}
+		}()
+	}
+	wg.Wait()
 	return
 }
 

pkg/meta/base_test.go

@@ -30,7 +30,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-redis/redis/v8"
 	"github.com/juicedata/juicefs/pkg/utils"
+	"xorm.io/xorm"
 )
 
 func TestRedisClient(t *testing.T) {
@@ -76,6 +78,7 @@ func testMeta(t *testing.T, m Meta) {
 	testOpenCache(t, m)
 	base.conf.CaseInsensi = true
 	testCaseIncensi(t, m)
+	testCheckAndRepair(t, m)
 	base.conf.ReadOnly = true
 	testReadOnly(t, m)
 }
@@ -1584,3 +1587,145 @@ func testAttrFlags(t *testing.T, m Meta) {
 		t.Fatalf("copy_file_range f: %s", st)
 	}
 }
+
+func setAttr(t *testing.T, m Meta, inode Ino, attr *Attr) {
+	var err error
+	switch m := m.(type) {
+	case *redisMeta:
+		err = m.txn(Background, func(tx *redis.Tx) error {
+			return tx.Set(Background, m.inodeKey(inode), m.marshal(attr), 0).Err()
+		}, m.inodeKey(inode))
+	case *dbMeta:
+		err = m.txn(func(s *xorm.Session) error {
+			_, err = s.ID(inode).AllCols().Update(&node{
+				Inode:  inode,
+				Type:   attr.Typ,
+				Flags:  attr.Flags,
+				Mode:   attr.Mode,
+				Uid:    attr.Uid,
+				Gid:    attr.Gid,
+				Mtime:  attr.Mtime * 1e6,
+				Ctime:  attr.Ctime * 1e6,
+				Atime:  attr.Atime * 1e6,
+				Nlink:  attr.Nlink,
+				Length: attr.Length,
+				Rdev:   attr.Rdev,
+				Parent: attr.Parent,
+			})
+			return err
+		})
+	case *kvMeta:
+		err = m.txn(func(tx kvTxn) error {
+			tx.set(m.inodeKey(inode), m.marshal(attr))
+			return nil
+		})
+	}
+	if err != nil {
+		t.Fatalf("setAttr: %v", err)
+	}
+}
+
+func testCheckAndRepair(t *testing.T, m Meta) {
+	var checkInode, d1Inode, d2Inode, d3Inode, d4Inode Ino
+	dirAttr := &Attr{Mode: 0644, Full: true, Typ: TypeDirectory, Nlink: 3}
+	if st := m.Mkdir(Background, RootInode, "check", 0640, 022, 0, &checkInode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	if st := m.Mkdir(Background, checkInode, "d1", 0640, 022, 0, &d1Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	if st := m.Mkdir(Background, d1Inode, "d2", 0640, 022, 0, &d2Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	if st := m.Mkdir(Background, d2Inode, "d3", 0640, 022, 0, &d3Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+	if st := m.Mkdir(Background, d3Inode, "d4", 0640, 022, 0, &d4Inode, dirAttr); st != 0 {
+		t.Fatalf("mkdir: %s", st)
+	}
+
+	if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, checkInode, dirAttr)
+
+	if st := m.GetAttr(Background, d1Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, d1Inode, dirAttr)
+
+	if st := m.GetAttr(Background, d2Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, d2Inode, dirAttr)
+
+	if st := m.GetAttr(Background, d3Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Nlink = 0
+	setAttr(t, m, d3Inode, dirAttr)
+
+	if st := m.GetAttr(Background, d4Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	dirAttr.Full = false
+	dirAttr.Nlink = 0
+	setAttr(t, m, d4Inode, dirAttr)
+
+	if st := m.Check(Background, "/check", false, false); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 0 {
+		t.Fatalf("checkInode nlink should is 0 now: %d", dirAttr.Nlink)
+	}
+
+	if st := m.Check(Background, "/check", true, false); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 3 || dirAttr.Parent != RootInode {
+		t.Fatalf("checkInode nlink should is 3 now: %d", dirAttr.Nlink)
+	}
+
+	if st := m.Check(Background, "/check/d1/d2", true, false); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	if st := m.GetAttr(Background, d2Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 3 || dirAttr.Parent != d1Inode {
+		t.Fatalf("d2Inode nlink should is 3 now: %d", dirAttr.Nlink)
+	}
+	if st := m.GetAttr(Background, d1Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if dirAttr.Nlink != 0 || dirAttr.Parent != checkInode {
+		t.Fatalf("d1Inode nlink should is 0 now: %d", dirAttr.Nlink)
+	}
+
+	if st := m.Check(Background, "/", true, true); st != 0 {
+		t.Fatalf("check: %s", st)
+	}
+	for _, ino := range []Ino{checkInode, d1Inode, d2Inode, d3Inode} {
+		if st := m.GetAttr(Background, ino, dirAttr); st != 0 {
+			t.Fatalf("getattr: %s", st)
+		}
+		if !dirAttr.Full || dirAttr.Nlink != 3 {
+			t.Fatalf("nlink should is 3 now: %d", dirAttr.Nlink)
+		}
+	}
+	if st := m.GetAttr(Background, d4Inode, dirAttr); st != 0 {
+		t.Fatalf("getattr: %s", st)
+	}
+	if !dirAttr.Full || dirAttr.Nlink != 2 || dirAttr.Parent != d3Inode {
+		t.Fatalf("d4Inode attr: %+v", *dirAttr)
+	}
+}

pkg/meta/interface.go

@@ -351,7 +351,7 @@ type Meta interface {
 	// GetPaths returns all paths of an inode
 	GetPaths(ctx Context, inode Ino) []string
 	// Check integrity of an absolute path and repair it if asked
-	Check(ctx Context, fpath string, repair bool) syscall.Errno
+	Check(ctx Context, fpath string, repair bool, recursive bool) syscall.Errno
 
 	// OnMsg add a callback for the given message type.
 	OnMsg(mtype uint32, cb MsgCallback)

pkg/meta/redis.go

@@ -1777,7 +1777,7 @@ func (m *redisMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*En
 		return errno(err)
 	}
 
-	if plus != 0 {
+	if plus != 0 && len(*entries) != 0 {
 		fillAttr := func(es []*Entry) error {
 			var keys = make([]string, len(es))
 			for i, e := range es {