cmd/config: add option to enable/disable dir stats (#3686)

cmd/config.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/juicedata/juicefs/pkg/meta"
 	"github.com/juicedata/juicefs/pkg/version"
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 )
 
@@ -80,6 +81,10 @@ func configManagementFlags() []cli.Flag {
 			Name:  "download-limit",
 			Usage: "default bandwidth limit of the volume for download in Mbps",
 		},
+		&cli.BoolFlag{
+			Name:  "dir-stats",
+			Usage: "enable dir stats, which is necessary for fast summary and dir quota",
+		},
 	})
 }
 
@@ -130,6 +135,7 @@ func config(ctx *cli.Context) error {
 		return nil
 	}
 
+	originDirStats := format.DirStats
 	var quota, storage, trash, clientVer bool
 	var msg strings.Builder
 	encrypted := format.KeyEncrypted
@@ -211,6 +217,11 @@ func config(ctx *cli.Context) error {
 				format.TrashDays = new
 				trash = true
 			}
+		case "dir-stats":
+			if new := ctx.Bool(flag); new != format.DirStats {
+				msg.WriteString(fmt.Sprintf("%10s: %t -> %t\n", flag, format.DirStats, new))
+				format.DirStats = new
+			}
 		case "min-client-version":
 			if new := ctx.String(flag); new != format.MinClientVersion {
 				if version.Parse(new) == nil {
@@ -266,6 +277,20 @@ func config(ctx *cli.Context) error {
 				return fmt.Errorf("Aborted.")
 			}
 		}
+		if originDirStats && !format.DirStats {
+			qs := make(map[string]*meta.Quota)
+			err := m.HandleQuota(meta.Background, meta.QuotaList, "", qs, false, false)
+			if err != nil {
+				return errors.Wrap(err, "list quotas")
+			}
+			if len(qs) != 0 {
+				paths := make([]string, 0, len(qs))
+				for path := range qs {
+					paths = append(paths, path)
+				}
+				return fmt.Errorf("cannot disable dir stats when there are still %d dir quotas: %v", len(qs), paths)
+			}
+		}
 		if clientVer && format.CheckVersion() != nil {
 			warn("Clients with the same version of this will be rejected after modification.")
 			if !yes && !userConfirmed() {

cmd/format.go

@@ -433,6 +433,7 @@ func format(c *cli.Context) error {
 			BlockSize:    fixObjectSize(c.Int("block-size")),
 			Compression:  c.String("compress"),
 			TrashDays:    c.Int("trash-days"),
+			DirStats:     true,
 			MetaVersion:  meta.MaxVersion,
 		}
 		if format.AccessKey == "" && os.Getenv("ACCESS_KEY") != "" {

cmd/mount_test.go

@@ -52,6 +52,7 @@ func Test_exposeMetrics(t *testing.T) {
 				Name:      "test",
 				BlockSize: 4096,
 				Capacity:  1 << 30,
+				DirStats:  true,
 			}
 			_ = client.Init(format, true)
 			var appCtx *cli.Context

cmd/object_test.go

@@ -148,6 +148,7 @@ func TestJFS(t *testing.T) {
 		Name:      "test",
 		BlockSize: 4096,
 		Capacity:  1 << 30,
+		DirStats:  true,
 	}
 	_ = m.Init(format, true)
 	var conf = vfs.Config{

pkg/fs/fs_test.go

@@ -275,6 +275,7 @@ func createTestFS(t *testing.T) *FileSystem {
 		Name:      "test",
 		BlockSize: 4096,
 		Capacity:  1 << 30,
+		DirStats:  true,
 	}
 	_ = m.Init(format, true)
 	var conf = vfs.Config{

pkg/fuse/fuse_test.go

@@ -51,6 +51,7 @@ func format(url string) {
 		Storage:   "file",
 		Bucket:    os.TempDir() + "/",
 		BlockSize: 4096,
+		DirStats:  true,
 	}
 	err := m.Init(format, true)
 	if err != nil {

pkg/meta/base.go

@@ -63,6 +63,7 @@ type engine interface {
 	doRefreshSession() error
 	doFindStaleSessions(limit int) ([]uint64, error) // limit < 0 means all
 	doCleanStaleSession(sid uint64) error
+	doInit(format *Format, force bool) error
 
 	scanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) error
 	compactChunk(inode Ino, indx uint32, force bool)
@@ -332,6 +333,9 @@ func (m *baseMeta) GetDirStat(ctx Context, inode Ino) (stat *dirStat, st syscall
 }
 
 func (m *baseMeta) updateDirStat(ctx Context, ino Ino, length, space, inodes int64) {
+	if !m.GetFormat().DirStats {
+		return
+	}
 	m.dirStatsLock.Lock()
 	defer m.dirStatsLock.Unlock()
 	stat := m.dirStats[ino]
@@ -346,6 +350,9 @@ func (m *baseMeta) updateParentStat(ctx Context, inode, parent Ino, length, spac
 		return
 	}
 	m.en.updateStats(space, 0)
+	if !m.GetFormat().DirStats {
+		return
+	}
 	if parent > 0 {
 		m.updateDirStat(ctx, parent, length, space, 0)
 		m.updateDirQuota(ctx, parent, space, 0)
@@ -371,6 +378,9 @@ func (m *baseMeta) flushDirStat() {
 }
 
 func (m *baseMeta) doFlushDirStat() {
+	if !m.GetFormat().DirStats {
+		return
+	}
 	m.dirStatsLock.Lock()
 	if len(m.dirStats) == 0 {
 		m.dirStatsLock.Unlock()
@@ -610,6 +620,9 @@ func (m *baseMeta) checkQuota(ctx Context, space, inodes int64, parents ...Ino)
 	if inodes > 0 && m.fmt.Inodes > 0 && atomic.LoadInt64(&m.usedInodes)+atomic.LoadInt64(&m.newInodes)+inodes > int64(m.fmt.Inodes) {
 		return syscall.ENOSPC
 	}
+	if !m.GetFormat().DirStats {
+		return 0
+	}
 	for _, ino := range parents {
 		if m.checkDirQuota(ctx, ino, space, inodes) {
 			return syscall.EDQUOT
@@ -663,6 +676,9 @@ func (m *baseMeta) getDirParent(ctx Context, inode Ino) (Ino, syscall.Errno) {
 }
 
 func (m *baseMeta) hasDirQuota(ctx Context, inode Ino) bool {
+	if !m.GetFormat().DirStats {
+		return false
+	}
 	var q *Quota
 	var st syscall.Errno
 	for {
@@ -684,6 +700,9 @@ func (m *baseMeta) hasDirQuota(ctx Context, inode Ino) bool {
 }
 
 func (m *baseMeta) checkDirQuota(ctx Context, inode Ino, space, inodes int64) bool {
+	if !m.GetFormat().DirStats {
+		return false
+	}
 	var q *Quota
 	var st syscall.Errno
 	for {
@@ -705,6 +724,9 @@ func (m *baseMeta) checkDirQuota(ctx Context, inode Ino, space, inodes int64) bo
 }
 
 func (m *baseMeta) updateDirQuota(ctx Context, inode Ino, space, inodes int64) {
+	if !m.GetFormat().DirStats {
+		return
+	}
 	var q *Quota
 	var st syscall.Errno
 	for {
@@ -729,6 +751,9 @@ func (m *baseMeta) flushQuotas() {
 	var newSpace, newInodes int64
 	for {
 		time.Sleep(time.Second * 3)
+		if !m.GetFormat().DirStats {
+			continue
+		}
 		m.quotaMu.RLock()
 		for ino, q := range m.dirQuotas {
 			newSpace = atomic.LoadInt64(&q.newSpace)
@@ -761,6 +786,10 @@ func (m *baseMeta) flushQuotas() {
 	}
 }
 
+func (m *baseMeta) Init(format *Format, force bool) error {
+	return m.en.doInit(format, force)
+}
+
 func (m *baseMeta) HandleQuota(ctx Context, cmd uint8, dpath string, quotas map[string]*Quota, strict, repair bool) error {
 	var inode Ino
 	if cmd != QuotaList {
@@ -774,6 +803,16 @@ func (m *baseMeta) HandleQuota(ctx Context, cmd uint8, dpath string, quotas map[
 
 	switch cmd {
 	case QuotaSet:
+		format, err := m.Load(false)
+		if err != nil {
+			return errors.Wrap(err, "load format")
+		}
+		if !format.DirStats {
+			format.DirStats = true
+			if err := m.en.doInit(format, false); err != nil {
+				return err
+			}
+		}
 		q, err := m.en.doGetQuota(ctx, inode)
 		if err != nil {
 			return err
@@ -2035,6 +2074,13 @@ func (m *baseMeta) resolve(ctx Context, dpath string, inode *Ino) syscall.Errno
 }
 
 func (m *baseMeta) GetFormat() Format {
+	if m.fmt == nil {
+		var err error
+		m.fmt, err = m.Load(false)
+		if err != nil {
+			logger.Fatalf("Load format: %s", err)
+		}
+	}
 	return *m.fmt
 }
 

pkg/meta/base_test.go

@@ -45,6 +45,10 @@ func testConfig() *Config {
 	return conf
 }
 
+func testFormat() *Format {
+	return &Format{Name: "test", DirStats: true}
+}
+
 func TestRedisClient(t *testing.T) {
 	m, err := newRedisMeta("redis", "127.0.0.1:6379/10", testConfig())
 	if err != nil || m.Name() != "redis" {
@@ -153,7 +157,7 @@ func testMetaClient(t *testing.T, m Meta) {
 		t.Fatalf("getattr root: %s", st)
 	}
 
-	if err := m.Init(&Format{Name: "test"}, true); err != nil {
+	if err := m.Init(testFormat(), true); err != nil {
 		t.Fatalf("initialize failed: %s", err)
 	}
 	if err := m.Init(&Format{Name: "test2"}, false); err == nil { // not allowed
@@ -700,7 +704,6 @@ func testMetaClient(t *testing.T, m Meta) {
 }
 
 func testStickyBit(t *testing.T, m Meta) {
-	_ = m.Init(&Format{Name: "test"}, false)
 	ctx := Background
 	var sticky, normal, inode Ino
 	var attr = &Attr{}
@@ -767,7 +770,6 @@ func testStickyBit(t *testing.T, m Meta) {
 }
 
 func testListLocks(t *testing.T, m Meta) {
-	_ = m.Init(&Format{Name: "test"}, false)
 	ctx := Background
 	var inode Ino
 	var attr = &Attr{}
@@ -842,7 +844,6 @@ func testListLocks(t *testing.T, m Meta) {
 }
 
 func testLocks(t *testing.T, m Meta) {
-	_ = m.Init(&Format{Name: "test"}, false)
 	ctx := Background
 	var inode Ino
 	var attr = &Attr{}
@@ -979,7 +980,6 @@ func testLocks(t *testing.T, m Meta) {
 }
 
 func testRemove(t *testing.T, m Meta) {
-	_ = m.Init(&Format{Name: "test"}, false)
 	ctx := Background
 	var inode, parent Ino
 	var attr = &Attr{}
@@ -1021,7 +1021,6 @@ func testRemove(t *testing.T, m Meta) {
 }
 
 func testCaseIncensi(t *testing.T, m Meta) {
-	_ = m.Init(&Format{Name: "test"}, false)
 	ctx := Background
 	var inode Ino
 	var attr = &Attr{}
@@ -1070,9 +1069,16 @@ type compactor interface {
 
 func testCompaction(t *testing.T, m Meta, trash bool) {
 	if trash {
-		_ = m.Init(&Format{Name: "test", TrashDays: 1}, false)
+		format := testFormat()
+		format.TrashDays = 1
+		_ = m.Init(format, false)
+		defer func() {
+			if err := m.Init(testFormat(), false); err != nil {
+				t.Fatalf("init: %v", err)
+			}
+		}()
 	} else {
-		_ = m.Init(&Format{Name: "test"}, false)
+		_ = m.Init(testFormat(), false)
 	}
 	var l sync.Mutex
 	deleted := make(map[uint64]int)
@@ -1186,7 +1192,6 @@ func testConcurrentWrite(t *testing.T, m Meta) {
 	m.OnMsg(CompactChunk, func(args ...interface{}) error {
 		return nil
 	})
-	_ = m.Init(&Format{Name: "test"}, false)
 
 	ctx := Background
 	var inode Ino
@@ -1289,7 +1294,6 @@ func testCopyFileRange(t *testing.T, m Meta) {
 	m.OnMsg(DeleteSlice, func(args ...interface{}) error {
 		return nil
 	})
-	_ = m.Init(&Format{Name: "test"}, false)
 
 	ctx := Background
 	var iin, iout Ino
@@ -1339,7 +1343,6 @@ func testCopyFileRange(t *testing.T, m Meta) {
 }
 
 func testCloseSession(t *testing.T, m Meta) {
-	_ = m.Init(&Format{Name: "test"}, false)
 	if err := m.NewSession(); err != nil {
 		t.Fatalf("new session: %s", err)
 	}
@@ -1399,9 +1402,16 @@ func testCloseSession(t *testing.T, m Meta) {
 }
 
 func testTrash(t *testing.T, m Meta) {
-	if err := m.Init(&Format{Name: "test", TrashDays: 1}, false); err != nil {
-		t.Fatalf("init: %s", err)
+	format := testFormat()
+	format.TrashDays = 1
+	if err := m.Init(format, false); err != nil {
+		t.Fatalf("init: %v", err)
 	}
+	defer func() {
+		if err := m.Init(testFormat(), false); err != nil {
+			t.Fatalf("init: %v", err)
+		}
+	}()
 	ctx := Background
 	var inode, parent Ino
 	var attr = &Attr{}
@@ -1495,9 +1505,6 @@ func testTrash(t *testing.T, m Meta) {
 }
 
 func testParents(t *testing.T, m Meta) {
-	if err := m.Init(&Format{Name: "test"}, false); err != nil {
-		t.Fatalf("init: %s", err)
-	}
 	ctx := Background
 	var inode, parent Ino
 	var attr = &Attr{}
@@ -2001,7 +2008,6 @@ func testCheckAndRepair(t *testing.T, m Meta) {
 func testDirStat(t *testing.T, m Meta) {
 	testDir := "testDirStat"
 	var testInode Ino
-
 	// test empty dir
 	if st := m.Mkdir(Background, RootInode, testDir, 0640, 022, 0, &testInode, nil); st != 0 {
 		t.Fatalf("mkdir: %s", st)

pkg/meta/benchmarks_test.go

@@ -622,7 +622,7 @@ func benchmarkData(b *testing.B, m Meta) {
 }
 
 func benchmarkAll(b *testing.B, m Meta) {
-	_ = m.Init(&Format{Name: "benchmarkAll"}, true)
+	_ = m.Init(&Format{Name: "benchmarkAll", DirStats: true}, true)
 	_ = m.NewSession()
 	benchmarkDir(b, m)
 	benchmarkFile(b, m)

pkg/meta/config.go

@@ -90,6 +90,7 @@ type Format struct {
 	MetaVersion      int    `json:",omitempty"`
 	MinClientVersion string `json:",omitempty"`
 	MaxClientVersion string `json:",omitempty"`
+	DirStats         bool   `json:",omitempty"`
 }
 
 func (f *Format) update(old *Format, force bool) error {