Merge branch 'main' into clean_detached_node

# Conflicts:
#	pkg/meta/sql.go
#	pkg/meta/tkv.go

cmd/main.go

@@ -55,6 +55,7 @@ func Main(args []string) error {
 		Commands: []*cli.Command{
 			cmdFormat(),
 			cmdConfig(),
+			cmdQuota(),
 			cmdDestroy(),
 			cmdGC(),
 			cmdFsck(),

cmd/quota.go

@@ -0,0 +1,133 @@
+/*
+ * JuiceFS, Copyright 2023 Juicedata, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/juicedata/juicefs/pkg/meta"
+
+	"github.com/urfave/cli/v2"
+)
+
+func cmdQuota() *cli.Command {
+	return &cli.Command{
+		Name:            "quota",
+		Category:        "ADMIN",
+		Usage:           "Manage directory quotas",
+		ArgsUsage:       "META-URL",
+		HideHelpCommand: true,
+		Description: `
+Examples:
+$ juicefs quota set redis://localhost --path /dir1 --capacity 1 --inodes 100
+$ juicefs quota get redis://localhost --path /dir1
+$ juicefs quota del redis://localhost --path /dir1
+$ juicefs quota ls redis://localhost`,
+		Subcommands: []*cli.Command{
+			{
+				Name:      "set",
+				Usage:     "Set quota to a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "get",
+				Usage:     "Get quota of a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "del",
+				Usage:     "Delete quota of a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "ls",
+				Usage:     "List all directory quotas",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "check",
+				Usage:     "Check quota consistency of a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "path",
+				Usage: "full path of the directory within the volume",
+			},
+			&cli.Uint64Flag{
+				Name:  "capacity",
+				Usage: "hard quota of the directory limiting its usage of space in GiB",
+			},
+			&cli.Uint64Flag{
+				Name:  "inodes",
+				Usage: "hard quota of the directory limiting its number of inodes",
+			},
+		},
+	}
+}
+
+func quota(c *cli.Context) error {
+	setup(c, 1)
+	var cmd uint8
+	switch c.Command.Name {
+	case "set":
+		cmd = meta.QuotaSet
+	case "get":
+		cmd = meta.QuotaGet
+	case "del":
+		cmd = meta.QuotaDel
+	case "ls":
+		cmd = meta.QuotaList
+	case "check":
+		cmd = meta.QuotaCheck
+	default:
+		logger.Fatalf("Invalid quota command: %s", c.Command.Name)
+	}
+	dpath := c.String("path")
+	if dpath == "" && cmd != meta.QuotaList {
+		logger.Fatalf("Please specify the directory with `--path <dir>` option")
+	}
+	removePassword(c.Args().Get(0))
+
+	m := meta.NewClient(c.Args().Get(0), nil)
+	qs := make(map[string]*meta.Quota)
+	if cmd == meta.QuotaSet {
+		q := &meta.Quota{MaxSpace: -1, MaxInodes: -1} // negative means no change
+		if c.IsSet("capacity") {
+			q.MaxSpace = int64(c.Uint64("capacity")) << 30
+		}
+		if c.IsSet("inodes") {
+			q.MaxInodes = int64(c.Uint64("inodes"))
+		}
+		qs[dpath] = q
+	}
+	if err := m.HandleQuota(meta.Background, cmd, dpath, qs); err != nil {
+		return err
+	}
+
+	for p, q := range qs {
+		// FIXME: need a better way to do print
+		fmt.Printf("%s: %+v\n", p, *q)
+	}
+	return nil
+}

docs/en/guide/how_to_set_up_metadata_engine.md

@@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem';
 
 JuiceFS is a decoupled structure that separates data and metadata. Metadata can be stored in any supported database (called Metadata Engine). Many databases are supported and they all comes with different performance and intended scenarios, refer to [our docs](../benchmark/metadata_engines_benchmark.md) for comparison.
 
+## The storage usage of metadata {#storage-usage}
+
+The storage space required for metadata is related to the length of the file name, the type and length of the file, and extended attributes. It is difficult to accurately estimate the metadata storage space requirements of a file system. For simplicity, we can approximate based on the storage space required for a single small file without extended attributes.
+
+- **Key-Value Database** (e.g. Redis, TiKV): 300 bytes/file
+- **Relational Database** (e.g. SQLite, MySQL, PostgreSQL): 600 bytes/file
+
+When the average file is larger (over 64MB), or the file is frequently modified and has a lot of fragments, or there are many extended attributes, or the average file name is long (over 50 bytes), more storage space is needed.
+
+When you need to migrate between two types of metadata engines, you can use this method to estimate the required storage space. For example, if you want to migrate the metadata engine from a relational database (MySQL) to a key-value database (Redis), and the current usage of MySQL is 30GB, then the target Redis needs to prepare at least 15GB or more of memory. The reverse is also true.
+
 ## Redis
 
 JuiceFS requires Redis version 4.0 and above. Redis Cluster is also supported, but in order to avoid transactions across different Redis instances, JuiceFS puts all metadata for one file system on a single Redis instance.

docs/zh_cn/guide/how_to_set_up_metadata_engine.md

@@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem';
 
 JuiceFS 采用数据和元数据分离的存储架构，元数据可以存储在任意支持的数据库中，称为「元数据存储引擎」。JuiceFS 支持众多元数据存储引擎，各个数据库性能、易用性、场景均有区别，具体性能对比可参考[该文档](../benchmark/metadata_engines_benchmark.md)。
 
+## 元数据存储用量 {#storage-usage}
+
+元数据所需的存储空间跟文件名的长度、文件的类型和长度以及扩展属性等相关，无法准确地估计一个文件系统的元数据存空间需求。简单起见，我们可以根据没有扩展属性的单个小文件所需的存储空间来做近似：
+
+- **键值（Key-Value）数据库**（如 Redis、TiKV）：300 字节／文件
+- **关系型数据库**（如 SQLite、MySQL、PostgreSQL）：600 字节／文件
+
+当平均文件更大（超过 64MB），或者文件被频繁修改导致有很多碎片，或者有很多扩展属性，或者平均文件名很长（超过 50 字节），都会导致需要更多的存储空间。
+
+当你需要在两种类型的元数据引擎之间迁移时，就可以据此来估算所需的存储空间。例如，假设你希望将元数据引擎从一个关系型数据库（MySQL）迁移到键值数据库（Redis），如果当前 MySQL 的用量为 30GB，那么目标 Redis 至少需要准备 15GB 以上的内存。反之亦然。
+
 ## Redis
 
 JuiceFS 要求使用 4.0 及以上版本的 Redis。JuiceFS 也支持使用 Redis Cluster 作为元数据引擎，但为了避免在 Redis 集群中执行跨节点事务，同一个文件系统的元数据总会坐落于单个 Redis 实例中。