Merge branch 'main' into clean_detached_node

# Conflicts:
#	pkg/meta/sql.go
#	pkg/meta/tkv.go

.markdownlint-cli2.jsonc

@@ -158,6 +158,7 @@
         "StarRocks",
         "ThriftServer",
         "TiKV",
+        "Trino",
         "UID",
         "UUID",
         "Ubuntu",

cmd/clone.go

@@ -66,7 +66,6 @@ func clone(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("abs of %s: %s", srcPath, err)
 	}
-	srcParent := filepath.Dir(srcAbsPath)
 	srcIno, err := utils.GetFileInode(srcPath)
 	if err != nil {
 		return fmt.Errorf("lookup inode for %s: %s", srcPath, err)
@@ -84,6 +83,22 @@ func clone(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("abs of %s: %s", dst, err)
 	}
+
+	srcMp, err := findMountpoint(srcAbsPath)
+	if err != nil {
+		return err
+	}
+	dstMp, err := findMountpoint(filepath.Dir(dstAbsPath))
+	if err != nil {
+		return err
+	}
+	if srcMp != dstMp {
+		return fmt.Errorf("the clone DST path should be at the same mount point as the SRC path")
+	}
+	if strings.HasPrefix(dstAbsPath, srcAbsPath) {
+		return fmt.Errorf("the clone DST path should not be under the SRC path")
+	}
+
 	dstParent := filepath.Dir(dstAbsPath)
 	dstName := filepath.Base(dstAbsPath)
 	dstParentIno, err := utils.GetFileInode(dstParent)
@@ -107,9 +122,9 @@ func clone(ctx *cli.Context) error {
 	wb.Put([]byte(dstName))
 	wb.Put16(uint16(umask))
 	wb.Put8(cmode)
-	f := openController(srcParent)
-	if f == nil {
-		return fmt.Errorf("%s is not inside JuiceFS", srcPath)
+	f, err := openController(srcMp)
+	if err == nil {
+		return err
 	}
 	defer f.Close()
 	if _, err = f.Write(wb.Bytes()); err != nil {
@@ -127,3 +142,16 @@ func clone(ctx *cli.Context) error {
 	}
 	return nil
 }
+
+func findMountpoint(fpath string) (string, error) {
+	for p := fpath; p != "/"; p = filepath.Dir(p) {
+		inode, err := utils.GetFileInode(p)
+		if err != nil {
+			return "", fmt.Errorf("get inode of %s: %s", p, err)
+		}
+		if inode == uint64(meta.RootInode) {
+			return p, nil
+		}
+	}
+	return "", fmt.Errorf("%s is not inside JuiceFS", fpath)
+}

cmd/info.go

@@ -111,9 +111,9 @@ func info(ctx *cli.Context) error {
 		if inode < uint64(meta.RootInode) {
 			logger.Fatalf("inode number shouldn't be less than %d", meta.RootInode)
 		}
-		f := openController(d)
-		if f == nil {
-			logger.Errorf("%s is not inside JuiceFS", path)
+		f, err := openController(d)
+		if err != nil {
+			logger.Errorf("Open control file for %s: %s", d, err)
 			continue
 		}
 
@@ -244,21 +244,19 @@ func ltypeToString(t uint32) string {
 }
 
 func legacyInfo(d, path string, inode uint64, recursive, raw uint8) {
-	f := openController(d)
-	defer f.Close()
-	if f == nil {
-		logger.Errorf("%s is not inside JuiceFS", path)
-		// continue to next path
+	f, err := openController(d)
+	if err != nil {
+		logger.Errorf("Open control file for %s: %s", d, err)
 		return
 	}
-
+	defer f.Close()
 	wb := utils.NewBuffer(8 + 10)
 	wb.Put32(meta.LegacyInfo)
 	wb.Put32(10)
 	wb.Put64(inode)
 	wb.Put8(recursive)
 	wb.Put8(raw)
-	_, err := f.Write(wb.Bytes())
+	_, err = f.Write(wb.Bytes())
 	if err != nil {
 		logger.Fatalf("write message: %s", err)
 	}

cmd/main.go

@@ -55,6 +55,7 @@ func Main(args []string) error {
 		Commands: []*cli.Command{
 			cmdFormat(),
 			cmdConfig(),
+			cmdQuota(),
 			cmdDestroy(),
 			cmdGC(),
 			cmdFsck(),

cmd/quota.go

@@ -0,0 +1,133 @@
+/*
+ * JuiceFS, Copyright 2023 Juicedata, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/juicedata/juicefs/pkg/meta"
+
+	"github.com/urfave/cli/v2"
+)
+
+func cmdQuota() *cli.Command {
+	return &cli.Command{
+		Name:            "quota",
+		Category:        "ADMIN",
+		Usage:           "Manage directory quotas",
+		ArgsUsage:       "META-URL",
+		HideHelpCommand: true,
+		Description: `
+Examples:
+$ juicefs quota set redis://localhost --path /dir1 --capacity 1 --inodes 100
+$ juicefs quota get redis://localhost --path /dir1
+$ juicefs quota del redis://localhost --path /dir1
+$ juicefs quota ls redis://localhost`,
+		Subcommands: []*cli.Command{
+			{
+				Name:      "set",
+				Usage:     "Set quota to a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "get",
+				Usage:     "Get quota of a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "del",
+				Usage:     "Delete quota of a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "ls",
+				Usage:     "List all directory quotas",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+			{
+				Name:      "check",
+				Usage:     "Check quota consistency of a directory",
+				ArgsUsage: "META-URL",
+				Action:    quota,
+			},
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "path",
+				Usage: "full path of the directory within the volume",
+			},
+			&cli.Uint64Flag{
+				Name:  "capacity",
+				Usage: "hard quota of the directory limiting its usage of space in GiB",
+			},
+			&cli.Uint64Flag{
+				Name:  "inodes",
+				Usage: "hard quota of the directory limiting its number of inodes",
+			},
+		},
+	}
+}
+
+func quota(c *cli.Context) error {
+	setup(c, 1)
+	var cmd uint8
+	switch c.Command.Name {
+	case "set":
+		cmd = meta.QuotaSet
+	case "get":
+		cmd = meta.QuotaGet
+	case "del":
+		cmd = meta.QuotaDel
+	case "ls":
+		cmd = meta.QuotaList
+	case "check":
+		cmd = meta.QuotaCheck
+	default:
+		logger.Fatalf("Invalid quota command: %s", c.Command.Name)
+	}
+	dpath := c.String("path")
+	if dpath == "" && cmd != meta.QuotaList {
+		logger.Fatalf("Please specify the directory with `--path <dir>` option")
+	}
+	removePassword(c.Args().Get(0))
+
+	m := meta.NewClient(c.Args().Get(0), nil)
+	qs := make(map[string]*meta.Quota)
+	if cmd == meta.QuotaSet {
+		q := &meta.Quota{MaxSpace: -1, MaxInodes: -1} // negative means no change
+		if c.IsSet("capacity") {
+			q.MaxSpace = int64(c.Uint64("capacity")) << 30
+		}
+		if c.IsSet("inodes") {
+			q.MaxInodes = int64(c.Uint64("inodes"))
+		}
+		qs[dpath] = q
+	}
+	if err := m.HandleQuota(meta.Background, cmd, dpath, qs); err != nil {
+		return err
+	}
+
+	for p, q := range qs {
+		// FIXME: need a better way to do print
+		fmt.Printf("%s: %+v\n", p, *q)
+	}
+	return nil
+}

cmd/rmr.go

@@ -42,25 +42,12 @@ $ juicefs rmr /mnt/jfs/foo`,
 	}
 }
 
-func openController(mp string) *os.File {
-	st, err := os.Stat(mp)
+func openController(path string) (*os.File, error) {
+	mp, err := findMountpoint(path)
 	if err != nil {
-		logger.Fatal(err)
+		return nil, err
 	}
-	if !st.IsDir() {
-		mp = filepath.Dir(mp)
-	}
-	for ; mp != "/"; mp = filepath.Dir(mp) {
-		f, err := os.OpenFile(filepath.Join(mp, ".control"), os.O_RDWR, 0)
-		if err == nil {
-			return f
-		}
-		if !os.IsNotExist(err) {
-			logger.Fatal(err)
-		}
-	}
-	logger.Fatalf("Path %s is not inside JuiceFS", mp)
-	panic("unreachable")
+	return os.OpenFile(filepath.Join(mp, ".control"), os.O_RDWR, 0)
 }
 
 func rmr(ctx *cli.Context) error {
@@ -84,9 +71,9 @@ func rmr(ctx *cli.Context) error {
 		if err != nil {
 			return fmt.Errorf("lookup inode for %s: %s", d, err)
 		}
-		f := openController(d)
-		if f == nil {
-			logger.Errorf("%s is not inside JuiceFS", path)
+		f, err := openController(d)
+		if err != nil {
+			logger.Errorf("Open control file for %s: %s", d, err)
 			continue
 		}
 		wb := utils.NewBuffer(8 + 8 + 1 + uint32(len(name)))

cmd/warmup.go

@@ -178,22 +178,15 @@ func warmup(ctx *cli.Context) error {
 
 	// find mount point
 	first := paths[0]
-	controller := openController(first)
-	if controller == nil {
-		logger.Fatalf("open control file for %s", first)
+	mp, err := findMountpoint(first)
+	if err != nil {
+		return err
 	}
-	defer controller.Close()
-
-	mp := first
-	for ; mp != "/"; mp = filepath.Dir(mp) {
-		inode, err := utils.GetFileInode(mp)
-		if err != nil {
-			logger.Fatalf("lookup inode for %s: %s", mp, err)
-		}
-		if inode == uint64(meta.RootInode) {
-			break
-		}
+	controller, err := openController(mp)
+	if err != nil {
+		return fmt.Errorf("open control file for %s: %s", first, err)
 	}
+	defer controller.Close()
 
 	threads := ctx.Uint("threads")
 	if threads == 0 {

docs/en/deployment/hadoop_java_sdk.md

@@ -33,6 +33,16 @@ If you want to use JuiceFS in a distributed environment, when creating a file sy
 
 Depending on the read and write load of computing tasks (such as Spark executor), JuiceFS Hadoop Java SDK may require an additional 4 * [`juicefs.memory-size`](#io-configurations) off-heap memory to speed up read and write performance. By default, it is recommended to configure at least 1.2GB of off-heap memory for compute tasks.
 
+### 5. Java runtime version
+
+JuiceFS Hadoop Java SDK is compiled with JDK 8 by default. If it needs to be used in a higher version of Java runtime (such as Java 17), the following options need to be added to the JVM parameters to allow the use of reflection API:
+
+```shell
+--add-exports=java.base/sun.nio.ch=ALL-UNNAMED
+```
+
+For more information on the above option, please refer to [official documentation](https://docs.oracle.com/en/java/javase/17/migrate/migrating-jdk-8-later-jdk-releases.html#GUID-7BB28E4D-99B3-4078-BDC4-FC24180CE82B).
+
 ## Install and compile the client
 
 ### Install the pre-compiled client
@@ -132,6 +142,7 @@ It is recommended to place the JAR file in a fixed location, and the other locat
 |-----------|---------------------------------------------------------------------------|
 | Spark     | `${SPARK_HOME}/jars`                                                      |
 | Presto    | `${PRESTO_HOME}/plugin/hive-hadoop2`                                      |
+| Trino     | `${TRINO_HOME}/plugin/hive`                                               |
 | Flink     | `${FLINK_HOME}/lib`                                                       |
 | StarRocks | `${StarRocks_HOME}/fe/lib/`, `${StarRocks_HOME}/be/lib/hadoop/common/lib` |
 

docs/en/guide/how_to_set_up_metadata_engine.md

@@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem';
 
 JuiceFS is a decoupled structure that separates data and metadata. Metadata can be stored in any supported database (called Metadata Engine). Many databases are supported and they all comes with different performance and intended scenarios, refer to [our docs](../benchmark/metadata_engines_benchmark.md) for comparison.
 
+## The storage usage of metadata {#storage-usage}
+
+The storage space required for metadata is related to the length of the file name, the type and length of the file, and extended attributes. It is difficult to accurately estimate the metadata storage space requirements of a file system. For simplicity, we can approximate based on the storage space required for a single small file without extended attributes.
+
+- **Key-Value Database** (e.g. Redis, TiKV): 300 bytes/file
+- **Relational Database** (e.g. SQLite, MySQL, PostgreSQL): 600 bytes/file
+
+When the average file is larger (over 64MB), or the file is frequently modified and has a lot of fragments, or there are many extended attributes, or the average file name is long (over 50 bytes), more storage space is needed.
+
+When you need to migrate between two types of metadata engines, you can use this method to estimate the required storage space. For example, if you want to migrate the metadata engine from a relational database (MySQL) to a key-value database (Redis), and the current usage of MySQL is 30GB, then the target Redis needs to prepare at least 15GB or more of memory. The reverse is also true.
+
 ## Redis
 
 JuiceFS requires Redis version 4.0 and above. Redis Cluster is also supported, but in order to avoid transactions across different Redis instances, JuiceFS puts all metadata for one file system on a single Redis instance.