From c573f2924a5e519ca30113fa82e0c346cdb81752 Mon Sep 17 00:00:00 2001 From: timfeirg Date: Tue, 2 Aug 2022 15:35:39 +0800 Subject: [PATCH] add / update fstab entry with --update-fstab closes https://github.com/juicedata/juicefs/issues/2432 --- .gitignore | 2 + cmd/bench_test.go | 2 +- cmd/fsck_test.go | 2 +- cmd/gc_test.go | 2 +- cmd/info_test.go | 2 +- cmd/integration_test.go | 2 +- cmd/mount.go | 138 +++++++++++++++++++++++++++++++++++++--- cmd/mount_test.go | 52 ++++++++++++--- cmd/mount_unix.go | 4 ++ cmd/rmr_test.go | 2 +- cmd/status_test.go | 2 +- cmd/warmup_test.go | 2 +- 12 files changed, 187 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index cefd9495da2c2..490f6adb2028a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ libjfs.h .DS_Store docs/node_modules cmd/cmd +*.dump +*.out diff --git a/cmd/bench_test.go b/cmd/bench_test.go index bd90a69153d7e..025fb02bfa30d 100644 --- a/cmd/bench_test.go +++ b/cmd/bench_test.go @@ -22,7 +22,7 @@ import ( ) func TestBench(t *testing.T) { - mountTemp(t, nil, false) + mountTemp(t, nil, []string{"--trash-days=0"}, nil) defer umountTemp(t) os.Setenv("SKIP_DROP_CACHES", "true") diff --git a/cmd/fsck_test.go b/cmd/fsck_test.go index ee9fc0306350f..b0b19a7984a0b 100644 --- a/cmd/fsck_test.go +++ b/cmd/fsck_test.go @@ -23,7 +23,7 @@ import ( ) func TestFsck(t *testing.T) { - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) for i := 0; i < 10; i++ { diff --git a/cmd/gc_test.go b/cmd/gc_test.go index f2a62f6ebdd87..cbb8862056fd4 100644 --- a/cmd/gc_test.go +++ b/cmd/gc_test.go @@ -65,7 +65,7 @@ func getFileCount(dir string) int { func TestGc(t *testing.T) { var bucket string - mountTemp(t, &bucket, false) + mountTemp(t, &bucket, []string{"--trash-days=0"}, nil) defer umountTemp(t) if err := writeSmallBlocks(testMountPoint); err != nil { diff --git a/cmd/info_test.go b/cmd/info_test.go index af89bf6fd8fda..f551efeb3c400 100644 --- a/cmd/info_test.go +++ b/cmd/info_test.go @@ -39,7 +39,7 @@ func TestInfo(t *testing.T) { patches := gomonkey.ApplyGlobalVar(os.Stdout, *tmpFile) defer patches.Reset() - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) if err = os.MkdirAll(fmt.Sprintf("%s/dir1", testMountPoint), 0777); err != nil { diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 6ea98fffd5733..e9c3648258ffc 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -74,7 +74,7 @@ func startWebdav(t *testing.T) { } func TestIntegration(t *testing.T) { - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) startGateway(t) startWebdav(t) diff --git a/cmd/mount.go b/cmd/mount.go index 967c2e4b435a2..cf244accb5fce 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -17,6 +17,7 @@ package cmd import ( + "bufio" "fmt" "net" "net/http" @@ -25,6 +26,7 @@ import ( "os/signal" "path/filepath" "runtime" + "sort" "strings" "syscall" "time" @@ -212,26 +214,33 @@ func daemonRun(c *cli.Context, addr string, vfsConf *vfs.Config, m meta.Meta) { } } } + _ = expandPathForEmbedded(addr) + // The default log to syslog is only in daemon mode. + utils.InitLoggers(!c.Bool("no-syslog")) + err := makeDaemon(c, vfsConf.Format.Name, vfsConf.Meta.MountPoint, m) + if err != nil { + logger.Fatalf("Failed to make daemon: %s", err) + } +} + +func expandPathForEmbedded(addr string) string { embeddedSchemes := []string{"sqlite3://", "badger://"} for _, es := range embeddedSchemes { if strings.HasPrefix(addr, es) { path := addr[len(es):] - path2, err := filepath.Abs(path) - if err == nil && path2 != path { + absPath, err := filepath.Abs(path) + if err == nil && absPath != path { for i, a := range os.Args { if a == addr { - os.Args[i] = es + path2 + expanded := es + absPath + os.Args[i] = expanded + return expanded } } } } } - // The default log to syslog is only in daemon mode. - utils.InitLoggers(!c.Bool("no-syslog")) - err := makeDaemon(c, vfsConf.Format.Name, vfsConf.Meta.MountPoint, m) - if err != nil { - logger.Fatalf("Failed to make daemon: %s", err) - } + return addr } func getVfsConf(c *cli.Context, metaConf *meta.Config, format *meta.Format, chunkConf *chunk.Config) *vfs.Config { @@ -392,6 +401,110 @@ func NewReloadableStorage(format *meta.Format, reload func() (*meta.Format, erro return holder, nil } +func tellFstabOptions(c *cli.Context) string { + opts := []string{""} + seen := make(map[string]bool) + for _, s := range os.Args[2:] { + s = strings.Split(s, "=")[0] + s = strings.TrimLeft(s, "-") + if !c.IsSet(s) { + continue + } + if seen[s] { + continue + } else { + seen[s] = true + } + if s == "o" { + opts = append(opts, c.String(s)) + } else if v := c.Bool(s); v { + opts = append(opts, s) + } else { + v := c.Generic(s) + formatted := fmt.Sprintf("%s", v) + if strings.HasPrefix(formatted, "[") && strings.HasSuffix(formatted, "]") { + trimmed := strings.TrimPrefix(formatted, "[") + trimmed = strings.TrimSuffix(trimmed, "]") + vals := strings.Fields(trimmed) + for _, val := range vals { + opts = append(opts, fmt.Sprintf("%s=%s", s, val)) + } + } else { + opts = append(opts, fmt.Sprintf("%s=%s", s, v)) + } + } + } + sort.Strings(opts) + return strings.Join(opts, ",") + +} + +var fstab = "/etc/fstab" + +func updateFstab(c *cli.Context) error { + if runtime.GOOS != "linux" { + logger.Infof("--update-fstab is ignored in %s", runtime.GOOS) + return nil + } + if _, err := os.Stat("/.dockerenv"); err == nil { + logger.Infoln("--update-fstab is ignored in container") + return nil + } + addr := expandPathForEmbedded(c.Args().Get(0)) + mp := c.Args().Get(1) + f, err := os.Open(fstab) + if err != nil { + return err + } + index, entryIndex := -1, -1 + var lines []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + lines = append(lines, line) + index += 1 + fields := strings.Fields(line) + if len(fields) < 6 { + continue + } + if fields[0] == addr && fields[1] == mp { + entryIndex = index + } + } + if err = scanner.Err(); err != nil { + return err + } + f.Close() + opts := tellFstabOptions(c) + entry := fmt.Sprintf("%s %s juicefs _netdev%s 0 0", addr, mp, opts) + if entryIndex >= 0 { + if entry == lines[entryIndex] { + return nil + } + lines[entryIndex] = entry + } else { + lines = append(lines, entry) + } + tempFstab := fstab + ".tmp" + tmpf, err := os.OpenFile(tempFstab, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + datawriter := bufio.NewWriter(tmpf) + for _, line := range lines { + if _, err := datawriter.WriteString(line + "\n"); err != nil { + return err + } + } + datawriter.Flush() + tmpf.Close() + err = os.Rename(tempFstab, fstab) + if err != nil { + return err + } + return nil +} + func mount(c *cli.Context) error { setup(c, 2) addr := c.Args().Get(0) @@ -421,6 +534,13 @@ func mount(c *cli.Context) error { } logger.Infof("Data use %s", blob) + if c.Bool("update-fstab") { + err = updateFstab(c) + if err != nil { + logger.Fatalf("failed to update fstab: %s", err) + } + } + chunkConf := getChunkConf(c, format) store := chunk.NewCachedStore(blob, *chunkConf, registerer) registerMetaMsg(metaCli, store, chunkConf) diff --git a/cmd/mount_test.go b/cmd/mount_test.go index 64dc996905d64..8e97a9f40c889 100644 --- a/cmd/mount_test.go +++ b/cmd/mount_test.go @@ -20,10 +20,12 @@ import ( "context" "fmt" "io" + "io/ioutil" "net/http" "net/url" "os" "reflect" + "runtime" "testing" "time" @@ -92,25 +94,29 @@ func resetTestMeta() *redis.Client { // using Redis return rdb } -func mountTemp(t *testing.T, bucket *string, trash bool) { +func mountTemp(t *testing.T, bucket *string, extraFormatOpts []string, extraMountOpts []string) { _ = resetTestMeta() testDir := t.TempDir() if bucket != nil { *bucket = testDir } - args := []string{"", "format", "--bucket", testDir, testMeta, testVolume} - if !trash { - args = append(args, "--trash-days=0") + formatArgs := []string{"", "format", "--bucket", testDir, testMeta, testVolume} + if extraFormatOpts != nil { + formatArgs = append(formatArgs, extraFormatOpts...) } - if err := Main(args); err != nil { + if err := Main(formatArgs); err != nil { t.Fatalf("format failed: %s", err) } // must do reset, otherwise will panic ResetHttp() + mountArgs := []string{"", "mount", "--enable-xattr", testMeta, testMountPoint, "--no-usage-report"} + if extraMountOpts != nil { + mountArgs = append(mountArgs, extraMountOpts...) + } go func() { - if err := Main([]string{"", "mount", "--enable-xattr", testMeta, testMountPoint, "--no-usage-report"}); err != nil { + if err := Main(mountArgs); err != nil { t.Errorf("mount failed: %s", err) } }() @@ -124,7 +130,7 @@ func umountTemp(t *testing.T) { } func TestMount(t *testing.T) { - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) if err := os.WriteFile(fmt.Sprintf("%s/f1.txt", testMountPoint), []byte("test"), 0644); err != nil { @@ -132,8 +138,38 @@ func TestMount(t *testing.T) { } } +func TestUpdateFstab(t *testing.T) { + if runtime.GOOS != "linux" { + t.SkipNow() + } + mockFstab, err := ioutil.TempFile("/tmp", "fstab") + if err != nil { + t.Fatalf("cannot make temp file: %s", err) + } + defer os.Remove(mockFstab.Name()) + + patches := gomonkey.ApplyGlobalVar(&fstab, mockFstab.Name()) + defer patches.Reset() + mountArgs := []string{"juicefs", "mount", "--enable-xattr", testMeta, testMountPoint, "--no-usage-report"} + mountOpts := []string{"--update-fstab", "--writeback", "--entry-cache=2", "--max-uploads", "3", "-o", "max_read=99"} + patches = gomonkey.ApplyGlobalVar(&os.Args, append(mountArgs, mountOpts...)) + defer patches.Reset() + mountTemp(t, nil, nil, mountOpts) + defer umountTemp(t) + + content, err := ioutil.ReadFile(mockFstab.Name()) + if err != nil { + t.Fatalf("error reading mocked fstab: %s", err) + } + rv := "redis://127.0.0.1:6379/11 /tmp/jfs-unit-test juicefs _netdev,enable-xattr,entry-cache=2,max-uploads=3,max_read=99,no-usage-report,update-fstab,writeback 0 0" + lv := string(content) + if lv != rv { + t.Logf("incorrect fstab entry: %s", lv) + } +} + func TestUmount(t *testing.T) { - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) umountTemp(t) inode, err := utils.GetFileInode(testMountPoint) diff --git a/cmd/mount_unix.go b/cmd/mount_unix.go index a4ce11073cb24..d8664003ba61f 100644 --- a/cmd/mount_unix.go +++ b/cmd/mount_unix.go @@ -136,6 +136,10 @@ func mount_flags() []cli.Flag { Name: "enable-xattr", Usage: "enable extended attributes (xattr)", }, + &cli.BoolFlag{ + Name: "update-fstab", + Usage: "add / update entry in /etc/fstab", + }, } return append(selfFlags, cacheFlags(1.0)...) } diff --git a/cmd/rmr_test.go b/cmd/rmr_test.go index 073473e3d360a..8816a3256830f 100644 --- a/cmd/rmr_test.go +++ b/cmd/rmr_test.go @@ -23,7 +23,7 @@ import ( ) func TestRmr(t *testing.T) { - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) paths := []string{"/dir1", "/dir2", "/dir3/dir2"} diff --git a/cmd/status_test.go b/cmd/status_test.go index 46c19dbbd1eca..b370e86a96d29 100644 --- a/cmd/status_test.go +++ b/cmd/status_test.go @@ -38,7 +38,7 @@ func TestStatus(t *testing.T) { patches := gomonkey.ApplyGlobalVar(os.Stdout, *tmpFile) defer patches.Reset() - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) if err = Main([]string{"", "status", testMeta}); err != nil { diff --git a/cmd/warmup_test.go b/cmd/warmup_test.go index ac0255de17a50..0977d19a7a144 100644 --- a/cmd/warmup_test.go +++ b/cmd/warmup_test.go @@ -27,7 +27,7 @@ import ( ) func TestWarmup(t *testing.T) { - mountTemp(t, nil, true) + mountTemp(t, nil, nil, nil) defer umountTemp(t) if err := os.WriteFile(fmt.Sprintf("%s/f1.txt", testMountPoint), []byte("test"), 0644); err != nil {