cmd/sync: support existing and ignore-existing flag (#3566)

juicedata · May 8, 2023 · 8fc327a · 8fc327a
1 parent 31d54d6
commit 8fc327a
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 48 deletions.
diff --git a/cmd/sync.go b/cmd/sync.go
@@ -138,6 +138,15 @@ Supported storage systems: https://juicefs.com/docs/community/how_to_setup_objec
 				Name:  "include",
 				Usage: "don't exclude Key matching PATTERN, need to be used with \"--exclude\" option",
 			},
+			&cli.BoolFlag{
+				Name:    "existing",
+				Aliases: []string{"ignore-non-existing"},
+				Usage:   "skip creating new files on destination",
+			},
+			&cli.BoolFlag{
+				Name:  "ignore-existing",
+				Usage: "skip updating files that already exist on destination",
+			},
 			&cli.BoolFlag{
 				Name:    "links",
 				Aliases: []string{"l"},

diff --git a/pkg/sync/config.go b/pkg/sync/config.go
@@ -24,31 +24,33 @@ import (
 )
 
 type Config struct {
-	StorageClass string
-	Start        string
-	End          string
-	Threads      int
-	HTTPPort     int
-	Update       bool
-	ForceUpdate  bool
-	Perms        bool
-	Dry          bool
-	DeleteSrc    bool
-	DeleteDst    bool
-	Dirs         bool
-	Exclude      []string
-	Include      []string
-	Links        bool
-	Limit        int64
-	Manager      string
-	Workers      []string
-	BWLimit      int
-	NoHTTPS      bool
-	Verbose      bool
-	Quiet        bool
-	CheckAll     bool
-	CheckNew     bool
-	Env          map[string]string
+	StorageClass   string
+	Start          string
+	End            string
+	Threads        int
+	HTTPPort       int
+	Update         bool
+	ForceUpdate    bool
+	Perms          bool
+	Dry            bool
+	DeleteSrc      bool
+	DeleteDst      bool
+	Dirs           bool
+	Exclude        []string
+	Include        []string
+	Existing       bool
+	IgnoreExisting bool
+	Links          bool
+	Limit          int64
+	Manager        string
+	Workers        []string
+	BWLimit        int
+	NoHTTPS        bool
+	Verbose        bool
+	Quiet          bool
+	CheckAll       bool
+	CheckNew       bool
+	Env            map[string]string
 }
 
 func envList() []string {
@@ -122,29 +124,31 @@ func NewConfigFromCli(c *cli.Context) *Config {
 	}
 
 	cfg := &Config{
-		Start:       c.String("start"),
-		End:         c.String("end"),
-		Threads:     c.Int("threads"),
-		Update:      c.Bool("update"),
-		ForceUpdate: c.Bool("force-update"),
-		Perms:       c.Bool("perms"),
-		Dirs:        c.Bool("dirs"),
-		Dry:         c.Bool("dry"),
-		DeleteSrc:   c.Bool("delete-src"),
-		DeleteDst:   c.Bool("delete-dst"),
-		Exclude:     c.StringSlice("exclude"),
-		Include:     c.StringSlice("include"),
-		Links:       c.Bool("links"),
-		Limit:       c.Int64("limit"),
-		Workers:     c.StringSlice("worker"),
-		Manager:     c.String("manager"),
-		BWLimit:     c.Int("bwlimit"),
-		NoHTTPS:     c.Bool("no-https"),
-		Verbose:     c.Bool("verbose"),
-		Quiet:       c.Bool("quiet"),
-		CheckAll:    c.Bool("check-all"),
-		CheckNew:    c.Bool("check-new"),
-		Env:         make(map[string]string),
+		Start:          c.String("start"),
+		End:            c.String("end"),
+		Threads:        c.Int("threads"),
+		Update:         c.Bool("update"),
+		ForceUpdate:    c.Bool("force-update"),
+		Perms:          c.Bool("perms"),
+		Dirs:           c.Bool("dirs"),
+		Dry:            c.Bool("dry"),
+		DeleteSrc:      c.Bool("delete-src"),
+		DeleteDst:      c.Bool("delete-dst"),
+		Exclude:        c.StringSlice("exclude"),
+		Include:        c.StringSlice("include"),
+		Existing:       c.Bool("existing"),
+		IgnoreExisting: c.Bool("ignore-existing"),
+		Links:          c.Bool("links"),
+		Limit:          c.Int64("limit"),
+		Workers:        c.StringSlice("worker"),
+		Manager:        c.String("manager"),
+		BWLimit:        c.Int("bwlimit"),
+		NoHTTPS:        c.Bool("no-https"),
+		Verbose:        c.Bool("verbose"),
+		Quiet:          c.Bool("quiet"),
+		CheckAll:       c.Bool("check-all"),
+		CheckNew:       c.Bool("check-new"),
+		Env:            make(map[string]string),
 	}
 	if cfg.Threads <= 0 {
 		logger.Warnf("threads should be larger than 0, reset it to 1")

diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go
@@ -723,8 +723,19 @@ func produce(tasks chan<- object.Object, src, dst object.ObjectStorage, srckeys,
 
 		// FIXME: there is a race when source is modified during coping
 		if dstobj == nil || obj.Key() < dstobj.Key() {
+			if config.Existing {
+				skipped.Increment()
+				handled.Increment()
+				continue
+			}
 			tasks <- obj
 		} else { // obj.key == dstobj.key
+			if config.IgnoreExisting {
+				skipped.Increment()
+				handled.Increment()
+				dstobj = nil
+				continue
+			}
 			if config.ForceUpdate ||
 				(config.Update && obj.Mtime().Unix() > dstobj.Mtime().Unix()) ||
 				(!config.Update && obj.Size() != dstobj.Size()) {