diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index 8bae589b9240..229f4e65822f 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -824,6 +824,13 @@ type rule struct { include bool } +func parseRule(name, p string) rule { + if runtime.GOOS == "windows" { + p = strings.Replace(p, "\\", "/", -1) + } + return rule{pattern: p, include: name == "-include"} +} + func parseIncludeRules(args []string) (rules []rule) { l := len(args) for i, a := range args { @@ -835,14 +842,14 @@ func parseIncludeRules(args []string) (rules []rule) { logger.Warnf("ignore invalid pattern: %s %s", a, args[i+1]) continue } - rules = append(rules, rule{pattern: args[i+1], include: a == "-include"}) + rules = append(rules, parseRule(a, args[i+1])) } else if strings.HasPrefix(a, "-include=") || strings.HasPrefix(a, "-exclude=") { if s := strings.Split(a, "="); len(s) == 2 && s[1] != "" { if _, err := path.Match(s[1], "xxxx"); err != nil { logger.Warnf("ignore invalid pattern: %s", a) continue } - rules = append(rules, rule{pattern: s[1], include: strings.HasPrefix(a, "-include=")}) + rules = append(rules, parseRule(s[0], s[1])) } } } @@ -867,19 +874,57 @@ func filter(keys <-chan object.Object, rules []rule) <-chan object.Object { return r } -func suffixForPattern(path, pattern string) string { - if strings.HasPrefix(pattern, "/") || - strings.HasSuffix(pattern, "/") && !strings.HasSuffix(path, "/") { - return path +func matchPrefix(p, s []string) bool { + if len(p) == 0 || len(s) == 0 { + return len(p) == len(s) + } + first := p[0] + n := len(s) + switch { + case first == "***": + return true + case strings.Contains(first, "**"): + for i := 1; i <= n; i++ { + if ok, _ := path.Match(first, strings.Join(s[:i], "*")); ok && matchPrefix(p[1:], s[i:]) { + return true + } + } + return false + default: + ok, _ := path.Match(first, s[0]) + return ok && matchPrefix(p[1:], s[1:]) + } +} + +func matchSuffix(p, s []string) bool { + if len(p) == 0 { + return true + } + last := p[len(p)-1] + if len(s) == 0 { + return last == "***" } - n := strings.Count(strings.Trim(pattern, "/"), "/") - m := strings.Count(strings.Trim(path, "/"), "/") - if n >= m { - return path + prefix := p[:len(p)-1] + n := len(s) + switch { + case last == "***": + for i := 0; i < n; i++ { + if matchSuffix(prefix, s[:i]) { + return true + } + } + return false + case strings.Contains(last, "**"): + for i := 0; i < n; i++ { + if ok, _ := path.Match(last, strings.Join(s[i:], "*")); ok && matchSuffix(prefix, s[:i]) { + return true + } + } + return false + default: + ok, _ := path.Match(last, s[n-1]) + return ok && matchSuffix(prefix, s[:n-1]) } - parts := strings.Split(path, "/") - n = len(strings.Split(pattern, "/")) - return strings.Join(parts[len(parts)-n:], "/") } // Consistent with rsync behavior, the matching order is adjusted according to the order of the "include" and "exclude" options @@ -889,16 +934,20 @@ func matchKey(rules []rule, key string) bool { if parts[i] == "" { continue } - prefix := strings.Join(parts[:i+1], "/") for _, rule := range rules { - var s string - if i < len(parts)-1 && strings.HasSuffix(rule.pattern, "/") { - s = "/" + ps := parts[:i+1] + p := strings.Split(rule.pattern, "/") + if i < len(parts)-1 && (p[len(p)-1] == "" || p[len(p)-1] == "***") { + ps = append(append([]string{}, ps...), "") // don't overwrite parts } - suffix := suffixForPattern(prefix+s, rule.pattern) - ok, err := path.Match(rule.pattern, suffix) - if err != nil { - logger.Fatalf("match %s with %s: %v", rule.pattern, suffix, err) + var ok bool + if p[0] == "" { + if ps[0] != "" { + p = p[1:] + } + ok = matchPrefix(p, ps) + } else { + ok = matchSuffix(p, ps) } if ok { if rule.include { @@ -1093,13 +1142,7 @@ func Sync(src, dst object.ObjectStorage, config *Config) error { } if len(config.Exclude) > 0 { - rules := parseIncludeRules(os.Args) - if runtime.GOOS == "windows" && (strings.HasPrefix(src.String(), "file:") || strings.HasPrefix(dst.String(), "file:")) { - for _, r := range rules { - r.pattern = strings.Replace(r.pattern, "\\", "/", -1) - } - } - config.rules = rules + config.rules = parseIncludeRules(os.Args) } if config.Manager == "" { diff --git a/pkg/sync/sync_test.go b/pkg/sync/sync_test.go index ba79c664d8d2..57185603f5e2 100644 --- a/pkg/sync/sync_test.go +++ b/pkg/sync/sync_test.go @@ -271,11 +271,11 @@ func TestParseRules(t *testing.T) { }, { args: []string{"--exclude", "a", "--include", "b"}, - wantRules: []rule{{pattern: "a", include: false}, {pattern: "b", include: true}}, + wantRules: []rule{{pattern: "a"}, {pattern: "b", include: true}}, }, { args: []string{"--include", "a", "--test", "t", "--exclude", "b"}, - wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: false}}, + wantRules: []rule{{pattern: "a", include: true}, {pattern: "b"}}, }, { args: []string{"--include", "a", "--test", "t", "--exclude"}, @@ -283,19 +283,19 @@ func TestParseRules(t *testing.T) { }, { args: []string{"--include", "a", "--exclude", "b", "--include", "c", "--exclude", "d"}, - wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: false}, {pattern: "c", include: true}, {pattern: "d", include: false}}, + wantRules: []rule{{pattern: "a", include: true}, {pattern: "b"}, {pattern: "c", include: true}, {pattern: "d"}}, }, { args: []string{"--include", "a", "--include", "b", "--test", "--exclude", "c", "--exclude", "d"}, - wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}, {pattern: "d", include: false}}, + wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}, {pattern: "d"}}, }, { args: []string{"--include=a", "--include=b", "--exclude=c", "--exclude=d", "--test=aaa"}, - wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}, {pattern: "d", include: false}}, + wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}, {pattern: "d"}}, }, { args: []string{"-include=a", "--test", "t", "--include=b", "--exclude=c", "-exclude="}, - wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}}, + wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}}, }, } for _, tt := range tests { @@ -589,32 +589,6 @@ func testKeysEqual(objsCh <-chan object.Object, expectedKeys []string) error { return nil } -func TestSuffixForPath(t *testing.T) { - type tcase struct { - pattern string - key string - want string - } - tests := []tcase{ - {pattern: "a*", key: "a1", want: "a1"}, - {pattern: "/a*", key: "a1", want: "a1"}, - {pattern: "a*/", key: "a1", want: "a1"}, - {pattern: "a*/b*", key: "a1", want: "a1"}, - {pattern: "a*", key: "a1/b1", want: "b1"}, - {pattern: "/a*", key: "a1/b1", want: "a1/b1"}, - {pattern: "/a*", key: "/a1/b1", want: "/a1/b1"}, - {pattern: "/a*/b*/c*", key: "/a1/b1", want: "/a1/b1"}, - {pattern: "/a", key: "a1/b1/c1/d1", want: "a1/b1/c1/d1"}, - {pattern: "a*/", key: "a1/b1", want: "a1/b1"}, - {pattern: "a*/b*", key: "a1/b1", want: "a1/b1"}, - {pattern: "a*/b*", key: "a1/b1/c1/d1", want: "c1/d1"}, - } - for _, tt := range tests { - if got := suffixForPattern(tt.key, tt.pattern); !reflect.DeepEqual(got, tt.want) { - t.Errorf("suffixForPattern(%s, %s) = %v, want %v", tt.key, tt.pattern, got, tt.want) - } - } -} func TestMatchObjects(t *testing.T) { type tcase struct { rules []rule @@ -622,26 +596,54 @@ func TestMatchObjects(t *testing.T) { want bool } tests := []tcase{ - {rules: []rule{{pattern: "a*", include: false}}, key: "a1", want: false}, - {rules: []rule{{pattern: "a*/b*", include: false}}, key: "a1/b1", want: false}, - {rules: []rule{{pattern: "/a*", include: false}}, key: "/a1", want: false}, - {rules: []rule{{pattern: "/a", include: false}}, key: "/a1", want: true}, - {rules: []rule{{pattern: "/a/b/c", include: false}}, key: "/a1", want: true}, - {rules: []rule{{pattern: "a*/b?", include: false}}, key: "a1/b1/c2/d1", want: false}, - {rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/", want: true}, - {rules: []rule{{pattern: "a*/b?/c.txt", include: false}}, key: "a1/b1", want: true}, - {rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/b1/", want: false}, - {rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/b1/c.txt", want: false}, - {rules: []rule{{pattern: "a*/", include: false}}, key: "a1/b1", want: false}, - {rules: []rule{{pattern: "a*/b*/", include: false}}, key: "a1/b1/c1/d.txt/", want: false}, - {rules: []rule{{pattern: "/a*/b*", include: false}}, key: "/a1/b1/c1/d.txt/", want: false}, - {rules: []rule{{pattern: "a*/b*/c", include: false}}, key: "a1/b1/c1/d.txt/", want: true}, - {rules: []rule{{pattern: "a", include: false}}, key: "a/b/c/d/", want: false}, - {rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg", include: false}}, key: "a/pkg/c/a.go", want: false}, - {rules: []rule{{pattern: "a", include: false}, {pattern: "pkg", include: true}}, key: "a/pkg/c/a.go", want: false}, - {rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg", include: false}}, key: "", want: true}, - {rules: []rule{{pattern: "a", include: true}, {pattern: "b/", include: false}, {pattern: "c", include: true}}, key: "a/b/c", want: false}, - {rules: []rule{{pattern: "a/", include: true}, {pattern: "a", include: false}}, key: "a/b", want: true}, + {rules: []rule{{pattern: "a*"}}, key: "a1"}, + {rules: []rule{{pattern: "a*/b*"}}, key: "a1/b1"}, + {rules: []rule{{pattern: "/a*"}}, key: "/a1"}, + {rules: []rule{{pattern: "/a"}}, key: "/a1", want: true}, + {rules: []rule{{pattern: "/a/b/c"}}, key: "/a1", want: true}, + {rules: []rule{{pattern: "a*/b?"}}, key: "a1/b1/c2/d1"}, + {rules: []rule{{pattern: "a*/b?/"}}, key: "a1/", want: true}, + {rules: []rule{{pattern: "a*/b?/c.txt"}}, key: "a1/b1", want: true}, + {rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/"}, + {rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/c.txt"}, + {rules: []rule{{pattern: "a*/"}}, key: "a1/b1"}, + {rules: []rule{{pattern: "a*/b*/"}}, key: "a1/b1/c1/d.txt/"}, + {rules: []rule{{pattern: "/a*/b*"}}, key: "/a1/b1/c1/d.txt/"}, + {rules: []rule{{pattern: "a*/b*/c"}}, key: "a1/b1/c1/d.txt/", want: true}, + {rules: []rule{{pattern: "a"}}, key: "a/b/c/d/"}, + {rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg"}}, key: "a/pkg/c/a.go"}, + {rules: []rule{{pattern: "a"}, {pattern: "pkg", include: true}}, key: "a/pkg/c/a.go"}, + {rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg"}}, key: "", want: true}, + {rules: []rule{{pattern: "a", include: true}, {pattern: "b/"}, {pattern: "c", include: true}}, key: "a/b/c"}, + {rules: []rule{{pattern: "a/", include: true}, {pattern: "a"}}, key: "a/b", want: true}, + {rules: []rule{{pattern: "/***"}}, key: "a"}, + {rules: []rule{{pattern: "/***"}}, key: "a/b"}, + {rules: []rule{{pattern: "/a/***"}}, key: "a/"}, + {rules: []rule{{pattern: "/a/***"}}, key: "a/b"}, + {rules: []rule{{pattern: "/a/***"}}, key: "a/b/c"}, + {rules: []rule{{pattern: "/a/***"}}, key: "b/a/", want: true}, + {rules: []rule{{pattern: "a/***"}}, key: "a/"}, + {rules: []rule{{pattern: "a/***"}}, key: "a/b"}, + {rules: []rule{{pattern: "a/***"}}, key: "a/b/c"}, + {rules: []rule{{pattern: "a/***"}}, key: "d/a/b/c"}, + {rules: []rule{{pattern: "a/***"}}, key: "a", want: true}, + {rules: []rule{{pattern: "a/***"}}, key: "ba", want: true}, + {rules: []rule{{pattern: "a/***"}}, key: "ba/", want: true}, + {rules: []rule{{pattern: "*/a/***"}}, key: "/a/"}, + {rules: []rule{{pattern: "*/a/***"}}, key: "b/a/"}, + {rules: []rule{{pattern: "*/a/***"}}, key: "b/a/c"}, + {rules: []rule{{pattern: "/*/a/***"}}, key: "/b/a/"}, + {rules: []rule{{pattern: "/*/a/***"}}, key: "/b/a/c"}, + {rules: []rule{{pattern: "/*/a/***"}}, key: "c/b/a/", want: true}, + {rules: []rule{{pattern: "a/**/b"}}, key: "a/c/b"}, + {rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/b"}, + {rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/e/b"}, + {rules: []rule{{pattern: "/**/b"}}, key: "a/c/b"}, + {rules: []rule{{pattern: "/**/b"}}, key: "a/c/d/b/"}, + {rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/b/"}, + {rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/ab/", want: true}, + {rules: []rule{{pattern: "a**b"}}, key: "a/c/d/b/"}, + {rules: []rule{{pattern: "a**b"}}, key: "b/c/d/b/", want: true}, } for _, c := range tests { if got := matchKey(c.rules, c.key); got != c.want { @@ -649,3 +651,24 @@ func TestMatchObjects(t *testing.T) { } } } + +func TestParseFilterRule(t *testing.T) { + type tcase struct { + args []string + rules []rule + } + cases := []tcase{ + {[]string{"--include", "a"}, []rule{{pattern: "a", include: true}}}, + {[]string{"--exclude", "a", "--include", "b"}, []rule{{pattern: "a"}, {pattern: "b", include: true}}}, + {[]string{"--include", "a", "--test", "t", "--exclude", "b"}, []rule{{pattern: "a", include: true}, {pattern: "b"}}}, + {[]string{"--include=a", "--test", "t", "--exclude"}, []rule{{pattern: "a", include: true}}}, + {[]string{"--include", "a", "--test", "t", "--exclude"}, []rule{{pattern: "a", include: true}}}, + {[]string{"-include=", "a", "--test", "t", "--exclude=*"}, []rule{{pattern: "*"}}}, + } + + for _, c := range cases { + if got := parseIncludeRules(c.args); !reflect.DeepEqual(got, c.rules) { + t.Errorf("parseIncludeRules(%+v) = %v, want %v", c.args, got, c.rules) + } + } +}