Skip to content

Commit

Permalink
sync: support three star to match all files recursively (#4422)
Browse files Browse the repository at this point in the history
  • Loading branch information
@davies
davies authored Feb 29, 2024
1 parent f8c988f commit 58722dd
Show file tree
Hide file tree
Showing 2 changed files with 146 additions and 80 deletions.
99 changes: 71 additions & 28 deletions pkg/sync/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,13 @@ type rule struct {
include bool
}

func parseRule(name, p string) rule {
if runtime.GOOS == "windows" {
p = strings.Replace(p, "\\", "/", -1)
}
return rule{pattern: p, include: name == "-include"}
}

func parseIncludeRules(args []string) (rules []rule) {
l := len(args)
for i, a := range args {
Expand All @@ -835,14 +842,14 @@ func parseIncludeRules(args []string) (rules []rule) {
logger.Warnf("ignore invalid pattern: %s %s", a, args[i+1])
continue
}
rules = append(rules, rule{pattern: args[i+1], include: a == "-include"})
rules = append(rules, parseRule(a, args[i+1]))
} else if strings.HasPrefix(a, "-include=") || strings.HasPrefix(a, "-exclude=") {
if s := strings.Split(a, "="); len(s) == 2 && s[1] != "" {
if _, err := path.Match(s[1], "xxxx"); err != nil {
logger.Warnf("ignore invalid pattern: %s", a)
continue
}
rules = append(rules, rule{pattern: s[1], include: strings.HasPrefix(a, "-include=")})
rules = append(rules, parseRule(s[0], s[1]))
}
}
}
Expand All @@ -867,19 +874,57 @@ func filter(keys <-chan object.Object, rules []rule) <-chan object.Object {
return r
}

func suffixForPattern(path, pattern string) string {
if strings.HasPrefix(pattern, "/") ||
strings.HasSuffix(pattern, "/") && !strings.HasSuffix(path, "/") {
return path
func matchPrefix(p, s []string) bool {
if len(p) == 0 || len(s) == 0 {
return len(p) == len(s)
}
first := p[0]
n := len(s)
switch {
case first == "***":
return true
case strings.Contains(first, "**"):
for i := 1; i <= n; i++ {
if ok, _ := path.Match(first, strings.Join(s[:i], "*")); ok && matchPrefix(p[1:], s[i:]) {
return true
}
}
return false
default:
ok, _ := path.Match(first, s[0])
return ok && matchPrefix(p[1:], s[1:])
}
}

func matchSuffix(p, s []string) bool {
if len(p) == 0 {
return true
}
last := p[len(p)-1]
if len(s) == 0 {
return last == "***"
}
n := strings.Count(strings.Trim(pattern, "/"), "/")
m := strings.Count(strings.Trim(path, "/"), "/")
if n >= m {
return path
prefix := p[:len(p)-1]
n := len(s)
switch {
case last == "***":
for i := 0; i < n; i++ {
if matchSuffix(prefix, s[:i]) {
return true
}
}
return false
case strings.Contains(last, "**"):
for i := 0; i < n; i++ {
if ok, _ := path.Match(last, strings.Join(s[i:], "*")); ok && matchSuffix(prefix, s[:i]) {
return true
}
}
return false
default:
ok, _ := path.Match(last, s[n-1])
return ok && matchSuffix(prefix, s[:n-1])
}
parts := strings.Split(path, "/")
n = len(strings.Split(pattern, "/"))
return strings.Join(parts[len(parts)-n:], "/")
}

// Consistent with rsync behavior, the matching order is adjusted according to the order of the "include" and "exclude" options
Expand All @@ -889,16 +934,20 @@ func matchKey(rules []rule, key string) bool {
if parts[i] == "" {
continue
}
prefix := strings.Join(parts[:i+1], "/")
for _, rule := range rules {
var s string
if i < len(parts)-1 && strings.HasSuffix(rule.pattern, "/") {
s = "/"
ps := parts[:i+1]
p := strings.Split(rule.pattern, "/")
if i < len(parts)-1 && (p[len(p)-1] == "" || p[len(p)-1] == "***") {
ps = append(append([]string{}, ps...), "") // don't overwrite parts
}
suffix := suffixForPattern(prefix+s, rule.pattern)
ok, err := path.Match(rule.pattern, suffix)
if err != nil {
logger.Fatalf("match %s with %s: %v", rule.pattern, suffix, err)
var ok bool
if p[0] == "" {
if ps[0] != "" {
p = p[1:]
}
ok = matchPrefix(p, ps)
} else {
ok = matchSuffix(p, ps)
}
if ok {
if rule.include {
Expand Down Expand Up @@ -1093,13 +1142,7 @@ func Sync(src, dst object.ObjectStorage, config *Config) error {
}

if len(config.Exclude) > 0 {
rules := parseIncludeRules(os.Args)
if runtime.GOOS == "windows" && (strings.HasPrefix(src.String(), "file:") || strings.HasPrefix(dst.String(), "file:")) {
for _, r := range rules {
r.pattern = strings.Replace(r.pattern, "\\", "/", -1)
}
}
config.rules = rules
config.rules = parseIncludeRules(os.Args)
}

if config.Manager == "" {
Expand Down
127 changes: 75 additions & 52 deletions pkg/sync/sync_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,31 +271,31 @@ func TestParseRules(t *testing.T) {
},
{
args: []string{"--exclude", "a", "--include", "b"},
wantRules: []rule{{pattern: "a", include: false}, {pattern: "b", include: true}},
wantRules: []rule{{pattern: "a"}, {pattern: "b", include: true}},
},
{
args: []string{"--include", "a", "--test", "t", "--exclude", "b"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b"}},
},
{
args: []string{"--include", "a", "--test", "t", "--exclude"},
wantRules: []rule{{pattern: "a", include: true}},
},
{
args: []string{"--include", "a", "--exclude", "b", "--include", "c", "--exclude", "d"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: false}, {pattern: "c", include: true}, {pattern: "d", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b"}, {pattern: "c", include: true}, {pattern: "d"}},
},
{
args: []string{"--include", "a", "--include", "b", "--test", "--exclude", "c", "--exclude", "d"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}, {pattern: "d", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}, {pattern: "d"}},
},
{
args: []string{"--include=a", "--include=b", "--exclude=c", "--exclude=d", "--test=aaa"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}, {pattern: "d", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}, {pattern: "d"}},
},
{
args: []string{"-include=a", "--test", "t", "--include=b", "--exclude=c", "-exclude="},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}},
},
}
for _, tt := range tests {
Expand Down Expand Up @@ -589,63 +589,86 @@ func testKeysEqual(objsCh <-chan object.Object, expectedKeys []string) error {
return nil
}

func TestSuffixForPath(t *testing.T) {
type tcase struct {
pattern string
key string
want string
}
tests := []tcase{
{pattern: "a*", key: "a1", want: "a1"},
{pattern: "/a*", key: "a1", want: "a1"},
{pattern: "a*/", key: "a1", want: "a1"},
{pattern: "a*/b*", key: "a1", want: "a1"},
{pattern: "a*", key: "a1/b1", want: "b1"},
{pattern: "/a*", key: "a1/b1", want: "a1/b1"},
{pattern: "/a*", key: "/a1/b1", want: "/a1/b1"},
{pattern: "/a*/b*/c*", key: "/a1/b1", want: "/a1/b1"},
{pattern: "/a", key: "a1/b1/c1/d1", want: "a1/b1/c1/d1"},
{pattern: "a*/", key: "a1/b1", want: "a1/b1"},
{pattern: "a*/b*", key: "a1/b1", want: "a1/b1"},
{pattern: "a*/b*", key: "a1/b1/c1/d1", want: "c1/d1"},
}
for _, tt := range tests {
if got := suffixForPattern(tt.key, tt.pattern); !reflect.DeepEqual(got, tt.want) {
t.Errorf("suffixForPattern(%s, %s) = %v, want %v", tt.key, tt.pattern, got, tt.want)
}
}
}
func TestMatchObjects(t *testing.T) {
type tcase struct {
rules []rule
key string
want bool
}
tests := []tcase{
{rules: []rule{{pattern: "a*", include: false}}, key: "a1", want: false},
{rules: []rule{{pattern: "a*/b*", include: false}}, key: "a1/b1", want: false},
{rules: []rule{{pattern: "/a*", include: false}}, key: "/a1", want: false},
{rules: []rule{{pattern: "/a", include: false}}, key: "/a1", want: true},
{rules: []rule{{pattern: "/a/b/c", include: false}}, key: "/a1", want: true},
{rules: []rule{{pattern: "a*/b?", include: false}}, key: "a1/b1/c2/d1", want: false},
{rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/", want: true},
{rules: []rule{{pattern: "a*/b?/c.txt", include: false}}, key: "a1/b1", want: true},
{rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/b1/", want: false},
{rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/b1/c.txt", want: false},
{rules: []rule{{pattern: "a*/", include: false}}, key: "a1/b1", want: false},
{rules: []rule{{pattern: "a*/b*/", include: false}}, key: "a1/b1/c1/d.txt/", want: false},
{rules: []rule{{pattern: "/a*/b*", include: false}}, key: "/a1/b1/c1/d.txt/", want: false},
{rules: []rule{{pattern: "a*/b*/c", include: false}}, key: "a1/b1/c1/d.txt/", want: true},
{rules: []rule{{pattern: "a", include: false}}, key: "a/b/c/d/", want: false},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg", include: false}}, key: "a/pkg/c/a.go", want: false},
{rules: []rule{{pattern: "a", include: false}, {pattern: "pkg", include: true}}, key: "a/pkg/c/a.go", want: false},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg", include: false}}, key: "", want: true},
{rules: []rule{{pattern: "a", include: true}, {pattern: "b/", include: false}, {pattern: "c", include: true}}, key: "a/b/c", want: false},
{rules: []rule{{pattern: "a/", include: true}, {pattern: "a", include: false}}, key: "a/b", want: true},
{rules: []rule{{pattern: "a*"}}, key: "a1"},
{rules: []rule{{pattern: "a*/b*"}}, key: "a1/b1"},
{rules: []rule{{pattern: "/a*"}}, key: "/a1"},
{rules: []rule{{pattern: "/a"}}, key: "/a1", want: true},
{rules: []rule{{pattern: "/a/b/c"}}, key: "/a1", want: true},
{rules: []rule{{pattern: "a*/b?"}}, key: "a1/b1/c2/d1"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/", want: true},
{rules: []rule{{pattern: "a*/b?/c.txt"}}, key: "a1/b1", want: true},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/c.txt"},
{rules: []rule{{pattern: "a*/"}}, key: "a1/b1"},
{rules: []rule{{pattern: "a*/b*/"}}, key: "a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "/a*/b*"}}, key: "/a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "a*/b*/c"}}, key: "a1/b1/c1/d.txt/", want: true},
{rules: []rule{{pattern: "a"}}, key: "a/b/c/d/"},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg"}}, key: "a/pkg/c/a.go"},
{rules: []rule{{pattern: "a"}, {pattern: "pkg", include: true}}, key: "a/pkg/c/a.go"},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg"}}, key: "", want: true},
{rules: []rule{{pattern: "a", include: true}, {pattern: "b/"}, {pattern: "c", include: true}}, key: "a/b/c"},
{rules: []rule{{pattern: "a/", include: true}, {pattern: "a"}}, key: "a/b", want: true},
{rules: []rule{{pattern: "/***"}}, key: "a"},
{rules: []rule{{pattern: "/***"}}, key: "a/b"},
{rules: []rule{{pattern: "/a/***"}}, key: "a/"},
{rules: []rule{{pattern: "/a/***"}}, key: "a/b"},
{rules: []rule{{pattern: "/a/***"}}, key: "a/b/c"},
{rules: []rule{{pattern: "/a/***"}}, key: "b/a/", want: true},
{rules: []rule{{pattern: "a/***"}}, key: "a/"},
{rules: []rule{{pattern: "a/***"}}, key: "a/b"},
{rules: []rule{{pattern: "a/***"}}, key: "a/b/c"},
{rules: []rule{{pattern: "a/***"}}, key: "d/a/b/c"},
{rules: []rule{{pattern: "a/***"}}, key: "a", want: true},
{rules: []rule{{pattern: "a/***"}}, key: "ba", want: true},
{rules: []rule{{pattern: "a/***"}}, key: "ba/", want: true},
{rules: []rule{{pattern: "*/a/***"}}, key: "/a/"},
{rules: []rule{{pattern: "*/a/***"}}, key: "b/a/"},
{rules: []rule{{pattern: "*/a/***"}}, key: "b/a/c"},
{rules: []rule{{pattern: "/*/a/***"}}, key: "/b/a/"},
{rules: []rule{{pattern: "/*/a/***"}}, key: "/b/a/c"},
{rules: []rule{{pattern: "/*/a/***"}}, key: "c/b/a/", want: true},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/b"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/b"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/e/b"},
{rules: []rule{{pattern: "/**/b"}}, key: "a/c/b"},
{rules: []rule{{pattern: "/**/b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/ab/", want: true},
{rules: []rule{{pattern: "a**b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a**b"}}, key: "b/c/d/b/", want: true},
}
for _, c := range tests {
if got := matchKey(c.rules, c.key); got != c.want {
t.Errorf("matchKey(%+v, %s) = %v, want %v", c.rules, c.key, got, c.want)
}
}
}

func TestParseFilterRule(t *testing.T) {
type tcase struct {
args []string
rules []rule
}
cases := []tcase{
{[]string{"--include", "a"}, []rule{{pattern: "a", include: true}}},
{[]string{"--exclude", "a", "--include", "b"}, []rule{{pattern: "a"}, {pattern: "b", include: true}}},
{[]string{"--include", "a", "--test", "t", "--exclude", "b"}, []rule{{pattern: "a", include: true}, {pattern: "b"}}},
{[]string{"--include=a", "--test", "t", "--exclude"}, []rule{{pattern: "a", include: true}}},
{[]string{"--include", "a", "--test", "t", "--exclude"}, []rule{{pattern: "a", include: true}}},
{[]string{"-include=", "a", "--test", "t", "--exclude=*"}, []rule{{pattern: "*"}}},
}

for _, c := range cases {
if got := parseIncludeRules(c.args); !reflect.DeepEqual(got, c.rules) {
t.Errorf("parseIncludeRules(%+v) = %v, want %v", c.args, got, c.rules)
}
}
}

0 comments on commit 58722dd

Please sign in to comment.