Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sync: support three star to match all files recursively #4422

Merged
merged 5 commits into from
Feb 29, 2024
Merged

sync: support three star to match all files recursively #4422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
128c884
sync: support three star to match all files recursively
davies Feb 23, 2024
95dbd19
fix unit test
davies Feb 24, 2024
bb2c1e6
support two star
davies Feb 27, 2024
73d50d4
add test for parse
davies Feb 28, 2024
c8c63d9
review
davies Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 71 additions & 28 deletions pkg/sync/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,13 @@ type rule struct {
include bool
}

func parseRule(name, p string) rule {
if runtime.GOOS == "windows" {
p = strings.Replace(p, "\\", "/", -1)
}
return rule{pattern: p, include: name == "-include"}
}

func parseIncludeRules(args []string) (rules []rule) {
l := len(args)
for i, a := range args {
Expand All @@ -834,14 +841,14 @@ func parseIncludeRules(args []string) (rules []rule) {
logger.Warnf("ignore invalid pattern: %s %s", a, args[i+1])
continue
}
rules = append(rules, rule{pattern: args[i+1], include: a == "-include"})
rules = append(rules, parseRule(a, args[i+1]))
} else if strings.HasPrefix(a, "-include=") || strings.HasPrefix(a, "-exclude=") {
if s := strings.Split(a, "="); len(s) == 2 && s[1] != "" {
if _, err := path.Match(s[1], "xxxx"); err != nil {
logger.Warnf("ignore invalid pattern: %s", a)
continue
}
rules = append(rules, rule{pattern: s[1], include: strings.HasPrefix(a, "-include=")})
rules = append(rules, parseRule(s[0], s[1]))
}
}
}
Expand All @@ -866,19 +873,57 @@ func filter(keys <-chan object.Object, rules []rule) <-chan object.Object {
return r
}

func suffixForPattern(path, pattern string) string {
if strings.HasPrefix(pattern, "/") ||
strings.HasSuffix(pattern, "/") && !strings.HasSuffix(path, "/") {
return path
func matchPrefix(p, s []string) bool {
if len(p) == 0 || len(s) == 0 {
return len(p) == len(s)
}
first := p[0]
n := len(s)
switch {
case first == "***":
return true
case strings.Contains(first, "**"):
for i := 1; i <= n; i++ {
if ok, _ := path.Match(first, strings.Join(s[:i], "*")); ok && matchPrefix(p[1:], s[i:]) {
return true
}
}
return false
default:
ok, _ := path.Match(first, s[0])
return ok && matchPrefix(p[1:], s[1:])
}
}

func matchSuffix(p, s []string) bool {
if len(p) == 0 {
return true
}
last := p[len(p)-1]
if len(s) == 0 {
return last == "***"
}
n := strings.Count(strings.Trim(pattern, "/"), "/")
m := strings.Count(strings.Trim(path, "/"), "/")
if n >= m {
return path
prefix := p[:len(p)-1]
n := len(s)
switch {
case last == "***":
for i := 0; i < n; i++ {
if matchSuffix(prefix, s[:i]) {
return true
}
}
return false
case strings.Contains(last, "**"):
for i := 0; i < n; i++ {
if ok, _ := path.Match(last, strings.Join(s[i:], "*")); ok && matchSuffix(prefix, s[:i]) {
return true
}
}
return false
default:
ok, _ := path.Match(last, s[n-1])
return ok && matchSuffix(prefix, s[:n-1])
}
parts := strings.Split(path, "/")
n = len(strings.Split(pattern, "/"))
return strings.Join(parts[len(parts)-n:], "/")
}

// Consistent with rsync behavior, the matching order is adjusted according to the order of the "include" and "exclude" options
Expand All @@ -888,16 +933,20 @@ func matchKey(rules []rule, key string) bool {
if parts[i] == "" {
continue
}
prefix := strings.Join(parts[:i+1], "/")
for _, rule := range rules {
var s string
if i < len(parts)-1 && strings.HasSuffix(rule.pattern, "/") {
s = "/"
ps := parts[:i+1]
p := strings.Split(rule.pattern, "/")
if i < len(parts)-1 && (p[len(p)-1] == "" || p[len(p)-1] == "***") {
ps = append(append([]string{}, ps...), "") // don't overwrite parts
}
suffix := suffixForPattern(prefix+s, rule.pattern)
ok, err := path.Match(rule.pattern, suffix)
if err != nil {
logger.Fatalf("match %s with %s: %v", rule.pattern, suffix, err)
var ok bool
if p[0] == "" {
if ps[0] != "" {
p = p[1:]
}
ok = matchPrefix(p, ps)
} else {
ok = matchSuffix(p, ps)
}
if ok {
if rule.include {
Expand Down Expand Up @@ -1092,13 +1141,7 @@ func Sync(src, dst object.ObjectStorage, config *Config) error {
}

if len(config.Exclude) > 0 {
rules := parseIncludeRules(os.Args)
if runtime.GOOS == "windows" && (strings.HasPrefix(src.String(), "file:") || strings.HasPrefix(dst.String(), "file:")) {
for _, r := range rules {
r.pattern = strings.Replace(r.pattern, "\\", "/", -1)
}
}
config.rules = rules
config.rules = parseIncludeRules(os.Args)
}

if config.Manager == "" {
Expand Down
127 changes: 75 additions & 52 deletions pkg/sync/sync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,31 +271,31 @@ func TestParseRules(t *testing.T) {
},
{
args: []string{"--exclude", "a", "--include", "b"},
wantRules: []rule{{pattern: "a", include: false}, {pattern: "b", include: true}},
wantRules: []rule{{pattern: "a"}, {pattern: "b", include: true}},
},
{
args: []string{"--include", "a", "--test", "t", "--exclude", "b"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b"}},
},
{
args: []string{"--include", "a", "--test", "t", "--exclude"},
wantRules: []rule{{pattern: "a", include: true}},
},
{
args: []string{"--include", "a", "--exclude", "b", "--include", "c", "--exclude", "d"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: false}, {pattern: "c", include: true}, {pattern: "d", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b"}, {pattern: "c", include: true}, {pattern: "d"}},
},
{
args: []string{"--include", "a", "--include", "b", "--test", "--exclude", "c", "--exclude", "d"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}, {pattern: "d", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}, {pattern: "d"}},
},
{
args: []string{"--include=a", "--include=b", "--exclude=c", "--exclude=d", "--test=aaa"},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}, {pattern: "d", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}, {pattern: "d"}},
},
{
args: []string{"-include=a", "--test", "t", "--include=b", "--exclude=c", "-exclude="},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c", include: false}},
wantRules: []rule{{pattern: "a", include: true}, {pattern: "b", include: true}, {pattern: "c"}},
},
}
for _, tt := range tests {
Expand Down Expand Up @@ -589,63 +589,86 @@ func testKeysEqual(objsCh <-chan object.Object, expectedKeys []string) error {
return nil
}

func TestSuffixForPath(t *testing.T) {
type tcase struct {
pattern string
key string
want string
}
tests := []tcase{
{pattern: "a*", key: "a1", want: "a1"},
{pattern: "/a*", key: "a1", want: "a1"},
{pattern: "a*/", key: "a1", want: "a1"},
{pattern: "a*/b*", key: "a1", want: "a1"},
{pattern: "a*", key: "a1/b1", want: "b1"},
{pattern: "/a*", key: "a1/b1", want: "a1/b1"},
{pattern: "/a*", key: "/a1/b1", want: "/a1/b1"},
{pattern: "/a*/b*/c*", key: "/a1/b1", want: "/a1/b1"},
{pattern: "/a", key: "a1/b1/c1/d1", want: "a1/b1/c1/d1"},
{pattern: "a*/", key: "a1/b1", want: "a1/b1"},
{pattern: "a*/b*", key: "a1/b1", want: "a1/b1"},
{pattern: "a*/b*", key: "a1/b1/c1/d1", want: "c1/d1"},
}
for _, tt := range tests {
if got := suffixForPattern(tt.key, tt.pattern); !reflect.DeepEqual(got, tt.want) {
t.Errorf("suffixForPattern(%s, %s) = %v, want %v", tt.key, tt.pattern, got, tt.want)
}
}
}
func TestMatchObjects(t *testing.T) {
type tcase struct {
rules []rule
key string
want bool
}
tests := []tcase{
{rules: []rule{{pattern: "a*", include: false}}, key: "a1", want: false},
{rules: []rule{{pattern: "a*/b*", include: false}}, key: "a1/b1", want: false},
{rules: []rule{{pattern: "/a*", include: false}}, key: "/a1", want: false},
{rules: []rule{{pattern: "/a", include: false}}, key: "/a1", want: true},
{rules: []rule{{pattern: "/a/b/c", include: false}}, key: "/a1", want: true},
{rules: []rule{{pattern: "a*/b?", include: false}}, key: "a1/b1/c2/d1", want: false},
{rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/", want: true},
{rules: []rule{{pattern: "a*/b?/c.txt", include: false}}, key: "a1/b1", want: true},
{rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/b1/", want: false},
{rules: []rule{{pattern: "a*/b?/", include: false}}, key: "a1/b1/c.txt", want: false},
{rules: []rule{{pattern: "a*/", include: false}}, key: "a1/b1", want: false},
{rules: []rule{{pattern: "a*/b*/", include: false}}, key: "a1/b1/c1/d.txt/", want: false},
{rules: []rule{{pattern: "/a*/b*", include: false}}, key: "/a1/b1/c1/d.txt/", want: false},
{rules: []rule{{pattern: "a*/b*/c", include: false}}, key: "a1/b1/c1/d.txt/", want: true},
{rules: []rule{{pattern: "a", include: false}}, key: "a/b/c/d/", want: false},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg", include: false}}, key: "a/pkg/c/a.go", want: false},
{rules: []rule{{pattern: "a", include: false}, {pattern: "pkg", include: true}}, key: "a/pkg/c/a.go", want: false},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg", include: false}}, key: "", want: true},
{rules: []rule{{pattern: "a", include: true}, {pattern: "b/", include: false}, {pattern: "c", include: true}}, key: "a/b/c", want: false},
{rules: []rule{{pattern: "a/", include: true}, {pattern: "a", include: false}}, key: "a/b", want: true},
{rules: []rule{{pattern: "a*"}}, key: "a1"},
{rules: []rule{{pattern: "a*/b*"}}, key: "a1/b1"},
{rules: []rule{{pattern: "/a*"}}, key: "/a1"},
{rules: []rule{{pattern: "/a"}}, key: "/a1", want: true},
{rules: []rule{{pattern: "/a/b/c"}}, key: "/a1", want: true},
{rules: []rule{{pattern: "a*/b?"}}, key: "a1/b1/c2/d1"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/", want: true},
{rules: []rule{{pattern: "a*/b?/c.txt"}}, key: "a1/b1", want: true},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/c.txt"},
{rules: []rule{{pattern: "a*/"}}, key: "a1/b1"},
{rules: []rule{{pattern: "a*/b*/"}}, key: "a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "/a*/b*"}}, key: "/a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "a*/b*/c"}}, key: "a1/b1/c1/d.txt/", want: true},
{rules: []rule{{pattern: "a"}}, key: "a/b/c/d/"},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg"}}, key: "a/pkg/c/a.go"},
{rules: []rule{{pattern: "a"}, {pattern: "pkg", include: true}}, key: "a/pkg/c/a.go"},
{rules: []rule{{pattern: "a.go", include: true}, {pattern: "pkg"}}, key: "", want: true},
{rules: []rule{{pattern: "a", include: true}, {pattern: "b/"}, {pattern: "c", include: true}}, key: "a/b/c"},
{rules: []rule{{pattern: "a/", include: true}, {pattern: "a"}}, key: "a/b", want: true},
{rules: []rule{{pattern: "/***"}}, key: "a"},
{rules: []rule{{pattern: "/***"}}, key: "a/b"},
{rules: []rule{{pattern: "/a/***"}}, key: "a/"},
{rules: []rule{{pattern: "/a/***"}}, key: "a/b"},
{rules: []rule{{pattern: "/a/***"}}, key: "a/b/c"},
{rules: []rule{{pattern: "/a/***"}}, key: "b/a/", want: true},
{rules: []rule{{pattern: "a/***"}}, key: "a/"},
{rules: []rule{{pattern: "a/***"}}, key: "a/b"},
{rules: []rule{{pattern: "a/***"}}, key: "a/b/c"},
{rules: []rule{{pattern: "a/***"}}, key: "d/a/b/c"},
{rules: []rule{{pattern: "a/***"}}, key: "a", want: true},
{rules: []rule{{pattern: "a/***"}}, key: "ba", want: true},
{rules: []rule{{pattern: "a/***"}}, key: "ba/", want: true},
{rules: []rule{{pattern: "*/a/***"}}, key: "/a/"},
{rules: []rule{{pattern: "*/a/***"}}, key: "b/a/"},
{rules: []rule{{pattern: "*/a/***"}}, key: "b/a/c"},
{rules: []rule{{pattern: "/*/a/***"}}, key: "/b/a/"},
{rules: []rule{{pattern: "/*/a/***"}}, key: "/b/a/c"},
{rules: []rule{{pattern: "/*/a/***"}}, key: "c/b/a/", want: true},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/b"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/b"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/e/b"},
{rules: []rule{{pattern: "/**/b"}}, key: "a/c/b"},
{rules: []rule{{pattern: "/**/b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/ab/", want: true},
{rules: []rule{{pattern: "a**b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a**b"}}, key: "b/c/d/b/", want: true},
}
for _, c := range tests {
if got := matchKey(c.rules, c.key); got != c.want {
t.Errorf("matchKey(%+v, %s) = %v, want %v", c.rules, c.key, got, c.want)
}
}
}

func TestParseFilterRule(t *testing.T) {
type tcase struct {
args []string
rules []rule
}
cases := []tcase{
{[]string{"--include", "a"}, []rule{{pattern: "a", include: true}}},
{[]string{"--exclude", "a", "--include", "b"}, []rule{{pattern: "a"}, {pattern: "b", include: true}}},
{[]string{"--include", "a", "--test", "t", "--exclude", "b"}, []rule{{pattern: "a", include: true}, {pattern: "b"}}},
{[]string{"--include=a", "--test", "t", "--exclude"}, []rule{{pattern: "a", include: true}}},
{[]string{"--include", "a", "--test", "t", "--exclude"}, []rule{{pattern: "a", include: true}}},
{[]string{"-include=", "a", "--test", "t", "--exclude=*"}, []rule{{pattern: "*"}}},
}

for _, c := range cases {
if got := parseIncludeRules(c.args); !reflect.DeepEqual(got, c.rules) {
t.Errorf("parseIncludeRules(%+v) = %v, want %v", c.args, got, c.rules)
}
}
}
Loading