From b1caa0025ab0717d0d75c395dbc25825b739d48c Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Sat, 18 Feb 2017 04:16:04 +1100 Subject: [PATCH 1/3] clean: switch -test to a regex -whitelist flag -test was _far_ too single-purpose and also unweildy in certain cases (it ignored "testdata" but not "fixtures" or similar). Instead of special casing everything, just add a whitelist flag that people can use for their own project requirements. Signed-off-by: Aleksa Sarai --- clean.go | 16 ++++++++-------- main.go | 45 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/clean.go b/clean.go index 8984a66..8de9483 100644 --- a/clean.go +++ b/clean.go @@ -72,14 +72,14 @@ func cleanVendor(vendorDir string, realDeps []*build.Package) error { return nil } - // When preserving tests don't delete testdata, let alone _test.go files. - if preserveTest { - if i.Name() == "testdata" { - return nil - } - if strings.HasSuffix(path, "_test.go") { - return nil - } + // Make sure we don't delete anything that matches the whitelist. The + // whitelist is relative to the vendor directory. + relPath, err := filepath.Rel(vendorDir, path) + if err != nil { + return err + } + if cleanWhitelist.matchString(relPath) { + return nil } if strings.HasPrefix(i.Name(), ".") || strings.HasPrefix(i.Name(), "_") { diff --git a/main.go b/main.go index 0be6f39..1545d45 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "log" "os" "path/filepath" + "regexp" "sort" "strings" "time" @@ -22,10 +23,41 @@ const ( ) var ( - verbose bool - preserveTest bool + verbose bool + cleanWhitelist regexpSlice ) +type regexpSlice []*regexp.Regexp + +var _ flag.Value = new(regexpSlice) + +func (rs *regexpSlice) Set(exp string) error { + regex, err := regexp.Compile(exp) + if err != nil { + return err + } + + *rs = append(*rs, regex) + return nil +} + +func (rs *regexpSlice) String() string { + exps := []string{} + for _, regex := range *rs { + exps = append(exps, fmt.Sprintf("%q", regex.String())) + } + return fmt.Sprintf("%v", exps) +} + +func (rs *regexpSlice) matchString(str string) bool { + for _, regex := range *rs { + if regex.MatchString(str) { + return true + } + } + return false +} + func init() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) @@ -33,7 +65,7 @@ func init() { flag.PrintDefaults() } flag.BoolVar(&verbose, "verbose", false, "shows all warnings") - flag.BoolVar(&preserveTest, "test", false, "preserve _test.go files in vendor'd projects") + flag.Var(&cleanWhitelist, "whitelist", "regular expressions to whitelist for cleaning phase of vendoring, relative to the vendor/ directory") } func validateArgs() { @@ -137,22 +169,22 @@ func main() { log.Fatal("There must not be vendor dir and vendor.conf file for initialization") } } + wd, err := os.Getwd() if err != nil { log.Fatalf("Error getting working directory: %v", err) } - wd, err = filepath.EvalSymlinks(wd) if err != nil { log.Fatalf("Error getting working directory after evalsymlinks: %v", err) } + vd := filepath.Join(wd, vendorDir) log.Println("Collecting initial packages") initPkgs, err := collectPkgs(wd) if err != nil { log.Fatalf("Error collecting initial packages: %v", err) } - vd := filepath.Join(wd, vendorDir) // variables for init var dlFunc func(string) (*build.Package, error) var deps []depEntry @@ -194,6 +226,9 @@ func main() { log.Fatalf("Error on collecting all dependencies: %v", err) } log.Println("Clean vendor dir from unused packages") + for _, regex := range cleanWhitelist { + log.Printf("\tIgnoring paths matching %q", regex.String()) + } if err := cleanVendor(vd, pkgs); err != nil { log.Fatal(err) } From 4c31acf0e1f6c3119fe6a26875e7551575bdb24a Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Sat, 18 Feb 2017 04:27:48 +1100 Subject: [PATCH 2/3] test: add tests for 'vndr -whitelist' Signed-off-by: Aleksa Sarai --- test/vndr_test.go | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/test/vndr_test.go b/test/vndr_test.go index 409a52c..5dec64b 100644 --- a/test/vndr_test.go +++ b/test/vndr_test.go @@ -2,6 +2,7 @@ package vndrtest import ( "bytes" + "fmt" "io/ioutil" "os" "os/exec" @@ -186,3 +187,87 @@ github.com/docker/swarmkit branch t.Fatalf("libcompose should not be reported: %s", out) } } + +func TestCleanWhitelist(t *testing.T) { + vndrBin, err := exec.LookPath("vndr") + if err != nil { + t.Fatal(err) + } + tmp, err := ioutil.TempDir("", "test-vndr-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + repoDir := filepath.Join(tmp, "src", testRepo) + if err := os.MkdirAll(repoDir, 0700); err != nil { + t.Fatal(err) + } + content := []byte(`github.com/containers/image master +github.com/projectatomic/skopeo master`) + vendorConf := filepath.Join(repoDir, "vendor.conf") + if err := ioutil.WriteFile(vendorConf, content, 0666); err != nil { + t.Fatal(err) + } + vndrCmd := exec.Command(vndrBin, + "-whitelist", `github\.com/containers/image/MAINTAINERS`, + "-whitelist", `github\.com/projectatomic/skopeo/integration/.*`) + vndrCmd.Dir = repoDir + setGopath(vndrCmd, tmp) + + out, err := vndrCmd.CombinedOutput() + if err != nil { + t.Logf("output: %v", string(out)) + t.Fatalf("error was not expected: %v", err) + } + + if !bytes.Contains(out, []byte(fmt.Sprintf(`Ignoring paths matching %q`, `github\.com/containers/image/MAINTAINERS`))) { + t.Logf("output: %v", string(out)) + t.Errorf(`output missing regular expression "github\.com/containers/image/MAINTAINERS"`) + } + if !bytes.Contains(out, []byte(fmt.Sprintf(`Ignoring paths matching %q`, `github\.com/projectatomic/skopeo/integration/.*`))) { + t.Logf("output: %v", string(out)) + t.Errorf(`output missing regular expression "github\.com/projectatomic/skopeo/integration/.*"`) + } + + // Make sure that the files were not "cleaned". + for _, path := range []string{ + "github.com/containers/image/MAINTAINERS", + "github.com/projectatomic/skopeo/integration", + } { + path = filepath.Join(repoDir, "vendor", path) + if _, err := os.Lstat(path); err != nil { + t.Errorf("%s was cleaned but shouldn't have been", path) + } + } + + // Run again to make sure the above will be cleaned. + vndrCmd = exec.Command(vndrBin) + vndrCmd.Dir = repoDir + setGopath(vndrCmd, tmp) + + out, err = vndrCmd.CombinedOutput() + if err != nil { + t.Logf("output: %v", string(out)) + t.Fatalf("[no -whitelist] error was not expected: %v", err) + } + + if bytes.Contains(out, []byte(fmt.Sprintf(`Ignoring paths matching %q`, `github\.com/containers/image/MAINTAINERS`))) { + t.Logf("output: %v", string(out)) + t.Errorf(`[no -whitelist] output missing regular expression "github\.com/containers/image/MAINTAINERS"`) + } + if bytes.Contains(out, []byte(fmt.Sprintf(`Ignoring paths matching %q`, `github\.com/projectatomic/skopeo/integration/.*`))) { + t.Logf("output: %v", string(out)) + t.Errorf(`[no -whitelist] output missing regular expression "github\.com/projectatomic/skopeo/integration/.*"`) + } + + // Make sure that the files were "cleaned". + for _, path := range []string{ + "github.com/containers/image/MAINTAINERS", + "github.com/projectatomic/skopeo/integration", + } { + path = filepath.Join(repoDir, "vendor", path) + if _, err := os.Lstat(path); err == nil { + t.Errorf("[no -whitelist] %s was NOT cleaned but should have been", path) + } + } +} From 9af37d14897b9701f9a20c6a4b736cb20ad9e979 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Sat, 18 Feb 2017 08:02:08 +1100 Subject: [PATCH 3/3] README: add some more information about -whitelist Signed-off-by: Aleksa Sarai --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0808e13..067a393 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,15 @@ [![Build Status](https://travis-ci.org/LK4D4/vndr.svg?branch=master)](https://travis-ci.org/LK4D4/vndr) Vndr is simple vendoring tool, which is inspired by Docker vendor script. -Vndr has only one option: `-verbose`. +Vndr has only two options: `-verbose` and `-whitelist`, both of which do +exactly what they say on the tin. + +* `-verbose` adds additional output, helpful for debugging issues. +* `-whitelist` allows you to specify several regular expressions for paths + which will *not* be cleaned in the final stage of vendoring -- this is useful + for running tests in a vendored project or otherwise ensuring that some + important files are retained after `vndr` is done cleaning unused files from + your `vendor/` directory. ## vendor.conf @@ -20,7 +28,7 @@ This config format is also accepted by [trash](https://github.com/rancher/trash) ## Initialization You can initiate your project with vendor directory and `vendor.conf` using command -`vndr init`. This will populate your vendor directory with latest versions of +`vndr init`. This will populate your vendor directory with latest versions of all dependecies and also write `vendor.conf` config which you can use for changing versions later.