From 13f2130e5b7967c34d9c94ce7411a89081d60a3c Mon Sep 17 00:00:00 2001 From: Dmitri Goutnik Date: Tue, 7 Apr 2020 11:15:32 -0500 Subject: [PATCH] make tags lookup off by default --- apis/apis.go | 6 ---- apis/github.go | 10 +++++-- flags/flags.go | 60 ++++++++++++++++++++++++++++++++++++++ main.go | 50 ++++--------------------------- tuple/parser.go | 7 +++-- tuple/tuple_online_test.go | 2 +- tuple/tuple_test.go | 4 +-- 7 files changed, 79 insertions(+), 60 deletions(-) create mode 100644 flags/flags.go diff --git a/apis/apis.go b/apis/apis.go index 6d4629b..a49dd14 100644 --- a/apis/apis.go +++ b/apis/apis.go @@ -8,12 +8,6 @@ import ( "strings" ) -const ( - OfflineKey = "M2T_OFFLINE" - GithubCredentialsKey = "M2T_GITHUB" - // GitlabsCredentialsKey = "M2T_GITLAB" -) - func get(url string, credsKey string) ([]byte, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/apis/github.go b/apis/github.go index a7010f0..4cf519d 100644 --- a/apis/github.go +++ b/apis/github.go @@ -6,6 +6,8 @@ import ( "fmt" "net/url" "strings" + + "github.com/dmgk/modules2tuple/flags" ) type GithubCommit struct { @@ -21,13 +23,15 @@ var githubRateLimitError = fmt.Sprintf(`Github API rate limit exceeded. Please e to let modules2tuple call Github API using basic authentication. To create a new token, navigate to https://github.com/settings/tokens/new (leave all checkboxes unchecked, modules2tuple doesn't need any access to your account) -- set %s=1 or pass "-offline" flag to module2tuple to disable network access`, OfflineKey, GithubCredentialsKey) +- set %s=0 and/or remove "-ghtags" flag to turn off Github tags lookup +- set %s=1 or pass "-offline" flag to module2tuple to disable network access`, + flags.GithubCredentialsKey, flags.LookupGithubTagsKey, flags.OfflineKey) func GetGithubCommit(account, project, tag string) (string, error) { projectID := fmt.Sprintf("%s/%s", url.PathEscape(account), url.PathEscape(project)) url := fmt.Sprintf("https://api.github.com/repos/%s/commits/%s", projectID, tag) - resp, err := get(url, GithubCredentialsKey) + resp, err := get(url, flags.GithubCredentialsKey) if err != nil { if strings.Contains(err.Error(), "API rate limit exceeded") { return "", errors.New(githubRateLimitError) @@ -47,7 +51,7 @@ func LookupGithubTag(account, project, tag string) (string, error) { projectID := fmt.Sprintf("%s/%s", url.PathEscape(account), url.PathEscape(project)) url := fmt.Sprintf("https://api.github.com/repos/%s/git/refs/tags", projectID) - resp, err := get(url, GithubCredentialsKey) + resp, err := get(url, flags.GithubCredentialsKey) if err != nil { if strings.Contains(err.Error(), "API rate limit exceeded") { return "", errors.New(githubRateLimitError) diff --git a/flags/flags.go b/flags/flags.go new file mode 100644 index 0000000..c2571d0 --- /dev/null +++ b/flags/flags.go @@ -0,0 +1,60 @@ +package flags + +import ( + "flag" + "fmt" + "html/template" + "os" + "path" +) + +const ( + GithubCredentialsKey = "M2T_GITHUB" + // GitlabsCredentialsKey = "M2T_GITLAB" + LookupGithubTagsKey = "M2T_GHTAGS" + OfflineKey = "M2T_OFFLINE" + PrefixKey = "M2T_PREFIX" +) + +var ( + LookupGithubTags = os.Getenv(LookupGithubTagsKey) == "1" + Offline = os.Getenv(OfflineKey) == "1" + PackagePrefix = os.Getenv(PrefixKey) + ShowVersion = false +) + +var helpTemplate = template.Must(template.New("help").Parse(` +Vendor package dependencies and then run {{.Name}} on vendor/modules.txt: + + $ go mod vendor + $ {{.Name}} vendor/modules.txt + +By default, generated GH_TUPLE entries will place packages under "vendor". +This can be changed by passing different prefix using -prefix option (e.g. +-prefix src). + +When generating GL_TUPLE entries, modules2tuple will attempt to use Gitlab +API to resolve short commit IDs and tags to the full 40-character IDs as +required by bsd.sites.mk. If network access is not available or not wanted, +this commit ID translation can be disabled with -offline flag. +`)) + +func init() { + basename := path.Base(os.Args[0]) + if PackagePrefix == "" { + PackagePrefix = "vendor" + } + + flag.BoolVar(&LookupGithubTags, "ghtags", LookupGithubTags, fmt.Sprintf("lookup tags with Github API (env %s)", LookupGithubTagsKey)) + flag.BoolVar(&Offline, "offline", Offline, fmt.Sprintf("disable all network access (env %s)", OfflineKey)) + flag.StringVar(&PackagePrefix, "prefix", PackagePrefix, fmt.Sprintf("package prefix (env %s)", PrefixKey)) + flag.BoolVar(&ShowVersion, "v", false, "show version") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [options] modules.txt\n", basename) + flag.PrintDefaults() + helpTemplate.Execute(os.Stderr, map[string]string{ + "Name": basename, + }) + } +} diff --git a/main.go b/main.go index 27a94f7..c17afbd 100644 --- a/main.go +++ b/main.go @@ -4,17 +4,17 @@ import ( "flag" "fmt" "os" - "path" - "text/template" - "github.com/dmgk/modules2tuple/apis" + "github.com/dmgk/modules2tuple/flags" "github.com/dmgk/modules2tuple/tuple" ) +var version = "devel" + func main() { flag.Parse() - if flagVersion { + if flags.ShowVersion { fmt.Fprintln(os.Stderr, version) os.Exit(0) } @@ -27,7 +27,7 @@ func main() { } var haveTuples bool - parser := tuple.NewParser(flagPackagePrefix, flagOffline) + parser := tuple.NewParser(flags.PackagePrefix, flags.Offline, flags.LookupGithubTags) tuples, errors := parser.Load(args[0]) if len(tuples) != 0 { fmt.Print(tuples) @@ -41,43 +41,3 @@ func main() { fmt.Println() } } - -var helpTemplate = template.Must(template.New("help").Parse(` -Vendor package dependencies and then run {{.Name}} on vendor/modules.txt: - - $ go mod vendor - $ {{.Name}} vendor/modules.txt - -By default, generated GH_TUPLE entries will place packages under "vendor". -This can be changed by passing different prefix using -prefix option (e.g. --prefix src). - -When generating GL_TUPLE entries, modules2tuple will attempt to use Gitlab -API to resolve short commit IDs and tags to the full 40-character IDs as -required by bsd.sites.mk. If network access is not available or not wanted, -this commit ID translation can be disabled with -offline flag. -`)) - -var ( - flagOffline = os.Getenv(apis.OfflineKey) != "" - flagPackagePrefix = "vendor" - flagVersion = false -) - -var version = "devel" - -func init() { - basename := path.Base(os.Args[0]) - - flag.BoolVar(&flagOffline, "offline", flagOffline, "disable network access") - flag.StringVar(&flagPackagePrefix, "prefix", "vendor", "package prefix") - flag.BoolVar(&flagVersion, "v", flagVersion, "show version") - - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [options] modules.txt\n", basename) - flag.PrintDefaults() - helpTemplate.Execute(os.Stderr, map[string]string{ - "Name": basename, - }) - } -} diff --git a/tuple/parser.go b/tuple/parser.go index 96bbb38..b246154 100644 --- a/tuple/parser.go +++ b/tuple/parser.go @@ -15,11 +15,12 @@ import ( type Parser struct { packagePrefix string offline bool + lookupTags bool } // NewParser creates a new modules.txt parser with given options. -func NewParser(packagePrefix string, offline bool) *Parser { - return &Parser{packagePrefix, offline} +func NewParser(packagePrefix string, offline bool, lookupTags bool) *Parser { + return &Parser{packagePrefix, offline, lookupTags} } // Read parses tuples from modules.txt contents provided as io.Reader. @@ -52,7 +53,7 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) { if !p.offline { switch t.Source.(type) { case GH: - if strings.HasPrefix(t.Tag, "v") { + if p.lookupTags && strings.HasPrefix(t.Tag, "v") { // Call Gihub API to check tags. Go seem to be able to magically // translate tags like "v1.0.4" to the "api/v1.0.4" which is really used // by upstream. We'll try to do the same. diff --git a/tuple/tuple_online_test.go b/tuple/tuple_online_test.go index 9bec6c9..369b1be 100644 --- a/tuple/tuple_online_test.go +++ b/tuple/tuple_online_test.go @@ -16,7 +16,7 @@ func TestUniqueProjectAndTag(t *testing.T) { ugorji:go:23ab95ef5dc3:ugorji_go/vendor/github.com/ugorji/go ` - tt, err := NewParser("vendor", false).Read(strings.NewReader(given)) + tt, err := NewParser("vendor", false, true).Read(strings.NewReader(given)) if err != nil { t.Fatal(err) } diff --git a/tuple/tuple_test.go b/tuple/tuple_test.go index 15f4292..13efaf2 100644 --- a/tuple/tuple_test.go +++ b/tuple/tuple_test.go @@ -193,7 +193,7 @@ GL_TUPLE= gitlab-org:gitaly-proto:v1.32.0:gitlab_org_gitaly_proto/vendor/gitlab. # ::v1.2.3:group_name/vendor/some_unknown.vanity_url.net/account/project ` - tt, err := NewParser("vendor", true).Read(strings.NewReader(given)) + tt, err := NewParser("vendor", true, false).Read(strings.NewReader(given)) if err == nil { t.Fatal("expected err to not be nil") } @@ -219,7 +219,7 @@ func TestUniqueGroups(t *testing.T) { minio:parquet-go:9d767baf1679:minio_parquet_go/vendor/github.com/minio/parquet-go ` - tt, err := NewParser("vendor", true).Read(strings.NewReader(given)) + tt, err := NewParser("vendor", true, false).Read(strings.NewReader(given)) if err != nil { t.Fatal(err) }