From 58de7c6d4838729c6c133d9b2461dc6b1f766b76 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 5 Apr 2019 09:26:24 -0400 Subject: [PATCH] cmd/go/internal/web: merge internal/web2 into web MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cmd/go/internal/web package was forked in order to support direct HTTPS fetches from widely-used hosting providers,¹ but direct fetches were subsequently dropped in CL 107657. The forked web2 package, with its GitHub-specific diagnostics and .netrc support, remained in use for module proxy support, but was not used for the initial '?go-get=1' path resolution, so the .netrc file was only used to fetch from already-resolved module protocol servers. This CL moves the .netrc support into its own (new) package, cmd/go/internal/auth, and consolidates the web and web2 packages back into just web. As a result, fetches via the web package now support .netrc, and fetches that previously used web2 now enforce the same security policies as web (such as prohibiting HTTPS-to-HTTP redirects). ¹https://github.com/golang/vgo/commit/63138cb6ceed7d6d4e51a8cbd568c64bd3e2b132 Fixes #29591 Fixes #29888 Fixes #30610 Updates #26232 Change-Id: Ia3a13526e443679cf14a72a1f3db96f336ce5e73 Reviewed-on: https://go-review.googlesource.com/c/go/+/170879 Run-TryBot: Russ Cox Reviewed-by: Russ Cox Reviewed-by: Jay Conrod --- src/cmd/go/internal/auth/auth.go | 23 ++ src/cmd/go/internal/auth/netrc.go | 111 ++++++ .../{web2/web_test.go => auth/netrc_test.go} | 4 +- src/cmd/go/internal/bug/bug.go | 10 +- src/cmd/go/internal/get/get.go | 2 +- src/cmd/go/internal/get/vcs.go | 102 ++++-- src/cmd/go/internal/get/vcs_test.go | 2 +- src/cmd/go/internal/modfetch/noweb.go | 24 -- src/cmd/go/internal/modfetch/proxy.go | 94 +++-- src/cmd/go/internal/modfetch/repo.go | 4 +- src/cmd/go/internal/modfetch/web.go | 31 -- src/cmd/go/internal/web/api.go | 102 ++++++ src/cmd/go/internal/web/bootstrap.go | 26 +- src/cmd/go/internal/web/http.go | 136 +++---- src/cmd/go/internal/web/security.go | 16 - src/cmd/go/internal/web2/web.go | 345 ------------------ src/cmd/go/internal/webtest/test.go | 314 ---------------- src/cmd/go/testdata/script/get_404_meta.txt | 10 + .../testdata/script/get_insecure_redirect.txt | 1 - src/cmd/go/testdata/script/mod_auth.txt | 31 ++ 20 files changed, 509 insertions(+), 879 deletions(-) create mode 100644 src/cmd/go/internal/auth/auth.go create mode 100644 src/cmd/go/internal/auth/netrc.go rename src/cmd/go/internal/{web2/web_test.go => auth/netrc_test.go} (95%) delete mode 100644 src/cmd/go/internal/modfetch/noweb.go delete mode 100644 src/cmd/go/internal/modfetch/web.go create mode 100644 src/cmd/go/internal/web/api.go delete mode 100644 src/cmd/go/internal/web/security.go delete mode 100644 src/cmd/go/internal/web2/web.go delete mode 100644 src/cmd/go/internal/webtest/test.go create mode 100644 src/cmd/go/testdata/script/get_404_meta.txt create mode 100644 src/cmd/go/testdata/script/mod_auth.txt diff --git a/src/cmd/go/internal/auth/auth.go b/src/cmd/go/internal/auth/auth.go new file mode 100644 index 00000000000000..12e3c74dcc7549 --- /dev/null +++ b/src/cmd/go/internal/auth/auth.go @@ -0,0 +1,23 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package auth provides access to user-provided authentication credentials. +package auth + +import "net/http" + +// AddCredentials fills in the user's credentials for req, if any. +// The return value reports whether any matching credentials were found. +func AddCredentials(req *http.Request) (added bool) { + // TODO(golang.org/issue/26232): Support arbitrary user-provided credentials. + netrcOnce.Do(readNetrc) + for _, l := range netrc { + if l.machine == req.URL.Host { + req.SetBasicAuth(l.login, l.password) + return true + } + } + + return false +} diff --git a/src/cmd/go/internal/auth/netrc.go b/src/cmd/go/internal/auth/netrc.go new file mode 100644 index 00000000000000..7a9bdbb72c8ac4 --- /dev/null +++ b/src/cmd/go/internal/auth/netrc.go @@ -0,0 +1,111 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" +) + +type netrcLine struct { + machine string + login string + password string +} + +var ( + netrcOnce sync.Once + netrc []netrcLine + netrcErr error +) + +func parseNetrc(data string) []netrcLine { + // See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html + // for documentation on the .netrc format. + var nrc []netrcLine + var l netrcLine + inMacro := false + for _, line := range strings.Split(data, "\n") { + if inMacro { + if line == "" { + inMacro = false + } + continue + } + + f := strings.Fields(line) + i := 0 + for ; i < len(f)-1; i += 2 { + // Reset at each "machine" token. + // “The auto-login process searches the .netrc file for a machine token + // that matches […]. Once a match is made, the subsequent .netrc tokens + // are processed, stopping when the end of file is reached or another + // machine or a default token is encountered.” + switch f[i] { + case "machine": + l = netrcLine{machine: f[i+1]} + case "default": + break + case "login": + l.login = f[i+1] + case "password": + l.password = f[i+1] + case "macdef": + // “A macro is defined with the specified name; its contents begin with + // the next .netrc line and continue until a null line (consecutive + // new-line characters) is encountered.” + inMacro = true + } + if l.machine != "" && l.login != "" && l.password != "" { + nrc = append(nrc, l) + l = netrcLine{} + } + } + + if i < len(f) && f[i] == "default" { + // “There can be only one default token, and it must be after all machine tokens.” + break + } + } + + return nrc +} + +func netrcPath() (string, error) { + if env := os.Getenv("NETRC"); env != "" { + return env, nil + } + dir, err := os.UserHomeDir() + if err != nil { + return "", err + } + base := ".netrc" + if runtime.GOOS == "windows" { + base = "_netrc" + } + return filepath.Join(dir, base), nil +} + +func readNetrc() { + path, err := netrcPath() + if err != nil { + netrcErr = err + return + } + + data, err := ioutil.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + netrcErr = err + } + return + } + + netrc = parseNetrc(string(data)) +} diff --git a/src/cmd/go/internal/web2/web_test.go b/src/cmd/go/internal/auth/netrc_test.go similarity index 95% rename from src/cmd/go/internal/web2/web_test.go rename to src/cmd/go/internal/auth/netrc_test.go index e6787a5b54e42e..e06c545390e0e9 100644 --- a/src/cmd/go/internal/web2/web_test.go +++ b/src/cmd/go/internal/auth/netrc_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package web2 +package auth import ( "reflect" @@ -43,7 +43,7 @@ login oops password too-late-in-file ` -func TestReadNetrc(t *testing.T) { +func TestParseNetrc(t *testing.T) { lines := parseNetrc(testNetrc) want := []netrcLine{ {"api.github.com", "user", "pwd"}, diff --git a/src/cmd/go/internal/bug/bug.go b/src/cmd/go/internal/bug/bug.go index e701f6eac9c943..468605c74ace9f 100644 --- a/src/cmd/go/internal/bug/bug.go +++ b/src/cmd/go/internal/bug/bug.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "io/ioutil" + urlpkg "net/url" "os" "os/exec" "path/filepath" @@ -62,7 +63,7 @@ func runBug(cmd *base.Command, args []string) { fmt.Fprintln(&buf, "```") body := buf.String() - url := "https://github.com/golang/go/issues/new?body=" + web.QueryEscape(body) + url := "https://github.com/golang/go/issues/new?body=" + urlpkg.QueryEscape(body) if !web.OpenBrowser(url) { fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n") fmt.Print(body) @@ -130,7 +131,12 @@ func printCDetails(w io.Writer) { } func inspectGoVersion(w io.Writer) { - data, err := web.Get("https://golang.org/VERSION?m=text") + data, err := web.GetBytes(&urlpkg.URL{ + Scheme: "https", + Host: "golang.org", + Path: "/VERSION", + RawQuery: "?m=text", + }) if err != nil { if cfg.BuildV { fmt.Printf("failed to read from golang.org/VERSION: %v\n", err) diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index fe15515efca8a6..c70013c3f5cf02 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -392,7 +392,7 @@ func downloadPackage(p *load.Package) error { blindRepo bool // set if the repo has unusual configuration ) - security := web.Secure + security := web.SecureOnly if Insecure { security = web.Insecure } diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/get/vcs.go index 6f60bc0631585b..bb1845e315e9c7 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/get/vcs.go @@ -11,7 +11,7 @@ import ( "internal/lazyregexp" "internal/singleflight" "log" - "net/url" + urlpkg "net/url" "os" "os/exec" "path/filepath" @@ -54,7 +54,7 @@ var defaultSecureScheme = map[string]bool{ } func (v *vcsCmd) isSecure(repo string) bool { - u, err := url.Parse(repo) + u, err := urlpkg.Parse(repo) if err != nil { // If repo is not a URL, it's not secure. return false @@ -188,19 +188,19 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error } out := strings.TrimSpace(string(outb)) - var repoURL *url.URL + var repoURL *urlpkg.URL if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { // Match SCP-like syntax and convert it to a URL. // Eg, "git@github.com:user/repo" becomes // "ssh://git@github.com/user/repo". - repoURL = &url.URL{ + repoURL = &urlpkg.URL{ Scheme: "ssh", - User: url.User(m[1]), + User: urlpkg.User(m[1]), Host: m[2], Path: m[3], } } else { - repoURL, err = url.Parse(out) + repoURL, err = urlpkg.Parse(out) if err != nil { return "", err } @@ -730,7 +730,7 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, match["repo"] = scheme + "://" + match["repo"] } else { for _, scheme := range vcs.scheme { - if security == web.Secure && !vcs.isSecureScheme(scheme) { + if security == web.SecureOnly && !vcs.isSecureScheme(scheme) { continue } if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil { @@ -754,20 +754,35 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, return nil, errUnknownSite } -// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not -// statically known by repoRootForImportPathStatic. +// urlForImportPath returns a partially-populated URL for the given Go import path. // -// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". -func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { +// The URL leaves the Scheme field blank so that web.Get will try any scheme +// allowed by the selected security mode. +func urlForImportPath(importPath string) (*urlpkg.URL, error) { slash := strings.Index(importPath, "/") if slash < 0 { slash = len(importPath) } - host := importPath[:slash] + host, path := importPath[:slash], importPath[slash:] if !strings.Contains(host, ".") { return nil, errors.New("import path does not begin with hostname") } - urlStr, body, err := web.GetMaybeInsecure(importPath, security) + if len(path) == 0 { + path = "/" + } + return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil +} + +// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not +// statically known by repoRootForImportPathStatic. +// +// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". +func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { + url, err := urlForImportPath(importPath) + if err != nil { + return nil, err + } + url, resp, err := web.Get(security, url) if err != nil { msg := "https fetch: %v" if security == web.Insecure { @@ -775,6 +790,7 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se } return nil, fmt.Errorf(msg, err) } + body := resp.Body defer body.Close() imports, err := parseMetaGoImports(body, mod) if err != nil { @@ -784,12 +800,12 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se mmi, err := matchGoImport(imports, importPath) if err != nil { if _, ok := err.(ImportMismatchError); !ok { - return nil, fmt.Errorf("parse %s: %v", urlStr, err) + return nil, fmt.Errorf("parse %s: %v", url, err) } - return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err) + return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", url, err) } if cfg.BuildV { - log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) + log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url) } // If the import was "uni.edu/bob/project", which said the // prefix was "uni.edu" and the RepoRoot was "evilroot.com", @@ -801,24 +817,24 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se if cfg.BuildV { log.Printf("get %q: verifying non-authoritative meta tag", importPath) } - urlStr0 := urlStr + url0 := *url var imports []metaImport - urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) + url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) if err != nil { return nil, err } metaImport2, err := matchGoImport(imports, importPath) if err != nil || mmi != metaImport2 { - return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) + return nil, fmt.Errorf("%s and %s disagree about go-import for %s", &url0, url, mmi.Prefix) } } if err := validateRepoRoot(mmi.RepoRoot); err != nil { - return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err) + return nil, fmt.Errorf("%s: invalid repo root %q: %v", url, mmi.RepoRoot, err) } vcs := vcsByCmd(mmi.VCS) if vcs == nil && mmi.VCS != "mod" { - return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) + return nil, fmt.Errorf("%s: unknown vcs %q", url, mmi.VCS) } rr := &RepoRoot{ @@ -834,7 +850,7 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se // validateRepoRoot returns an error if repoRoot does not seem to be // a valid URL with scheme. func validateRepoRoot(repoRoot string) error { - url, err := url.Parse(repoRoot) + url, err := urlpkg.Parse(repoRoot) if err != nil { return err } @@ -856,9 +872,9 @@ var ( // // The importPath is of the form "golang.org/x/tools". // It is an error if no imports are found. -// urlStr will still be valid if err != nil. -// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" -func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) { +// url will still be valid if err != nil. +// The returned url will be of the form "https://golang.org/x/tools?go-get=1" +func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) { setCache := func(res fetchResult) (fetchResult, error) { fetchCacheMu.Lock() defer fetchCacheMu.Unlock() @@ -874,25 +890,31 @@ func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.Secu } fetchCacheMu.Unlock() - urlStr, body, err := web.GetMaybeInsecure(importPrefix, security) + url, err := urlForImportPath(importPrefix) if err != nil { - return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) + return setCache(fetchResult{err: err}) } + url, resp, err := web.Get(security, url) + if err != nil { + return setCache(fetchResult{url: url, err: fmt.Errorf("fetch %s: %v", url, err)}) + } + body := resp.Body + defer body.Close() imports, err := parseMetaGoImports(body, mod) if err != nil { - return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) + return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", url, err)}) } if len(imports) == 0 { - err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) + err = fmt.Errorf("fetch %s: no go-import meta tag", url) } - return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) + return setCache(fetchResult{url: url, imports: imports, err: err}) }) res := resi.(fetchResult) - return res.urlStr, res.imports, res.err + return res.url, res.imports, res.err } type fetchResult struct { - urlStr string // e.g. "https://foo.com/x/bar?go-get=1" + url *urlpkg.URL imports []metaImport err error } @@ -1074,8 +1096,13 @@ func bitbucketVCS(match map[string]string) error { var resp struct { SCM string `json:"scm"` } - url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm") - data, err := web.Get(url) + url := &urlpkg.URL{ + Scheme: "https", + Host: "api.bitbucket.org", + Path: expand(match, "/2.0/repositories/{bitname}"), + RawQuery: "fields=scm", + } + data, err := web.GetBytes(url) if err != nil { if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 { // this may be a private repository. If so, attempt to determine which @@ -1117,7 +1144,12 @@ func launchpadVCS(match map[string]string) error { if match["project"] == "" || match["series"] == "" { return nil } - _, err := web.Get(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) + url := &urlpkg.URL{ + Scheme: "https", + Host: "code.launchpad.net", + Path: expand(match, "/{project}{series}/.bzr/branch-format"), + } + _, err := web.GetBytes(url) if err != nil { match["root"] = expand(match, "launchpad.net/{project}") match["repo"] = expand(match, "https://{root}") diff --git a/src/cmd/go/internal/get/vcs_test.go b/src/cmd/go/internal/get/vcs_test.go index d13721bed1a489..91800baa83d216 100644 --- a/src/cmd/go/internal/get/vcs_test.go +++ b/src/cmd/go/internal/get/vcs_test.go @@ -181,7 +181,7 @@ func TestRepoRootForImportPath(t *testing.T) { } for _, test := range tests { - got, err := RepoRootForImportPath(test.path, IgnoreMod, web.Secure) + got, err := RepoRootForImportPath(test.path, IgnoreMod, web.SecureOnly) want := test.want if want == nil { diff --git a/src/cmd/go/internal/modfetch/noweb.go b/src/cmd/go/internal/modfetch/noweb.go deleted file mode 100644 index 9d713dcc669687..00000000000000 --- a/src/cmd/go/internal/modfetch/noweb.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build cmd_go_bootstrap - -package modfetch - -import ( - "fmt" - "io" -) - -func webGetGoGet(url string, body *io.ReadCloser) error { - return fmt.Errorf("no network in go_bootstrap") -} - -func webGetBytes(url string, body *[]byte) error { - return fmt.Errorf("no network in go_bootstrap") -} - -func webGetBody(url string, body *io.ReadCloser) error { - return fmt.Errorf("no network in go_bootstrap") -} diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index cbf476d1e49945..ec9caf1556c118 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -8,7 +8,11 @@ import ( "encoding/json" "fmt" "io" - "net/url" + "io/ioutil" + urlpkg "net/url" + "os" + pathpkg "path" + "path/filepath" "strings" "time" @@ -17,6 +21,7 @@ import ( "cmd/go/internal/modfetch/codehost" "cmd/go/internal/module" "cmd/go/internal/semver" + "cmd/go/internal/web" ) var HelpGoproxy = &base.Command{ @@ -99,34 +104,85 @@ func lookupProxy(path string) (Repo, error) { if strings.Contains(proxyURL, ",") { return nil, fmt.Errorf("invalid $GOPROXY setting: cannot have comma") } - u, err := url.Parse(proxyURL) - if err != nil || u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "file" { - // Don't echo $GOPROXY back in case it has user:password in it (sigh). - return nil, fmt.Errorf("invalid $GOPROXY setting: malformed URL or invalid scheme (must be http, https, file)") + r, err := newProxyRepo(proxyURL, path) + if err != nil { + return nil, err } - return newProxyRepo(u.String(), path) + return r, nil } type proxyRepo struct { - url string + url *urlpkg.URL path string } func newProxyRepo(baseURL, path string) (Repo, error) { + url, err := urlpkg.Parse(baseURL) + if err != nil { + return nil, err + } + switch url.Scheme { + case "file": + if *url != (urlpkg.URL{Scheme: url.Scheme, Path: url.Path, RawPath: url.RawPath}) { + return nil, fmt.Errorf("proxy URL %q uses file scheme with non-path elements", web.PasswordRedacted(url)) + } + case "http", "https": + case "": + return nil, fmt.Errorf("proxy URL %q missing scheme", web.PasswordRedacted(url)) + default: + return nil, fmt.Errorf("unsupported proxy scheme %q", url.Scheme) + } + enc, err := module.EncodePath(path) if err != nil { return nil, err } - return &proxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(enc), path}, nil + + url.Path = strings.TrimSuffix(url.Path, "/") + "/" + enc + url.RawPath = strings.TrimSuffix(url.RawPath, "/") + "/" + pathEscape(enc) + return &proxyRepo{url, path}, nil } func (p *proxyRepo) ModulePath() string { return p.path } +func (p *proxyRepo) getBytes(path string) ([]byte, error) { + body, err := p.getBody(path) + if err != nil { + return nil, err + } + defer body.Close() + return ioutil.ReadAll(body) +} + +func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { + fullPath := pathpkg.Join(p.url.Path, path) + if p.url.Scheme == "file" { + rawPath, err := urlpkg.PathUnescape(fullPath) + if err != nil { + return nil, err + } + return os.Open(filepath.FromSlash(rawPath)) + } + + url := new(urlpkg.URL) + *url = *p.url + url.Path = fullPath + url.RawPath = pathpkg.Join(url.RawPath, pathEscape(path)) + + _, resp, err := web.Get(web.DefaultSecurity, url) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected status (%s): %v", web.PasswordRedacted(url), resp.Status) + } + return resp.Body, nil +} + func (p *proxyRepo) Versions(prefix string) ([]string, error) { - var data []byte - err := webGetBytes(p.url+"/@v/list", &data) + data, err := p.getBytes("@v/list") if err != nil { return nil, err } @@ -142,8 +198,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) { } func (p *proxyRepo) latest() (*RevInfo, error) { - var data []byte - err := webGetBytes(p.url+"/@v/list", &data) + data, err := p.getBytes("@v/list") if err != nil { return nil, err } @@ -172,12 +227,11 @@ func (p *proxyRepo) latest() (*RevInfo, error) { } func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { - var data []byte encRev, err := module.EncodeVersion(rev) if err != nil { return nil, err } - err = webGetBytes(p.url+"/@v/"+pathEscape(encRev)+".info", &data) + data, err := p.getBytes("@v/" + encRev + ".info") if err != nil { return nil, err } @@ -189,9 +243,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { } func (p *proxyRepo) Latest() (*RevInfo, error) { - var data []byte - u := p.url + "/@latest" - err := webGetBytes(u, &data) + data, err := p.getBytes("@latest") if err != nil { // TODO return err if not 404 return p.latest() @@ -204,12 +256,11 @@ func (p *proxyRepo) Latest() (*RevInfo, error) { } func (p *proxyRepo) GoMod(version string) ([]byte, error) { - var data []byte encVer, err := module.EncodeVersion(version) if err != nil { return nil, err } - err = webGetBytes(p.url+"/@v/"+pathEscape(encVer)+".mod", &data) + data, err := p.getBytes("@v/" + encVer + ".mod") if err != nil { return nil, err } @@ -217,12 +268,11 @@ func (p *proxyRepo) GoMod(version string) ([]byte, error) { } func (p *proxyRepo) Zip(dst io.Writer, version string) error { - var body io.ReadCloser encVer, err := module.EncodeVersion(version) if err != nil { return err } - err = webGetBody(p.url+"/@v/"+pathEscape(encVer)+".zip", &body) + body, err := p.getBody("@v/" + encVer + ".zip") if err != nil { return err } @@ -242,5 +292,5 @@ func (p *proxyRepo) Zip(dst io.Writer, version string) error { // That is, it escapes things like ? and # (which really shouldn't appear anyway). // It does not escape / to %2F: our REST API is designed so that / can be left as is. func pathEscape(s string) string { - return strings.ReplaceAll(url.PathEscape(s), "%2F", "/") + return strings.ReplaceAll(urlpkg.PathEscape(s), "%2F", "/") } diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index ab6e46dd74de7d..c3c4adebb6b10f 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -209,7 +209,7 @@ func lookup(path string) (r Repo, err error) { return lookupProxy(path) } - security := web.Secure + security := web.SecureOnly if get.Insecure { security = web.Insecure } @@ -254,7 +254,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { // Note: Because we are converting a code reference from a legacy // version control system, we ignore meta tags about modules // and use only direct source control entries (get.IgnoreMod). - security := web.Secure + security := web.SecureOnly if get.Insecure { security = web.Insecure } diff --git a/src/cmd/go/internal/modfetch/web.go b/src/cmd/go/internal/modfetch/web.go deleted file mode 100644 index b327bf293d1de6..00000000000000 --- a/src/cmd/go/internal/modfetch/web.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !cmd_go_bootstrap - -package modfetch - -import ( - "io" - - web "cmd/go/internal/web2" -) - -// webGetGoGet fetches a go-get=1 URL and returns the body in *body. -// It allows non-200 responses, as usual for these URLs. -func webGetGoGet(url string, body *io.ReadCloser) error { - return web.Get(url, web.Non200OK(), web.Body(body)) -} - -// webGetBytes returns the body returned by an HTTP GET, as a []byte. -// It insists on a 200 response. -func webGetBytes(url string, body *[]byte) error { - return web.Get(url, web.ReadAllBody(body)) -} - -// webGetBody returns the body returned by an HTTP GET, as a io.ReadCloser. -// It insists on a 200 response. -func webGetBody(url string, body *io.ReadCloser) error { - return web.Get(url, web.Body(body)) -} diff --git a/src/cmd/go/internal/web/api.go b/src/cmd/go/internal/web/api.go new file mode 100644 index 00000000000000..5dc81de9b66110 --- /dev/null +++ b/src/cmd/go/internal/web/api.go @@ -0,0 +1,102 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package web defines minimal helper routines for accessing HTTP/HTTPS +// resources without requiring external dependenicies on the net package. +// +// If the cmd_go_bootstrap build tag is present, web avoids the use of the net +// package and returns errors for all network operations. +package web + +import ( + "fmt" + "io" + "io/ioutil" + urlpkg "net/url" +) + +// SecurityMode specifies whether a function should make network +// calls using insecure transports (eg, plain text HTTP). +// The zero value is "secure". +type SecurityMode int + +const ( + SecureOnly SecurityMode = iota // Reject plain HTTP; validate HTTPS. + DefaultSecurity // Allow plain HTTP if explicit; validate HTTPS. + Insecure // Allow plain HTTP if not explicitly HTTPS; skip HTTPS validation. +) + +type HTTPError struct { + status string + StatusCode int + url *urlpkg.URL +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("%s: %s", e.url, e.status) +} + +// GetBytes returns the body of the requested resource, or an error if the +// response status was not http.StatusOk. +// +// GetBytes is a convenience wrapper around Get. +func GetBytes(url *urlpkg.URL) ([]byte, error) { + url, resp, err := Get(DefaultSecurity, url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url} + return nil, err + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("%s: %v", url, err) + } + return b, nil +} + +type Response struct { + Status string + StatusCode int + Header map[string][]string + Body io.ReadCloser +} + +// Get returns the body of the HTTP or HTTPS resource specified at the given URL. +// +// If the URL does not include an explicit scheme, Get first tries "https". +// If the server does not respond under that scheme and the security mode is +// Insecure, Get then tries "http". +// The returned URL indicates which scheme was actually used. +// +// For the "https" scheme only, credentials are attached using the +// cmd/go/internal/auth package. If the URL itself includes a username and +// password, it will not be attempted under the "http" scheme unless the +// security mode is Insecure. +// +// Get returns a non-nil error only if the request did not receive a response +// under any applicable scheme. (A non-2xx response does not cause an error.) +func Get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) { + return get(security, url) +} + +// PasswordRedacted returns url directly if it does not encode a password, +// or else a copy of url with the password redacted. +func PasswordRedacted(url *urlpkg.URL) *urlpkg.URL { + if url.User != nil { + if _, ok := url.User.Password(); ok { + redacted := *url + redacted.User = urlpkg.UserPassword(url.User.Username(), "[redacted]") + return &redacted + } + } + return url +} + +// OpenBrowser attempts to open the requested URL in a web browser. +func OpenBrowser(url string) (opened bool) { + return openBrowser(url) +} diff --git a/src/cmd/go/internal/web/bootstrap.go b/src/cmd/go/internal/web/bootstrap.go index d1d4621a44b688..84e9d356448117 100644 --- a/src/cmd/go/internal/web/bootstrap.go +++ b/src/cmd/go/internal/web/bootstrap.go @@ -6,32 +6,18 @@ // This code is compiled only into the bootstrap 'go' binary. // These stubs avoid importing packages with large dependency -// trees, like the use of "net/http" in vcs.go. +// trees that potentially require C linking, +// like the use of "net/http" in vcs.go. package web import ( "errors" - "io" + urlpkg "net/url" ) -var errHTTP = errors.New("no http in bootstrap go command") - -type HTTPError struct { - StatusCode int -} - -func (e *HTTPError) Error() string { - panic("unreachable") -} - -func Get(url string) ([]byte, error) { - return nil, errHTTP -} - -func GetMaybeInsecure(importPath string, security SecurityMode) (string, io.ReadCloser, error) { - return "", nil, errHTTP +func get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) { + return nil, nil, errors.New("no http in bootstrap go command") } -func QueryEscape(s string) string { panic("unreachable") } -func OpenBrowser(url string) bool { panic("unreachable") } +func openBrowser(url string) bool { return false } diff --git a/src/cmd/go/internal/web/http.go b/src/cmd/go/internal/web/http.go index c1714b4d38b1aa..0711f812091a79 100644 --- a/src/cmd/go/internal/web/http.go +++ b/src/cmd/go/internal/web/http.go @@ -14,13 +14,12 @@ package web import ( "crypto/tls" "fmt" - "io" - "io/ioutil" "log" "net/http" - "net/url" + urlpkg "net/url" "time" + "cmd/go/internal/auth" "cmd/go/internal/cfg" "cmd/internal/browser" ) @@ -50,81 +49,92 @@ var securityPreservingHTTPClient = &http.Client{ }, } -type HTTPError struct { - status string - StatusCode int - url string -} - -func (e *HTTPError) Error() string { - return fmt.Sprintf("%s: %s", e.url, e.status) -} - -// Get returns the data from an HTTP GET request for the given URL. -func Get(url string) ([]byte, error) { - resp, err := securityPreservingHTTPClient.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url} - - return nil, err - } - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("%s: %v", url, err) - } - return b, nil -} +func get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) { + fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) { + if cfg.BuildV { + log.Printf("Fetching %s", url) + } -// GetMaybeInsecure returns the body of either the importPath's -// https resource or, if unavailable and permitted by the security mode, the http resource. -func GetMaybeInsecure(importPath string, security SecurityMode) (urlStr string, body io.ReadCloser, err error) { - fetch := func(scheme string) (urlStr string, res *http.Response, err error) { - u, err := url.Parse(scheme + "://" + importPath) + req, err := http.NewRequest("GET", url.String(), nil) if err != nil { - return "", nil, err + return nil, nil, err } - u.RawQuery = "go-get=1" - urlStr = u.String() - if cfg.BuildV { - log.Printf("Fetching %s", urlStr) + if url.Scheme == "https" { + auth.AddCredentials(req) } - if security == Insecure && scheme == "https" { // fail earlier - res, err = impatientInsecureHTTPClient.Get(urlStr) + + var res *http.Response + if security == Insecure && url.Scheme == "https" { // fail earlier + res, err = impatientInsecureHTTPClient.Do(req) } else { - res, err = securityPreservingHTTPClient.Get(urlStr) + res, err = securityPreservingHTTPClient.Do(req) } - return + return url, res, err } - closeBody := func(res *http.Response) { - if res != nil { - res.Body.Close() + + var ( + fetched *urlpkg.URL + res *http.Response + err error + ) + if url.Scheme == "" || url.Scheme == "https" { + secure := new(urlpkg.URL) + *secure = *url + secure.Scheme = "https" + + fetched, res, err = fetch(secure) + if err != nil { + if cfg.BuildV { + log.Printf("https fetch failed: %v", err) + } + if security != Insecure || url.Scheme == "https" { + // HTTPS failed, and we can't fall back to plain HTTP. + // Report the error from the HTTPS attempt. + return nil, nil, err + } } } - urlStr, res, err := fetch("https") - if err != nil { - if cfg.BuildV { - log.Printf("https fetch failed: %v", err) + + if res == nil { + switch url.Scheme { + case "http": + if security == SecureOnly { + return nil, nil, fmt.Errorf("URL %q is not secure", PasswordRedacted(url)) + } + case "": + if security != Insecure { + panic("should have returned after HTTPS failure") + } + default: + return nil, nil, fmt.Errorf("unsupported scheme %s", url.Scheme) } - if security == Insecure { - closeBody(res) - urlStr, res, err = fetch("http") + + insecure := new(urlpkg.URL) + *insecure = *url + insecure.Scheme = "http" + if insecure.User != nil && security != Insecure { + return nil, nil, fmt.Errorf("refusing to pass credentials to insecure URL %q", PasswordRedacted(insecure)) + } + + fetched, res, err = fetch(insecure) + if err != nil { + // HTTP failed, and we already tried HTTPS if applicable. + // Report the error from the HTTP attempt. + return nil, nil, err } } - if err != nil { - closeBody(res) - return "", nil, err - } + // Note: accepting a non-200 OK here, so people can serve a // meta import in their http 404 page. if cfg.BuildV { - log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode) + log.Printf("Parsing meta tags from %s (status code %d)", PasswordRedacted(fetched), res.StatusCode) } - return urlStr, res.Body, nil + return fetched, &Response{ + Status: res.Status, + StatusCode: res.StatusCode, + Header: map[string][]string(res.Header), + Body: res.Body, + }, nil } -func QueryEscape(s string) string { return url.QueryEscape(s) } -func OpenBrowser(url string) bool { return browser.Open(url) } +func openBrowser(url string) bool { return browser.Open(url) } diff --git a/src/cmd/go/internal/web/security.go b/src/cmd/go/internal/web/security.go deleted file mode 100644 index 1dc6f1b076fb46..00000000000000 --- a/src/cmd/go/internal/web/security.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package web defines helper routines for accessing HTTP/HTTPS resources. -package web - -// SecurityMode specifies whether a function should make network -// calls using insecure transports (eg, plain text HTTP). -// The zero value is "secure". -type SecurityMode int - -const ( - Secure SecurityMode = iota - Insecure -) diff --git a/src/cmd/go/internal/web2/web.go b/src/cmd/go/internal/web2/web.go deleted file mode 100644 index 02b828ffa67545..00000000000000 --- a/src/cmd/go/internal/web2/web.go +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package web2 - -import ( - "bytes" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "runtime" - "runtime/debug" - "strings" - "sync" -) - -var TraceGET = false -var webstack = false - -func init() { - flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests") - flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests") -} - -type netrcLine struct { - machine string - login string - password string -} - -var ( - netrcOnce sync.Once - netrc []netrcLine - netrcErr error -) - -func parseNetrc(data string) []netrcLine { - // See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html - // for documentation on the .netrc format. - var nrc []netrcLine - var l netrcLine - inMacro := false - for _, line := range strings.Split(data, "\n") { - if inMacro { - if line == "" { - inMacro = false - } - continue - } - - f := strings.Fields(line) - i := 0 - for ; i < len(f)-1; i += 2 { - // Reset at each "machine" token. - // “The auto-login process searches the .netrc file for a machine token - // that matches […]. Once a match is made, the subsequent .netrc tokens - // are processed, stopping when the end of file is reached or another - // machine or a default token is encountered.” - switch f[i] { - case "machine": - l = netrcLine{machine: f[i+1]} - case "default": - break - case "login": - l.login = f[i+1] - case "password": - l.password = f[i+1] - case "macdef": - // “A macro is defined with the specified name; its contents begin with - // the next .netrc line and continue until a null line (consecutive - // new-line characters) is encountered.” - inMacro = true - } - if l.machine != "" && l.login != "" && l.password != "" { - nrc = append(nrc, l) - l = netrcLine{} - } - } - - if i < len(f) && f[i] == "default" { - // “There can be only one default token, and it must be after all machine tokens.” - break - } - } - - return nrc -} - -func havePassword(machine string) bool { - netrcOnce.Do(readNetrc) - for _, line := range netrc { - if line.machine == machine { - return true - } - } - return false -} - -func netrcPath() (string, error) { - if env := os.Getenv("NETRC"); env != "" { - return env, nil - } - dir, err := os.UserHomeDir() - if err != nil { - return "", err - } - base := ".netrc" - if runtime.GOOS == "windows" { - base = "_netrc" - } - return filepath.Join(dir, base), nil -} - -func readNetrc() { - path, err := netrcPath() - if err != nil { - netrcErr = err - return - } - - data, err := ioutil.ReadFile(path) - if err != nil { - if !os.IsNotExist(err) { - netrcErr = err - } - return - } - - netrc = parseNetrc(string(data)) -} - -type getState struct { - req *http.Request - resp *http.Response - body io.ReadCloser - non200ok bool -} - -type Option interface { - option(*getState) error -} - -func Non200OK() Option { - return optionFunc(func(g *getState) error { - g.non200ok = true - return nil - }) -} - -type optionFunc func(*getState) error - -func (f optionFunc) option(g *getState) error { - return f(g) -} - -func DecodeJSON(dst interface{}) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - return json.NewDecoder(g.body).Decode(dst) - } - return nil - }) -} - -func ReadAllBody(body *[]byte) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - var err error - *body, err = ioutil.ReadAll(g.body) - return err - } - return nil - }) -} - -func Body(body *io.ReadCloser) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - *body = g.body - g.body = nil - } - return nil - }) -} - -func Header(hdr *http.Header) Option { - return optionFunc(func(g *getState) error { - if g.resp != nil { - *hdr = CopyHeader(g.resp.Header) - } - return nil - }) -} - -func CopyHeader(hdr http.Header) http.Header { - if hdr == nil { - return nil - } - h2 := make(http.Header) - for k, v := range hdr { - v2 := make([]string, len(v)) - copy(v2, v) - h2[k] = v2 - } - return h2 -} - -var cache struct { - mu sync.Mutex - byURL map[string]*cacheEntry -} - -type cacheEntry struct { - mu sync.Mutex - resp *http.Response - body []byte -} - -var httpDo = http.DefaultClient.Do - -func SetHTTPDoForTesting(do func(*http.Request) (*http.Response, error)) { - if do == nil { - do = http.DefaultClient.Do - } - httpDo = do -} - -func Get(url string, options ...Option) error { - if TraceGET || webstack || cfg.BuildV { - log.Printf("Fetching %s", url) - if webstack { - log.Println(string(debug.Stack())) - } - } - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - - netrcOnce.Do(readNetrc) - for _, l := range netrc { - if l.machine == req.URL.Host { - req.SetBasicAuth(l.login, l.password) - break - } - } - - g := &getState{req: req} - for _, o := range options { - if err := o.option(g); err != nil { - return err - } - } - - cache.mu.Lock() - e := cache.byURL[url] - if e == nil { - e = new(cacheEntry) - if !strings.HasPrefix(url, "file:") { - if cache.byURL == nil { - cache.byURL = make(map[string]*cacheEntry) - } - cache.byURL[url] = e - } - } - cache.mu.Unlock() - - e.mu.Lock() - if strings.HasPrefix(url, "file:") { - body, err := ioutil.ReadFile(req.URL.Path) - if err != nil { - e.mu.Unlock() - return err - } - e.body = body - e.resp = &http.Response{ - StatusCode: 200, - } - } else if e.resp == nil { - resp, err := httpDo(req) - if err != nil { - e.mu.Unlock() - return err - } - e.resp = resp - // TODO: Spool to temp file. - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - resp.Body = nil - if err != nil { - e.mu.Unlock() - return err - } - e.body = body - } - g.resp = e.resp - g.body = ioutil.NopCloser(bytes.NewReader(e.body)) - e.mu.Unlock() - - defer func() { - if g.body != nil { - g.body.Close() - } - }() - - if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") { - base.Errorf("%s", githubMessage) - } - if !g.non200ok && g.resp.StatusCode != 200 { - return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status) - } - - for _, o := range options { - if err := o.option(g); err != nil { - return err - } - } - return err -} - -var githubMessage = `go: 403 response from api.github.com - -GitHub applies fairly small rate limits to unauthenticated users, and -you appear to be hitting them. To authenticate, please visit -https://github.com/settings/tokens and click "Generate New Token" to -create a Personal Access Token. The token only needs "public_repo" -scope, but you can add "repo" if you want to access private -repositories too. - -Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows): - - machine api.github.com login YOU password TOKEN - -Sorry for the interruption. -` diff --git a/src/cmd/go/internal/webtest/test.go b/src/cmd/go/internal/webtest/test.go deleted file mode 100644 index 94b20a33ffc7d5..00000000000000 --- a/src/cmd/go/internal/webtest/test.go +++ /dev/null @@ -1,314 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package webtest - -import ( - "bufio" - "bytes" - "encoding/hex" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "sort" - "strconv" - "strings" - "sync" - "unicode/utf8" - - web "cmd/go/internal/web2" -) - -var mode = flag.String("webtest", "replay", "set webtest `mode` - record, replay, bypass") - -func Hook() { - if *mode == "bypass" { - return - } - web.SetHTTPDoForTesting(Do) -} - -func Unhook() { - web.SetHTTPDoForTesting(nil) -} - -func Print() { - web.SetHTTPDoForTesting(DoPrint) -} - -var responses struct { - mu sync.Mutex - byURL map[string]*respEntry -} - -type respEntry struct { - status string - code int - hdr http.Header - body []byte -} - -func Serve(url string, status string, hdr http.Header, body []byte) { - if status == "" { - status = "200 OK" - } - code, err := strconv.Atoi(strings.Fields(status)[0]) - if err != nil { - panic("bad Serve status - " + status + " - " + err.Error()) - } - - responses.mu.Lock() - defer responses.mu.Unlock() - - if responses.byURL == nil { - responses.byURL = make(map[string]*respEntry) - } - responses.byURL[url] = &respEntry{status: status, code: code, hdr: web.CopyHeader(hdr), body: body} -} - -func Do(req *http.Request) (*http.Response, error) { - if req.Method != "GET" { - return nil, fmt.Errorf("bad method - must be GET") - } - - responses.mu.Lock() - e := responses.byURL[req.URL.String()] - responses.mu.Unlock() - - if e == nil { - if *mode == "record" { - loaded.mu.Lock() - if len(loaded.did) != 1 { - loaded.mu.Unlock() - return nil, fmt.Errorf("cannot use -webtest=record with multiple loaded response files") - } - var file string - for file = range loaded.did { - break - } - loaded.mu.Unlock() - return doSave(file, req) - } - e = &respEntry{code: 599, status: "599 unexpected request (no canned response)"} - } - resp := &http.Response{ - Status: e.status, - StatusCode: e.code, - Header: web.CopyHeader(e.hdr), - Body: ioutil.NopCloser(bytes.NewReader(e.body)), - } - return resp, nil -} - -func DoPrint(req *http.Request) (*http.Response, error) { - return doSave("", req) -} - -func doSave(file string, req *http.Request) (*http.Response, error) { - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - data, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, err - } - resp.Body = ioutil.NopCloser(bytes.NewReader(data)) - - var f *os.File - if file == "" { - f = os.Stderr - } else { - f, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - log.Fatal(err) - } - defer f.Close() - } - - fmt.Fprintf(f, "GET %s\n", req.URL.String()) - fmt.Fprintf(f, "%s\n", resp.Status) - var keys []string - for k := range resp.Header { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - if k == "Set-Cookie" { - continue - } - for _, v := range resp.Header[k] { - fmt.Fprintf(f, "%s: %s\n", k, v) - } - } - fmt.Fprintf(f, "\n") - if utf8.Valid(data) && !bytes.Contains(data, []byte("\nGET")) && !isHexDump(data) { - fmt.Fprintf(f, "%s\n\n", data) - } else { - fmt.Fprintf(f, "%s\n", hex.Dump(data)) - } - return resp, err -} - -var loaded struct { - mu sync.Mutex - did map[string]bool -} - -func LoadOnce(file string) { - loaded.mu.Lock() - if loaded.did[file] { - loaded.mu.Unlock() - return - } - if loaded.did == nil { - loaded.did = make(map[string]bool) - } - loaded.did[file] = true - loaded.mu.Unlock() - - f, err := os.Open(file) - if err != nil { - log.Fatal(err) - } - defer f.Close() - - b := bufio.NewReader(f) - var ungetLine string - nextLine := func() string { - if ungetLine != "" { - l := ungetLine - ungetLine = "" - return l - } - line, err := b.ReadString('\n') - if err != nil { - if err == io.EOF { - return "" - } - log.Fatalf("%s: unexpected read error: %v", file, err) - } - return line - } - - for { - line := nextLine() - if line == "" { // EOF - break - } - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "#") || line == "" { - continue - } - if !strings.HasPrefix(line, "GET ") { - log.Fatalf("%s: malformed GET line: %s", file, line) - } - url := line[len("GET "):] - status := nextLine() - if _, err := strconv.Atoi(strings.Fields(status)[0]); err != nil { - log.Fatalf("%s: malformed status line (after GET %s): %s", file, url, status) - } - hdr := make(http.Header) - for { - kv := strings.TrimSpace(nextLine()) - if kv == "" { - break - } - i := strings.Index(kv, ":") - if i < 0 { - log.Fatalf("%s: malformed header line (after GET %s): %s", file, url, kv) - } - k, v := kv[:i], strings.TrimSpace(kv[i+1:]) - hdr[k] = append(hdr[k], v) - } - - var body []byte - Body: - for n := 0; ; n++ { - line := nextLine() - if n == 0 && isHexDump([]byte(line)) { - ungetLine = line - b, err := parseHexDump(nextLine) - if err != nil { - log.Fatalf("%s: malformed hex dump (after GET %s): %v", file, url, err) - } - body = b - break - } - if line == "" { // EOF - for i := 0; i < 2; i++ { - if len(body) > 0 && body[len(body)-1] == '\n' { - body = body[:len(body)-1] - } - } - break - } - body = append(body, line...) - for line == "\n" { - line = nextLine() - if strings.HasPrefix(line, "GET ") { - ungetLine = line - body = body[:len(body)-1] - if len(body) > 0 { - body = body[:len(body)-1] - } - break Body - } - body = append(body, line...) - } - } - - Serve(url, status, hdr, body) - } -} - -func isHexDump(data []byte) bool { - return bytes.HasPrefix(data, []byte("00000000 ")) || bytes.HasPrefix(data, []byte("0000000 ")) -} - -// parseHexDump parses the hex dump in text, which should be the -// output of "hexdump -C" or Plan 9's "xd -b" or Go's hex.Dump -// and returns the original data used to produce the dump. -// It is meant to enable storing golden binary files as text, so that -// changes to the golden files can be seen during code reviews. -func parseHexDump(nextLine func() string) ([]byte, error) { - var out []byte - for { - line := nextLine() - if line == "" || line == "\n" { - break - } - if i := strings.Index(line, "|"); i >= 0 { // remove text dump - line = line[:i] - } - f := strings.Fields(line) - if len(f) > 1+16 { - return nil, fmt.Errorf("parsing hex dump: too many fields on line %q", line) - } - if len(f) == 0 || len(f) == 1 && f[0] == "*" { // all zeros block omitted - continue - } - addr64, err := strconv.ParseUint(f[0], 16, 0) - if err != nil { - return nil, fmt.Errorf("parsing hex dump: invalid address %q", f[0]) - } - addr := int(addr64) - if len(out) < addr { - out = append(out, make([]byte, addr-len(out))...) - } - for _, x := range f[1:] { - val, err := strconv.ParseUint(x, 16, 8) - if err != nil { - return nil, fmt.Errorf("parsing hexdump: invalid hex byte %q", x) - } - out = append(out, byte(val)) - } - } - return out, nil -} diff --git a/src/cmd/go/testdata/script/get_404_meta.txt b/src/cmd/go/testdata/script/get_404_meta.txt new file mode 100644 index 00000000000000..32f13c936736a6 --- /dev/null +++ b/src/cmd/go/testdata/script/get_404_meta.txt @@ -0,0 +1,10 @@ +# golang.org/issue/13037: 'go get' was not parsing tags in 404 served over HTTPS. + +[!net] skip + +env GO111MODULE=off +go get -d -insecure bazil.org/fuse/fs/fstestutil + +env GO111MODULE=on +env GOPROXY=direct +go get -d -insecure bazil.org/fuse/fs/fstestutil diff --git a/src/cmd/go/testdata/script/get_insecure_redirect.txt b/src/cmd/go/testdata/script/get_insecure_redirect.txt index c3520bfcab0fd4..e05ced67a3b435 100644 --- a/src/cmd/go/testdata/script/get_insecure_redirect.txt +++ b/src/cmd/go/testdata/script/get_insecure_redirect.txt @@ -1,4 +1,3 @@ -# golang.org/issue/13037: 'go get' was not parsing tags in 404 served over HTTPS. # golang.org/issue/29591: 'go get' was following plain-HTTP redirects even without -insecure. [!net] skip diff --git a/src/cmd/go/testdata/script/mod_auth.txt b/src/cmd/go/testdata/script/mod_auth.txt new file mode 100644 index 00000000000000..b47db9c073baac --- /dev/null +++ b/src/cmd/go/testdata/script/mod_auth.txt @@ -0,0 +1,31 @@ +[!net] skip + +env GO111MODULE=on +env GOPROXY=direct + +# Without credentials, downloading a module from a path that requires HTTPS +# basic auth should fail. +env NETRC=$WORK/empty +! go list all + +# With credentials from a netrc file, it should succeed. +env NETRC=$WORK/netrc +go mod tidy +go list all +stdout vcs-test.golang.org/auth/or401 +stdout vcs-test.golang.org/auth/or404 + +-- go.mod -- +module private.example.com +-- main.go -- +package useprivate + +import ( + _ "vcs-test.golang.org/auth/or401" + _ "vcs-test.golang.org/auth/or404" +) +-- $WORK/empty -- +-- $WORK/netrc -- +machine vcs-test.golang.org + login aladdin + password opensesame