cmd/go/internal/web: merge internal/web2 into web

The cmd/go/internal/web package was forked in order to support direct
HTTPS fetches from widely-used hosting providers,¹ but direct fetches
were subsequently dropped in CL 107657. The forked web2 package, with
its GitHub-specific diagnostics and .netrc support, remained in use
for module proxy support, but was not used for the initial '?go-get=1'
path resolution, so the .netrc file was only used to fetch from
already-resolved module protocol servers.

This CL moves the .netrc support into its own (new) package,
cmd/go/internal/auth, and consolidates the web and web2 packages back
into just web. As a result, fetches via the web package now support
.netrc, and fetches that previously used web2 now enforce the same
security policies as web (such as prohibiting HTTPS-to-HTTP
redirects).

¹golang/vgo@63138cb

Fixes #29591
Fixes #29888
Fixes #30610
Updates #26232

Change-Id: Ia3a13526e443679cf14a72a1f3db96f336ce5e73
Reviewed-on: https://go-review.googlesource.com/c/go/+/170879
Run-TryBot: Russ Cox <[email protected]>
Reviewed-by: Russ Cox <[email protected]>
Reviewed-by: Jay Conrod <[email protected]>

src/cmd/go/internal/auth/auth.go

@@ -0,0 +1,23 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package auth provides access to user-provided authentication credentials.
+package auth
+
+import "net/http"
+
+// AddCredentials fills in the user's credentials for req, if any.
+// The return value reports whether any matching credentials were found.
+func AddCredentials(req *http.Request) (added bool) {
+	// TODO(golang.org/issue/26232): Support arbitrary user-provided credentials.
+	netrcOnce.Do(readNetrc)
+	for _, l := range netrc {
+		if l.machine == req.URL.Host {
+			req.SetBasicAuth(l.login, l.password)
+			return true
+		}
+	}
+
+	return false
+}

src/cmd/go/internal/auth/netrc.go

@@ -0,0 +1,111 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+type netrcLine struct {
+	machine  string
+	login    string
+	password string
+}
+
+var (
+	netrcOnce sync.Once
+	netrc     []netrcLine
+	netrcErr  error
+)
+
+func parseNetrc(data string) []netrcLine {
+	// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
+	// for documentation on the .netrc format.
+	var nrc []netrcLine
+	var l netrcLine
+	inMacro := false
+	for _, line := range strings.Split(data, "\n") {
+		if inMacro {
+			if line == "" {
+				inMacro = false
+			}
+			continue
+		}
+
+		f := strings.Fields(line)
+		i := 0
+		for ; i < len(f)-1; i += 2 {
+			// Reset at each "machine" token.
+			// “The auto-login process searches the .netrc file for a machine token
+			// that matches […]. Once a match is made, the subsequent .netrc tokens
+			// are processed, stopping when the end of file is reached or another
+			// machine or a default token is encountered.”
+			switch f[i] {
+			case "machine":
+				l = netrcLine{machine: f[i+1]}
+			case "default":
+				break
+			case "login":
+				l.login = f[i+1]
+			case "password":
+				l.password = f[i+1]
+			case "macdef":
+				// “A macro is defined with the specified name; its contents begin with
+				// the next .netrc line and continue until a null line (consecutive
+				// new-line characters) is encountered.”
+				inMacro = true
+			}
+			if l.machine != "" && l.login != "" && l.password != "" {
+				nrc = append(nrc, l)
+				l = netrcLine{}
+			}
+		}
+
+		if i < len(f) && f[i] == "default" {
+			// “There can be only one default token, and it must be after all machine tokens.”
+			break
+		}
+	}
+
+	return nrc
+}
+
+func netrcPath() (string, error) {
+	if env := os.Getenv("NETRC"); env != "" {
+		return env, nil
+	}
+	dir, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	base := ".netrc"
+	if runtime.GOOS == "windows" {
+		base = "_netrc"
+	}
+	return filepath.Join(dir, base), nil
+}
+
+func readNetrc() {
+	path, err := netrcPath()
+	if err != nil {
+		netrcErr = err
+		return
+	}
+
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			netrcErr = err
+		}
+		return
+	}
+
+	netrc = parseNetrc(string(data))
+}

src/cmd/go/internal/web2/web_test.go → src/cmd/go/internal/auth/netrc_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package web2
+package auth
 
 import (
 	"reflect"
@@ -43,7 +43,7 @@ login oops
 password too-late-in-file
 `
 
-func TestReadNetrc(t *testing.T) {
+func TestParseNetrc(t *testing.T) {
 	lines := parseNetrc(testNetrc)
 	want := []netrcLine{
 		{"api.github.com", "user", "pwd"},

src/cmd/go/internal/bug/bug.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	urlpkg "net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -62,7 +63,7 @@ func runBug(cmd *base.Command, args []string) {
 	fmt.Fprintln(&buf, "```")
 
 	body := buf.String()
-	url := "https://github.com/golang/go/issues/new?body=" + web.QueryEscape(body)
+	url := "https://github.com/golang/go/issues/new?body=" + urlpkg.QueryEscape(body)
 	if !web.OpenBrowser(url) {
 		fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n")
 		fmt.Print(body)
@@ -130,7 +131,12 @@ func printCDetails(w io.Writer) {
 }
 
 func inspectGoVersion(w io.Writer) {
-	data, err := web.Get("https://golang.org/VERSION?m=text")
+	data, err := web.GetBytes(&urlpkg.URL{
+		Scheme:   "https",
+		Host:     "golang.org",
+		Path:     "/VERSION",
+		RawQuery: "?m=text",
+	})
 	if err != nil {
 		if cfg.BuildV {
 			fmt.Printf("failed to read from golang.org/VERSION: %v\n", err)

src/cmd/go/internal/get/get.go

@@ -392,7 +392,7 @@ func downloadPackage(p *load.Package) error {
 		blindRepo      bool // set if the repo has unusual configuration
 	)
 
-	security := web.Secure
+	security := web.SecureOnly
 	if Insecure {
 		security = web.Insecure
 	}