Merge branch 'master' into regen_gocloud

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/jstemmer/go-junit-report v0.9.1
 	go.opencensus.io v0.22.4
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
-	golang.org/x/net v0.0.0-20200904194848-62affa334b73
+	golang.org/x/net v0.0.0-20200927032502-5d4f70055728
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
 	golang.org/x/text v0.3.3
 	golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab

go.sum

@@ -255,8 +255,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgN
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0=
+golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

internal/gapicgen/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/google/go-github/v32 v32.1.0
 	github.com/kr/text v0.2.0 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/shurcooL/githubv4 v0.0.0-20200915023059-bc5e4feb2971
+	github.com/shurcooL/githubv4 v0.0.0-20200928013246-d292edc3691b
 	github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect
 	github.com/stretchr/testify v1.5.1 // indirect
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43

internal/gapicgen/go.sum

@@ -155,8 +155,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shurcooL/githubv4 v0.0.0-20200915023059-bc5e4feb2971 h1:H/EGuDmV4TUM5cJx+5KehAPmTDXx/z2FjOLZSf6B6Xg=
-github.com/shurcooL/githubv4 v0.0.0-20200915023059-bc5e4feb2971/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo=
+github.com/shurcooL/githubv4 v0.0.0-20200928013246-d292edc3691b h1:0/ecDXh/HTHRtSDSFnD2/Ta1yQ5J76ZspVY4u0/jGFk=
+github.com/shurcooL/githubv4 v0.0.0-20200928013246-d292edc3691b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo=
 github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk=
 github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg=
 github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=

internal/godocfx/go.sum

@@ -151,8 +151,8 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0=
+golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

internal/godocfx/godocfx_test.go

@@ -36,21 +36,21 @@ func TestMain(m *testing.M) {
 
 func TestParse(t *testing.T) {
 	testPath := "cloud.google.com/go/storage"
-	pages, toc, module, err := parse(testPath)
+	r, err := parse(testPath, nil)
 	if err != nil {
 		t.Fatalf("Parse: %v", err)
 	}
-	if got, want := len(toc), 1; got != want {
+	if got, want := len(r.toc), 1; got != want {
 		t.Fatalf("Parse got len(toc) = %d, want %d", got, want)
 	}
-	if got, want := len(pages), 1; got != want {
+	if got, want := len(r.pages), 1; got != want {
 		t.Errorf("Parse got len(pages) = %d, want %d", got, want)
 	}
-	if got := module.Path; got != testPath {
+	if got := r.module.Path; got != testPath {
 		t.Fatalf("Parse got module = %q, want %q", got, testPath)
 	}
 
-	page := pages[testPath]
+	page := r.pages[testPath]
 
 	// Check invariants for every item.
 	for _, item := range page.Items {
@@ -64,8 +64,7 @@ func TestParse(t *testing.T) {
 	}
 
 	// Check there is at least one type, const, variable, and function.
-	// Note: no method because they aren't printed for Namespaces yet.
-	wants := []string{"type", "const", "variable", "function"}
+	wants := []string{"type", "const", "variable", "function", "method"}
 	for _, want := range wants {
 		found := false
 		for _, c := range page.Items {
@@ -83,9 +82,10 @@ func TestParse(t *testing.T) {
 func TestGoldens(t *testing.T) {
 	gotDir := "testdata/out"
 	goldenDir := "testdata/golden"
+	extraFiles := []string{"README.md"}
 
 	testPath := "cloud.google.com/go/storage"
-	pages, toc, module, err := parse(testPath)
+	r, err := parse(testPath, extraFiles)
 	if err != nil {
 		t.Fatalf("parse: %v", err)
 	}
@@ -95,7 +95,7 @@ func TestGoldens(t *testing.T) {
 	if updateGoldens {
 		os.RemoveAll(goldenDir)
 
-		if err := write(goldenDir, pages, toc, module); err != nil {
+		if err := write(goldenDir, r, extraFiles); err != nil {
 			t.Fatalf("write: %v", err)
 		}
 
@@ -110,7 +110,7 @@ func TestGoldens(t *testing.T) {
 		return
 	}
 
-	if err := write(gotDir, pages, toc, module); err != nil {
+	if err := write(gotDir, r, extraFiles); err != nil {
 		t.Fatalf("write: %v", err)
 	}
 

internal/godocfx/main.go

@@ -39,13 +39,13 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"golang.org/x/tools/go/packages"
 	"gopkg.in/yaml.v2"
 )
 
@@ -58,17 +58,20 @@ func main() {
 		log.Fatalf("%s missing required argument: module path", os.Args[0])
 	}
 
-	pages, toc, module, err := parse(flag.Arg(0))
+	extraFiles := []string{
+		"README.md",
+	}
+	r, err := parse(flag.Arg(0), extraFiles)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	if *print {
-		if err := yaml.NewEncoder(os.Stdout).Encode(pages); err != nil {
+		if err := yaml.NewEncoder(os.Stdout).Encode(r.pages); err != nil {
 			log.Fatal(err)
 		}
 		fmt.Println("----- toc.yaml")
-		if err := yaml.NewEncoder(os.Stdout).Encode(toc); err != nil {
+		if err := yaml.NewEncoder(os.Stdout).Encode(r.toc); err != nil {
 			log.Fatal(err)
 		}
 		return
@@ -78,22 +81,22 @@ func main() {
 		os.RemoveAll(*outDir)
 	}
 
-	if err := write(*outDir, pages, toc, module); err != nil {
+	if err := write(*outDir, r, extraFiles); err != nil {
 		log.Fatalf("write: %v", err)
 	}
 }
 
-func write(outDir string, pages map[string]*page, toc tableOfContents, module *packages.Module) error {
+func write(outDir string, r *result, extraFiles []string) error {
 	if err := os.MkdirAll(outDir, os.ModePerm); err != nil {
 		return fmt.Errorf("os.MkdirAll: %v", err)
 	}
-	for path, p := range pages {
+	for path, p := range r.pages {
 		// Make the module root page the index.
-		if path == module.Path {
+		if path == r.module.Path {
 			path = "index"
 		}
 		// Trim the module path from all other paths.
-		path = strings.TrimPrefix(path, module.Path+"/")
+		path = strings.TrimPrefix(path, r.module.Path+"/")
 		path = filepath.Join(outDir, path+".yml")
 		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
 			return fmt.Errorf("os.MkdirAll: %v", err)
@@ -115,9 +118,23 @@ func write(outDir string, pages map[string]*page, toc tableOfContents, module *p
 		}
 		defer f.Close()
 		fmt.Fprintln(f, "### YamlMime:TableOfContent")
-		if err := yaml.NewEncoder(f).Encode(toc); err != nil {
+		if err := yaml.NewEncoder(f).Encode(r.toc); err != nil {
+			return err
+		}
+	}
+
+	for _, path := range extraFiles {
+		src, err := os.Open(filepath.Join(r.module.Dir, path))
+		if err != nil {
 			return err
 		}
+		dst, err := os.Create(filepath.Join(outDir, path))
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(dst, src); err != nil {
+			return nil
+		}
 	}
 
 	// Write the docuploader docs.metadata file. Not for DocFX.
@@ -145,6 +162,6 @@ func write(outDir string, pages map[string]*page, toc tableOfContents, module *p
 }
 name: %q
 version: %q
-language: "go"`, now.Unix(), now.Nanosecond(), module.Path, module.Version)
+language: "go"`, now.Unix(), now.Nanosecond(), r.module.Path, r.module.Version)
 	return nil
 }

internal/godocfx/parse.go

@@ -26,6 +26,7 @@ import (
 	"go/printer"
 	"go/token"
 	"log"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -41,6 +42,7 @@ type tocItem struct {
 	UID   string     `yaml:"uid,omitempty"`
 	Name  string     `yaml:"name,omitempty"`
 	Items []*tocItem `yaml:"items,omitempty"`
+	Href  string     `yaml:"href,omitempty"`
 }
 
 func (t *tocItem) addItem(i *tocItem) {
@@ -92,24 +94,32 @@ func (i *item) addChild(c child) {
 
 var onlyGo = []string{"go"}
 
+type result struct {
+	pages  map[string]*page
+	toc    tableOfContents
+	module *packages.Module
+}
+
 // parse parses the directory into a map of import path -> page and a TOC.
 //
 // glob is the path to parse, usually ending in `...`. glob is passed directly
 // to packages.Load as-is.
-func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, error) {
+//
+// extraFiles is a list of paths relative to the module root to include.
+func parse(glob string, extraFiles []string) (*result, error) {
 	config := &packages.Config{
 		Mode:  packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedModule,
 		Tests: true,
 	}
 
 	pkgs, err := packages.Load(config, glob)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("packages.Load: %v", err)
+		return nil, fmt.Errorf("packages.Load: %v", err)
 	}
 	packages.PrintErrors(pkgs) // Don't fail everything because of one package.
 
 	if len(pkgs) == 0 {
-		return nil, nil, nil, fmt.Errorf("pattern %q matched 0 packages", glob)
+		return nil, fmt.Errorf("pattern %q matched 0 packages", glob)
 	}
 
 	pages := map[string]*page{}
@@ -122,7 +132,7 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
 
 	// First, collect all of the files grouped by package, including test
 	// packages.
-	allPkgFiles := map[string][]string{}
+	goPkgFiles := map[string][]string{}
 	for _, pkg := range pkgs {
 		id := pkg.ID
 		// See https://pkg.go.dev/golang.org/x/tools/go/packages#Config.
@@ -144,15 +154,15 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
 		for _, f := range pkg.Syntax {
 			name := pkg.Fset.File(f.Pos()).Name()
 			if strings.HasSuffix(name, ".go") {
-				allPkgFiles[id] = append(allPkgFiles[id], name)
+				goPkgFiles[id] = append(goPkgFiles[id], name)
 			}
 		}
 	}
 
 	// Test files don't have Module set. Filter out packages in skipped modules.
 	pkgFiles := map[string][]string{}
 	pkgNames := []string{}
-	for pkgPath, files := range allPkgFiles {
+	for pkgPath, files := range goPkgFiles {
 		skip := false
 		for skipped := range skippedModules {
 			if strings.HasPrefix(pkgPath, skipped) {
@@ -175,26 +185,41 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
 		for _, f := range pkgFiles[pkgPath] {
 			pf, err := parser.ParseFile(fset, f, nil, parser.ParseComments)
 			if err != nil {
-				return nil, nil, nil, fmt.Errorf("ParseFile: %v", err)
+				return nil, fmt.Errorf("ParseFile: %v", err)
 			}
 			parsedFiles = append(parsedFiles, pf)
 		}
 
 		// Parse out GoDoc.
 		docPkg, err := doc.NewFromFiles(fset, parsedFiles, pkgPath)
 		if err != nil {
-			return nil, nil, nil, fmt.Errorf("doc.NewFromFiles: %v", err)
+			return nil, fmt.Errorf("doc.NewFromFiles: %v", err)
 		}
 
 		// Extra filter in case the file filtering didn't catch everything.
 		if !strings.HasPrefix(docPkg.ImportPath, module.Path) {
 			continue
 		}
 
-		toc = append(toc, &tocItem{
+		pkgTOCITem := &tocItem{
 			UID:  docPkg.ImportPath,
 			Name: docPkg.ImportPath,
-		})
+		}
+		toc = append(toc, pkgTOCITem)
+
+		// If the package path == module path, add the extra files to the TOC
+		// as a child of the module==pkg item.
+		if pkgPath == module.Path {
+			for _, path := range extraFiles {
+				base := filepath.Base(path)
+				name := strings.TrimSuffix(base, filepath.Ext(base))
+				name = strings.Title(name)
+				pkgTOCITem.addItem(&tocItem{
+					Href: path,
+					Name: name,
+				})
+			}
+		}
 
 		pkgItem := &item{
 			UID:      docPkg.ImportPath,
@@ -345,7 +370,12 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
 		sort.Strings(skipped)
 		log.Printf("Warning: Only processed %s@%s, skipped:\n%s\n", module.Path, module.Version, strings.Join(skipped, "\n"))
 	}
-	return pages, toc, module, nil
+
+	return &result{
+		pages:  pages,
+		toc:    toc,
+		module: module,
+	}, nil
 }
 
 // processExamples converts the examples to []example.

internal/godocfx/testdata/golden/README.md

@@ -0,0 +1,32 @@
+## Cloud Storage [![GoDoc](https://godoc.org/cloud.google.com/go/storage?status.svg)](https://godoc.org/cloud.google.com/go/storage)
+
+- [About Cloud Storage](https://cloud.google.com/storage/)
+- [API documentation](https://cloud.google.com/storage/docs)
+- [Go client documentation](https://godoc.org/cloud.google.com/go/storage)
+- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/storage)
+
+### Example Usage
+
+First create a `storage.Client` to use throughout your application:
+
+[snip]:# (storage-1)
+```go
+client, err := storage.NewClient(ctx)
+if err != nil {
+	log.Fatal(err)
+}
+```
+
+[snip]:# (storage-2)
+```go
+// Read the object1 from bucket.
+rc, err := client.Bucket("bucket").Object("object1").NewReader(ctx)
+if err != nil {
+	log.Fatal(err)
+}
+defer rc.Close()
+body, err := ioutil.ReadAll(rc)
+if err != nil {
+	log.Fatal(err)
+}
+```

internal/godocfx/testdata/golden/toc.yml

@@ -1,3 +1,6 @@
 ### YamlMime:TableOfContent
 - uid: cloud.google.com/go/storage
   name: cloud.google.com/go/storage
+  items:
+  - name: README
+    href: README.md