go/tools: add gopackagesdriver

go/private/actions/stdlib.bzl

@@ -51,9 +51,24 @@ def _should_use_sdk_stdlib(go):
             not go.mode.pure and
             go.mode.link == LINKMODE_NORMAL)
 
+def _build_stdlib_list_json(go):
+    out = go.declare_file(go, "stdlib.pkg.json")
+    args = go.builder_args(go, "stdliblist")
+    args.add("-out", out)
+    go.actions.run(
+        inputs = go.sdk_files,
+        outputs = [out],
+        mnemonic = "GoStdlibList",
+        executable = go.toolchain._builder,
+        arguments = [args],
+        env = go.env,
+    )
+    return out
+
 def _sdk_stdlib(go):
     return GoStdLib(
         root_file = go.sdk.root_file,
+        list_json = _build_stdlib_list_json(go),
         libs = go.sdk.libs,
     )
 
@@ -100,5 +115,6 @@ def _build_stdlib(go):
     )
     return GoStdLib(
         root_file = root_file,
+        list_json = _build_stdlib_list_json(go),
         libs = [pkg],
     )

go/tools/builders/BUILD.bazel

@@ -35,6 +35,7 @@ filegroup(
         "read.go",
         "replicate.go",
         "stdlib.go",
+        "stdliblist.go",
     ] + select({
         "@bazel_tools//src/conditions:windows": ["path_windows.go"],
         "//conditions:default": ["path.go"],

go/tools/builders/builder.go

@@ -59,6 +59,8 @@ func main() {
 		action = pack
 	case "stdlib":
 		action = stdlib
+	case "stdliblist":
+		action = stdliblist
 	default:
 		log.Fatalf("unknown action: %s", verb)
 	}

go/tools/builders/stdliblist.go

@@ -0,0 +1,209 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"go/build"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Copy and pasted from golang.org/x/tools/go/packages
+type flatPackagesError struct {
+	Pos  string // "file:line:col" or "file:line" or "" or "-"
+	Msg  string
+	Kind flatPackagesErrorKind
+}
+
+type flatPackagesErrorKind int
+
+const (
+	UnknownError flatPackagesErrorKind = iota
+	ListError
+	ParseError
+	TypeError
+)
+
+func (err flatPackagesError) Error() string {
+	pos := err.Pos
+	if pos == "" {
+		pos = "-" // like token.Position{}.String()
+	}
+	return pos + ": " + err.Msg
+}
+
+// flatPackage is the JSON form of Package
+// It drops all the type and syntax fields, and transforms the Imports
+type flatPackage struct {
+	ID              string
+	Name            string              `json:",omitempty"`
+	PkgPath         string              `json:",omitempty"`
+	Standard        bool                `json:",omitempty"`
+	Errors          []flatPackagesError `json:",omitempty"`
+	GoFiles         []string            `json:",omitempty"`
+	CompiledGoFiles []string            `json:",omitempty"`
+	OtherFiles      []string            `json:",omitempty"`
+	ExportFile      string              `json:",omitempty"`
+	Imports         map[string]string   `json:",omitempty"`
+}
+
+type goListPackage struct {
+	Dir        string // directory containing package sources
+	ImportPath string // import path of package in dir
+	Name       string // package name
+	Target     string // install path
+	Goroot     bool   // is this package in the Go root?
+	Standard   bool   // is this package part of the standard Go library?
+	Root       string // Go root or Go path dir containing this package
+	Export     string // file containing export data (when using -export)
+	// Source files
+	GoFiles           []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
+	CgoFiles          []string // .go source files that import "C"
+	CompiledGoFiles   []string // .go files presented to compiler (when using -compiled)
+	IgnoredGoFiles    []string // .go source files ignored due to build constraints
+	IgnoredOtherFiles []string // non-.go source files ignored due to build constraints
+	CFiles            []string // .c source files
+	CXXFiles          []string // .cc, .cxx and .cpp source files
+	MFiles            []string // .m source files
+	HFiles            []string // .h, .hh, .hpp and .hxx source files
+	FFiles            []string // .f, .F, .for and .f90 Fortran source files
+	SFiles            []string // .s source files
+	SwigFiles         []string // .swig files
+	SwigCXXFiles      []string // .swigcxx files
+	SysoFiles         []string // .syso object files to add to archive
+	TestGoFiles       []string // _test.go files in package
+	XTestGoFiles      []string // _test.go files outside package
+	// Embedded files
+	EmbedPatterns      []string // //go:embed patterns
+	EmbedFiles         []string // files matched by EmbedPatterns
+	TestEmbedPatterns  []string // //go:embed patterns in TestGoFiles
+	TestEmbedFiles     []string // files matched by TestEmbedPatterns
+	XTestEmbedPatterns []string // //go:embed patterns in XTestGoFiles
+	XTestEmbedFiles    []string // files matched by XTestEmbedPatterns
+	// Dependency information
+	Imports   []string          // import paths used by this package
+	ImportMap map[string]string // map from source import to ImportPath (identity entries omitted)
+	// Error information
+	Incomplete bool                 // this package or a dependency has an error
+	Error      *flatPackagesError   // error loading package
+	DepsErrors []*flatPackagesError // errors loading dependencies
+}
+
+func stdlibPackageID(importPath string) string {
+	return "@io_bazel_rules_go//stdlib:" + importPath
+}
+
+func execRootPath(execRoot, p string) string {
+	dir, _ := filepath.Rel(execRoot, p)
+	return filepath.Join("__BAZEL_EXECROOT__", dir)
+}
+
+func absoluteSourcesPaths(execRoot, pkgDir string, srcs []string) []string {
+	ret := []string{}
+	pkgDir = execRootPath(execRoot, pkgDir)
+	for _, src := range srcs {
+		ret = append(ret, filepath.Join(pkgDir, src))
+	}
+	return ret
+}
+
+func packageToPackage(execRoot string, pkg *goListPackage) *flatPackage {
+	// Don't use generated files from the stdlib
+	goFiles := absoluteSourcesPaths(execRoot, pkg.Dir, pkg.GoFiles)
+
+	newPkg := &flatPackage{
+		ID:              stdlibPackageID(pkg.ImportPath),
+		Name:            pkg.Name,
+		PkgPath:         pkg.ImportPath,
+		ExportFile:      execRootPath(execRoot, pkg.Target),
+		Imports:         map[string]string{},
+		Standard:        pkg.Standard,
+		GoFiles:         goFiles,
+		CompiledGoFiles: goFiles,
+	}
+	for _, imp := range pkg.Imports {
+		newPkg.Imports[imp] = stdlibPackageID(imp)
+	}
+	// We don't support CGo for now
+	delete(newPkg.Imports, "C")
+	return newPkg
+}
+
+// stdlib builds the standard library in the appropriate mode into a new goroot.
+func stdliblist(args []string) error {
+	// process the args
+	flags := flag.NewFlagSet("stdliblist", flag.ExitOnError)
+	goenv := envFlags(flags)
+	out := flags.String("out", "", "Path to output go list json")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	if err := goenv.checkFlags(); err != nil {
+		return err
+	}
+
+	// Ensure paths are absolute.
+	absPaths := []string{}
+	for _, path := range filepath.SplitList(os.Getenv("PATH")) {
+		absPaths = append(absPaths, abs(path))
+	}
+	os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator)))
+	os.Setenv("GOROOT", abs(os.Getenv("GOROOT")))
+	// Make sure we have an absolute path to the C compiler.
+	// TODO(#1357): also take absolute paths of includes and other paths in flags.
+	os.Setenv("CC", abs(os.Getenv("CC")))
+
+	execRoot := abs(".")
+
+	cachePath := abs(*out + ".gocache")
+	defer os.RemoveAll(cachePath)
+	os.Setenv("GOCACHE", cachePath)
+	os.Setenv("GOPATH", cachePath)
+
+	listArgs := goenv.goCmd("list")
+	if len(build.Default.BuildTags) > 0 {
+		listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, " "))
+	}
+	listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo")
+
+	jsonFile, err := os.Create(*out)
+	if err != nil {
+		return err
+	}
+	defer jsonFile.Close()
+
+	jsonData := &bytes.Buffer{}
+	if err := goenv.runCommandToFile(jsonData, listArgs); err != nil {
+		return err
+	}
+
+	encoder := json.NewEncoder(jsonFile)
+	decoder := json.NewDecoder(jsonData)
+	for decoder.More() {
+		var pkg *goListPackage
+		if err := decoder.Decode(&pkg); err != nil {
+			return err
+		}
+		if err := encoder.Encode(packageToPackage(execRoot, pkg)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

go/tools/gopackagesdriver/BUILD.bazel

@@ -0,0 +1,22 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bazel.go",
+        "bazel_json_builder.go",
+        "driver_request.go",
+        "flatpackage.go",
+        "json_packages_driver.go",
+        "main.go",
+        "packageregistry.go",
+    ],
+    importpath = "github.com/bazelbuild/rules_go/go/tools/gopackagesdriver",
+    visibility = ["//visibility:private"],
+)
+
+go_binary(
+    name = "gopackagesdriver",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

go/tools/gopackagesdriver/aspect.bzl

@@ -0,0 +1,116 @@
+load(
+    "//go/private:providers.bzl",
+    "GoArchive",
+    "GoStdLib",
+)
+load(
+    "//go/private:context.bzl",
+    "go_context",
+)
+load(
+    "@bazel_skylib//lib:paths.bzl",
+    "paths",
+)
+load(
+    "@bazel_skylib//lib:collections.bzl",
+    "collections",
+)
+
+GoPkgInfo = provider()
+
+def _is_file_external(f):
+    return f.owner.workspace_root != ""
+
+def _file_path(f):
+    if f.is_source and not _is_file_external(f):
+        return paths.join("__BAZEL_WORKSPACE__", f.path)
+    return paths.join("__BAZEL_EXECROOT__", f.path)
+
+def _go_pkg_info_aspect_impl(target, ctx):
+    # Fetch the stdlib JSON file from the inner most target
+    stdlib_json = None
+
+    deps_transitive_json = []
+    deps_transitive_x = []
+    if hasattr(ctx.rule.attr, "deps"):
+        for dep in ctx.rule.attr.deps:
+            if GoPkgInfo in dep:
+                pkg_info = dep[GoPkgInfo]
+                deps_transitive_json.append(pkg_info.transitive_json)
+                deps_transitive_x.append(pkg_info.transitive_x)
+                # Fetch the stdlib json from the first dependency
+                if not stdlib_json:
+                    stdlib_json = pkg_info.stdlib_json
+
+    # If deps are embedded, do not gather their json or x since they are
+    # included in the current target, but do gather their deps'.
+    if hasattr(ctx.rule.attr, "embed"):
+        for dep in ctx.rule.attr.embed:
+            if GoPkgInfo in dep:
+                pkg_info = dep[GoPkgInfo]
+                deps_transitive_json.append(pkg_info.deps_transitive_json)
+                deps_transitive_x.append(pkg_info.deps_transitive_x)
+
+    pkg_json = None
+    x = None
+    if GoArchive in target:
+        archive = target[GoArchive]
+        x = archive.data.export_file
+        pkg = struct(
+            ID = str(archive.data.label),
+            PkgPath = archive.data.importpath,
+            ExportFile = _file_path(archive.data.export_file),
+            GoFiles = [
+                _file_path(src)
+                for src in archive.data.orig_srcs
+            ],
+            CompiledGoFiles = [
+                _file_path(src)
+                for src in archive.data.srcs
+            ],
+        )
+        pkg_json = ctx.actions.declare_file(archive.data.name + ".pkg.json")
+        ctx.actions.write(pkg_json, content = pkg.to_json())
+        # If there was no stdlib json in any dependencies, fetch it from the
+        # current go_ node.
+        if not stdlib_json:
+            stdlib_json = ctx.attr._go_stdlib[GoStdLib].list_json
+
+    pkg_info = GoPkgInfo(
+        json = pkg_json,
+        stdlib_json = stdlib_json,
+        transitive_json = depset(
+            direct = [pkg_json] if pkg_json else [],
+            transitive = deps_transitive_json,
+        ),
+        deps_transitive_json = depset(
+            transitive = deps_transitive_json,
+        ),
+        x = x,
+        transitive_x = depset(
+            direct = [x] if x else None,
+            transitive = deps_transitive_x,
+        ),
+        deps_transitive_x = depset(
+            transitive = deps_transitive_x,
+        ),
+    )
+
+    return [
+        pkg_info,
+        OutputGroupInfo(
+            go_pkg_driver_json = pkg_info.transitive_json,
+            go_pkg_driver_x = pkg_info.transitive_x,
+            go_pkg_driver_stdlib_json = depset([pkg_info.stdlib_json] if pkg_info.stdlib_json else [])
+        ),
+    ]
+
+go_pkg_info_aspect = aspect(
+    implementation = _go_pkg_info_aspect_impl,
+    attr_aspects = ["embed", "deps"],
+    attrs = {
+        "_go_stdlib": attr.label(
+            default = "@io_bazel_rules_go//:stdlib",
+        ),
+    },
+)