Add support for go modules/plugins.

As we call the similar C binary a module I went with that name for the
Go version as well. There's some trickery needed, Go does currently not
support nocgo for modules, as noted in
golang/go#30986
so that functionality has to be disabled. Also this triggers the CC
environment variable problem again so I had to change the test CC value
used.

docs/arguments/destdir.md

@@ -4,7 +4,8 @@
 
 Normally PROG and MODULE installs in predefined target directories, but in some
 cases you might need to override this, for example for perl modules or to
-install to libexec/ instead of bin/.
+install to libexec/ instead of bin/. This argument works for GOPROG and
+GOMODULE as well.
 
 There are some special values for the first step in `destdir`. You write these
 like a normal directory path but they will be translated by Sebuild into other

docs/descriptors/gomodule.md

@@ -0,0 +1,20 @@
+# Go Dynamic Modules - GOMODULE
+
+    GOMODULE(foo
+    )
+
+Similar to [GOPROG](goprog.md) this uses go build to compile the directory into
+a binary. Unlike goprog, this descriptor creates a dynamic loadable module
+instead of a runnable progam. By default the module is created in the
+`modules/` [destination directory](../arguments/destdir.md) and will have a
+`.so` file suffix.
+
+In the Go echosystem, these are known as plugins and you use the standard
+[plugin](https://godoc.org/plugin) package to load them.
+
+GOMODULE support most the same arguments as GOPROG does, so refer to
+[GOPROG arguments](goprog.md#arguments) for descriptions of additional
+arguments. Special conditions and ninja variables are described on that
+page as well. Since Go doesn't support compiling these modules without
+cgo the `nocgo` argument is not available. Other settings disabling cgo
+are also ignored.

docs/index.md

@@ -51,6 +51,7 @@ directory.
 * [Go Program - GOPROG](descriptors/goprog.md)
 * [Go Tests - GOTEST](descriptors/gotest.md)
 * [Dynamic Modules - MODULE](descriptors/module.md)
+* [Go Dynamic Modules - GOMODULE](descriptors/gomodule.md)
 * [Scripts, Configuration and Other Files - INSTALL](descriptors/install.md)
 
 The next set is for generating intermediate libraries and tool files in the

internal/tools/gobuild.sh

@@ -125,6 +125,9 @@ case "$mode" in
 	""|prog)
 		go build $GOBUILD_FLAGS -o "$out" "${EXTLDFLAGS[@]}" $PKG || exit 1
 	;;
+	module)
+		go build $GOBUILD_FLAGS -o "$out" -buildmode=plugin "${EXTLDFLAGS[@]}" $PKG || exit 1
+	;;
 	*)
 		# go build links an executable to extract the symbols. If this is a plugin there'll be
 		# unresolved symbols. Ignore now, handle in final link.

pkg/buildbuild/descriptor.go

@@ -67,6 +67,7 @@ type Analyser struct {
 var DefaultDescriptors = map[string]Descriptor{
 	"PROG":          &ProgTemplate,
 	"GOPROG":        &GoprogTemplate,
+	"GOMODULE":      &GomoduleTemplate,
 	"GOTEST":        &GotestTemplate,
 	"TOOL_PROG":     &ToolProgTemplate,
 	"TOOL_INSTALL":  &ToolInstallTemplate,

pkg/buildbuild/go.go

@@ -2,11 +2,15 @@
 
 package buildbuild
 
-import "strings"
+import (
+	"fmt"
+	"strings"
+)
 
 type GoProgDesc struct {
 	LinkDesc
 	Pkg    string
+	Mode   string
 	NoCgo  bool
 	GOOS   string
 	GOARCH string
@@ -21,6 +25,7 @@ type GoTestDesc struct {
 func (tmpl *GoProgDesc) NewFromTemplate(bd, tname string, flavors []string) Descriptor {
 	return &GoProgDesc{
 		LinkDesc: *tmpl.LinkDesc.NewFromTemplate(bd, tname, flavors),
+		Mode:     tmpl.Mode,
 	}
 }
 
@@ -31,7 +36,12 @@ func (tmpl *GoTestDesc) NewFromTemplate(bd, tname string, flavors []string) Desc
 }
 
 func (g *GoProgDesc) Parse(ops *GlobalOps, realsrcdir string, args map[string][]string) Descriptor {
-	desc := g.GenericParse(g, ops, realsrcdir, args, LinkerExtra("gopkg", "nocgo", "goos", "goarch"))
+	lextra := LinkerExtra("gopkg", "nocgo", "goos", "goarch")
+	// Go plugins currently does not support cgo disabled.
+	if g.Mode == "module" {
+		lextra = LinkerExtra("gopkg", "goos", "goarch")
+	}
+	desc := g.GenericParse(g, ops, realsrcdir, args, lextra)
 	g.LinkerParse(realsrcdir, args)
 	g.Pkg = strings.Join(args["gopkg"], " ")
 	g.NoCgo = args["nocgo"] != nil
@@ -58,9 +68,9 @@ func (g *GoProgDesc) Finalize(ops *GlobalOps) {
 		eas = append(eas, "gopkg="+g.Pkg)
 	}
 	if g.NoCgo {
-		eas = append(eas, "gomode=prog-nocgo")
+		eas = append(eas, fmt.Sprintf("gomode=%s-nocgo", g.Mode))
 	} else {
-		eas = append(eas, "gomode=prog")
+		eas = append(eas, "gomode="+g.Mode)
 	}
 	if g.GOOS != "" {
 		eas = append(eas, "goos="+g.GOOS)
@@ -69,7 +79,13 @@ func (g *GoProgDesc) Finalize(ops *GlobalOps) {
 		eas = append(eas, "goarch="+g.GOARCH)
 	}
 
-	target := g.AddTarget(g.TargetName, "gobuild", []string{g.Srcdir}, g.Destdir, "", eas, g.TargetOptions)
+	tname := g.TargetName
+	if g.Mode == "module" {
+		tname += ".so"
+		// Buildmode plugin currently does not support cgo disabled.
+		eas = append(eas, "cgo_enabled=1")
+	}
+	target := g.AddTarget(tname, "gobuild", []string{g.Srcdir}, g.Destdir, "", eas, g.TargetOptions)
 	AddGodeps(target, ops)
 	g.GeneralDesc.Finalize(ops)
 }
@@ -124,6 +140,7 @@ func (g *GoTestDesc) Finalize(ops *GlobalOps) {
 }
 
 var GoprogTemplate = GoProgDesc{
+	Mode: "prog",
 	LinkDesc: LinkDesc{
 		GeneralDesc: GeneralDesc{
 			Destdir:       "dest_bin",
@@ -132,6 +149,16 @@ var GoprogTemplate = GoProgDesc{
 	},
 }
 
+var GomoduleTemplate = GoProgDesc{
+	Mode: "module",
+	LinkDesc: LinkDesc{
+		GeneralDesc: GeneralDesc{
+			Destdir:       "dest_mod",
+			TargetOptions: map[string]bool{"all": true, "incdeps": true, "libdeps": true},
+		},
+	},
+}
+
 var GotestTemplate = GoTestDesc{
 	LinkDesc: LinkDesc{
 		GeneralDesc: GeneralDesc{

rules/rules.ninja

@@ -132,7 +132,7 @@ gobuild_tool=GOPATH="$gopath" GOBUILD_FLAGS=$gobuild_flags GOBUILD_TEST_FLAGS=$g
 rule gobuild
     command = GOOS="$goos" GOARCH="$goarch" $gobuild_tool "$gopkg" "$in" "$out" "$objdir/depfile-$gomode" "-I $incdir $includes $platform_includes" "-L $libdir -L $scmcoord_contrib/lib $ldlibs" $gomode "$builddir"
     depfile = $objdir/depfile-$gomode
-    description = building go program $out from $in
+    description = building go $gomode $out from $in
 
 rule gobuildlib
     command = GOOS="$goos" GOARCH="$goarch" $gobuild_tool "$gopkg" "$in" "$out" "$depfile" "$picflag -I $incdir $includes $platform_includes" "-L $libdir -L $scmcoord_contrib/lib $ldlibs" $gomode "$builddir"

test/Builddesc

@@ -3,6 +3,8 @@
 COMPONENT([
 	goarch
 	go/src/gopath
+	gomodules/loader
+	gomodules/module
 	collect_special
 	srcdir
 	c

test/compile.sh

@@ -4,7 +4,7 @@
 set -xe
 test -z "$BUILDPATH" && BUILDPATH=build
 
-CC='env cc' seb -condition cfoo -condition cbar
+CC="cc -std=gnu11" seb -condition cfoo -condition cbar
 touch Builddesc # to make ninja invoke seb.
 ninja -f $BUILDPATH/build.ninja
 
@@ -21,6 +21,9 @@ grep -q rt0_386_darwin $BUILDPATH/regress/bin/goarch
 # Check prod flavor is executable.
 $BUILDPATH/prod/bin/goarch
 
+# Load the built go module.
+$BUILDPATH/regress/bin/loader $BUILDPATH/regress/modules/gomod.so
+
 ninja -f $BUILDPATH/build.ninja $BUILDPATH/regress/gotest/gopath
 ninja -f $BUILDPATH/build.ninja $BUILDPATH/regress/gocover/gopath-coverage.html
 env GOPATH=$PWD/go $BUILDPATH/regress/gotest/gopath.test
@@ -40,4 +43,4 @@ if ! (ninja -f $BUILDPATH/build.ninja $BUILDPATH/regress/gocover/gopath-coverage
 test -f $BUILDPATH/obj/regress/lib/enabled
 ! test -f $BUILDPATH/obj/regress/lib/disabled
 
-[ -n "$NODEPTEST" ] || CC='env cc' BUILDBUILD_ARGS="-condition cfoo -condition cbar" ../contrib/helpers/dep-tester.sh
+[ -n "$NODEPTEST" ] || CC="cc -std=gnu11" BUILDBUILD_ARGS="-condition cfoo -condition cbar" ../contrib/helpers/dep-tester.sh

test/gomodules/loader/Builddesc

@@ -0,0 +1,2 @@
+GOPROG(loader
+)

test/gomodules/loader/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"plugin"
+)
+
+func main() {
+	for _, pstr := range os.Args[1:] {
+		p, err := plugin.Open(pstr)
+		if err != nil {
+			log.Fatal(err)
+		}
+		test, err := p.Lookup("Test")
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(test.(func() string)())
+	}
+}

test/gomodules/module/Builddesc

@@ -0,0 +1,2 @@
+GOMODULE(gomod
+)

test/gomodules/module/module.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"context"
+	"log"
+	"net"
+	"strings"
+)
+
+func Test() string {
+	// Use the net package to try to explicitly use cgo
+	addrs, err := net.DefaultResolver.LookupHost(context.Background(), "localhost")
+	if err != nil {
+		log.Fatal(err)
+	}
+	return strings.Join(addrs, ", ")
+}

test/rules.ninja

@@ -1,7 +1,7 @@
 # Copyright 2018 Schibsted
 
 rule persist_cc
-    command = test "$cc" = "env cc" && touch $out
+    command = test "$cc" = "cc -std=gnu11" && touch $out
 
 rule test_zlib_config
     command = test -n "$zlib_config" && touch $out