kola: Add run-ext-bin command

mantle/cmd/kola/kola.go

@@ -84,6 +84,22 @@ will be ignored.
 		SilenceUsage: true,
 	}
 
+	cmdRunExtBin = &cobra.Command{
+		Use:   "run-ext-bin",
+		Short: "Run an external test (single binary)",
+		Long: `Run an externally defined test
+
+This injects a single binary into the target system and executes it as a systemd
+unit.  The test is considered successful when the service exits normally, and
+failed if the test exits non-zero - but the service being killed by e.g. SIGTERM
+is ignored.  This is intended to allow rebooting the system.
+`,
+
+		Args:    cobra.ExactArgs(1),
+		PreRunE: preRun,
+		RunE:    runRunExtBin,
+	}
+
 	cmdHttpServer = &cobra.Command{
 		Use:   "http-server",
 		Short: "Run a static webserver",
@@ -122,10 +138,14 @@ This can be useful for e.g. serving locally built OSTree repos to qemu.
 	findParentImage    bool
 	qemuImageDir       string
 	qemuImageDirIsTemp bool
+
+	extDependencyDir string
 )
 
 func init() {
 	root.AddCommand(cmdRun)
+	root.AddCommand(cmdRunExtBin)
+	cmdRunExtBin.Flags().StringVar(&extDependencyDir, "depdir", "", "Copy (rsync) dir to target, available as ${KOLA_EXT_DATA}")
 
 	root.AddCommand(cmdList)
 	cmdList.Flags().BoolVar(&listJSON, "json", false, "format output in JSON")
@@ -190,6 +210,22 @@ func runRun(cmd *cobra.Command, args []string) error {
 	return runErr
 }
 
+func runRunExtBin(cmd *cobra.Command, args []string) error {
+	extbin := args[0]
+
+	outputDir, err := kola.SetupOutputDir(outputDir, kolaPlatform)
+	if err != nil {
+		return err
+	}
+
+	runErr := kola.RunExtBin(kolaPlatform, outputDir, extbin, extDependencyDir)
+	if err := writeProps(); err != nil {
+		return errors.Wrapf(err, "writing properties")
+	}
+
+	return runErr
+}
+
 func writeProps() error {
 	f, err := os.OpenFile(filepath.Join(outputDir, "properties.json"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
 	if err != nil {

mantle/cmd/kolet/kolet.go

@@ -15,9 +15,14 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"strings"
+	"time"
 
+	systemddbus "github.com/coreos/go-systemd/v22/dbus"
 	"github.com/coreos/pkg/capnslog"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
 	"github.com/coreos/mantle/cli"
@@ -27,6 +32,12 @@ import (
 	_ "github.com/coreos/mantle/kola/registry"
 )
 
+const (
+	// From /usr/include/bits/siginfo-consts.h
+	CLD_EXITED int32 = 1
+	CLD_KILLED int32 = 2
+)
+
 var (
 	plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "kolet")
 
@@ -41,6 +52,13 @@ var (
 		Short: "Run a given test's native function",
 		Run:   run,
 	}
+
+	cmdRunExtUnit = &cobra.Command{
+		Use:          "run-test-unit [unitname]",
+		Short:        "Monitor execution of a systemd unit",
+		RunE:         runExtUnit,
+		SilenceUsage: true,
+	}
 )
 
 func run(cmd *cobra.Command, args []string) {
@@ -80,10 +98,110 @@ func registerTestMap(m map[string]*register.Test) {
 	}
 }
 
+// dispatchRunExtUnit returns true if unit completed successfully, false if
+// it's still running (or unit was terminated by SIGTERM)
+func dispatchRunExtUnit(unitname string, sdconn *systemddbus.Conn) (bool, error) {
+	props, err := sdconn.GetAllProperties(unitname)
+	if err != nil {
+		return false, errors.Wrapf(err, "listing unit properties")
+	}
+
+	result := props["Result"]
+	if result == "exit-code" {
+		return false, fmt.Errorf("Unit %s exited with code %d", unitname, props["ExecMainStatus"])
+	}
+
+	state := props["ActiveState"]
+	substate := props["SubState"]
+
+	switch state {
+	case "inactive":
+		fmt.Printf("Starting %s\n", unitname)
+		sdconn.StartUnit(unitname, "fail", nil)
+		return false, nil
+	case "activating":
+		return false, nil
+	case "active":
+		{
+			switch substate {
+			case "exited":
+				maincode := props["ExecMainCode"]
+				mainstatus := props["ExecMainStatus"]
+				switch maincode {
+				case CLD_EXITED:
+					if mainstatus == int32(0) {
+						return true, nil
+					} else {
+						// I don't think this can happen, we'd have exit-code above; but just
+						// for completeness
+						return false, fmt.Errorf("Unit %s failed with code %d", unitname, mainstatus)
+					}
+				case CLD_KILLED:
+					// SIGTERM; we explicitly allow that, expecting we're rebooting.
+					if mainstatus == 15 {
+						fmt.Printf("Unit %s terminated via SIGTERM, assuming reboot\n", unitname)
+						return false, nil
+					} else {
+						return true, fmt.Errorf("Unit %s killed by signal %d", unitname, mainstatus)
+					}
+				default:
+					return false, fmt.Errorf("Unit %s had unhandled code %d", unitname, maincode)
+				}
+			case "running":
+				return false, nil
+			case "failed":
+				return true, fmt.Errorf("Unit %s in substate 'failed'", unitname)
+			default:
+				// Pass through other states
+				return false, nil
+			}
+		}
+	default:
+		return false, fmt.Errorf("Unhandled systemd unit state %s", state)
+	}
+}
+
+func runExtUnit(cmd *cobra.Command, args []string) error {
+	unitname := args[0]
+	// Restrict this to services, don't need to support anything else right now
+	if !strings.HasSuffix(unitname, ".service") {
+		unitname = unitname + ".service"
+	}
+	sdconn, err := systemddbus.NewSystemConnection()
+	if err != nil {
+		return errors.Wrapf(err, "systemd connection")
+	}
+	if err := sdconn.Subscribe(); err != nil {
+		return err
+	}
+	dispatchRunExtUnit(unitname, sdconn)
+	unitevents, uniterrs := sdconn.SubscribeUnits(time.Second)
+
+	for {
+		select {
+		case m := <-unitevents:
+			for n := range m {
+				if n == unitname {
+					r, err := dispatchRunExtUnit(unitname, sdconn)
+					if err != nil {
+						return err
+					}
+					if r {
+						return nil
+					}
+				}
+			}
+		case m := <-uniterrs:
+			return m
+		}
+	}
+}
+
 func main() {
 	registerTestMap(register.Tests)
 	registerTestMap(register.UpgradeTests)
 	root.AddCommand(cmdRun)
+	root.AddCommand(cmdRunExtUnit)
 
 	cli.Execute(root)
 }

mantle/go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/coreos/container-linux-config-transpiler v0.8.0
 	github.com/coreos/coreos-cloudinit v1.11.0
 	github.com/coreos/go-semver v0.3.0
+	github.com/coreos/go-systemd/v22 v22.0.0
 	github.com/coreos/ign-converter v0.0.0-20200228175238-237c8512310a
 	github.com/coreos/ignition v0.35.0
 	github.com/coreos/ignition/v2 v2.1.1

mantle/go.sum

@@ -50,6 +50,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK
 github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/ign-converter v0.0.0-20200228175238-237c8512310a h1:QaJLlJP5GAWdqMoijWSS7ZqNma+Berrs7NuVi2olAiU=
 github.com/coreos/ign-converter v0.0.0-20200228175238-237c8512310a/go.mod h1:eOYp0mzbRAnWFWZ64q8IeWWHFTQu9y2lxndiuVVGVkE=
 github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ=
@@ -88,6 +90,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384 h1:xNwo3yd3PZYRDAr/Dz0sBfDWY6El2xPCKJrwJVfMFjY=
 github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
+github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

mantle/kola/README-kola-ext.md

@@ -0,0 +1,40 @@
+Integrating external test suites
+===
+
+Fedora CoreOS is comprised of a number of upstream projects, from
+the Linux kernel to systemd, ostree, Ignition, podman and
+numerous others.
+
+We want to support keeping tests in their respective upstream
+repositories, and allow these projects to target Fedora CoreOS
+in e.g. their own CI.  And we also want to run unmodified
+upstream tests, *without rebuilding* the project.
+
+Using kola run-ext-bin
+===
+
+The `kola run-ext-bin` is one way to accomplish this.  The
+core idea is to express your test suite as a binary (plus an optional
+directory of dependencies) that will be uploaded to a CoreOS
+system and run as a systemd unit.
+
+Currently this systemd unit runs with full privileges - tests
+are assumed to be (potentially) destructive and a general assumption
+is tests are run in easily disposable virtual machines.
+
+A best practice for doing this is to write your tests in a language
+that compiles to a single binary - Rust and Go for example, but
+there exist for example tools like [PyInstaller](https://realpython.com/pyinstaller-python/#pyinstaller)
+too.
+
+This mechanism is suitable for testing most userspace components
+of CoreOS; for example, one can have the binary drive a container runtime.
+
+An important feature of `run-ext-bin` is support for rebooting the host system.
+This allows one to easily test OS updates.  To do this, simply invoke the usual
+`reboot` - the test framework will monitor the target systemd unit
+and ignore the case where it exits with `SIGTERM`.
+
+A test is considered failed if the unit exits with any non-zero exit
+status or dies from any signal other than `SIGTERM`.
+