kola: Support loading tests from /usr/lib/coreos-assembler/tests/kola

Makefile

@@ -72,4 +72,5 @@ install:
 	ln -sf ../lib/coreos-assembler/coreos-assembler $(DESTDIR)$(PREFIX)/bin/
 	ln -sf ../lib/coreos-assembler/cp-reflink $(DESTDIR)$(PREFIX)/bin/
 	ln -sf coreos-assembler $(DESTDIR)$(PREFIX)/bin/cosa
+	install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/tests/kola
 	cd mantle && $(MAKE) install DESTDIR=$(DESTDIR)

mantle/cmd/kola/kola.go

@@ -172,6 +172,19 @@ func preRun(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func registerExternals() error {
+	if err := kola.RegisterExternalTestsWithPrefix("/usr/lib/coreos-assembler", "ext"); err != nil {
+		return err
+	}
+	for _, d := range runExternals {
+		err := kola.RegisterExternalTests(d)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func runRun(cmd *cobra.Command, args []string) error {
 	var patterns []string
 	if len(args) == 0 {
@@ -186,11 +199,8 @@ func runRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	for _, d := range runExternals {
-		err := kola.RegisterExternalTests(d)
-		if err != nil {
-			return err
-		}
+	if err := registerExternals(); err != nil {
+		return err
 	}
 
 	runErr := kola.RunTests(patterns, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
@@ -327,11 +337,8 @@ func writeProps() error {
 }
 
 func runList(cmd *cobra.Command, args []string) error {
-	for _, d := range runExternals {
-		err := kola.RegisterExternalTests(d)
-		if err != nil {
-			return err
-		}
+	if err := registerExternals(); err != nil {
+		return err
 	}
 	var testlist []*item
 	for name, test := range register.Tests {

mantle/kola/README-kola-ext.md

@@ -13,14 +13,16 @@ repositories, and allow these projects to target Fedora CoreOS
 in e.g. their own CI.  And we also want to run unmodified
 upstream tests, *without rebuilding* the project.
 
-Using kola run -E/--exttest
+Using kola run with externally defined tests
 ---
 
 The `--exttest` (`-E`) argument to `kola run` one way to accomplish this; you
 provide the path to an upstream project git repository.  Tests will be found
 in the `tests/kola` directory.  If this project contains binaries that require
 building, it is assumed that `make` (or equivalent) has already been invoked.
 
+In addition to using `-E`, you may also copy tests to `/usr/lib/coreos-assembler/tests/kola`.
+
 The `tests/kola` directory will be traversed recursively to find tests.
 
 The core idea is to express a test as a single binary (plus an optional

mantle/kola/harness.go

@@ -60,6 +60,12 @@ import (
 	"github.com/coreos/mantle/util"
 )
 
+// InstalledTestsDir is a directory where "installed" external
+// can be placed; for example, a project like ostree can install
+// tests at /usr/lib/coreos-assembler/tests/kola/ostree/...
+// and this will be automatically picked up.
+const InstalledTestsDir = "/usr/lib/coreos-assembler/tests/kola"
+
 var (
 	plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "kola")
 
@@ -647,6 +653,20 @@ func registerTestDir(dir, testprefix string, children []os.FileInfo) error {
 	return nil
 }
 
+func RegisterExternalTestsWithPrefix(dir, prefix string) error {
+	testsdir := filepath.Join(dir, "tests/kola")
+	children, err := ioutil.ReadDir(testsdir)
+	if err != nil {
+		return errors.Wrapf(err, "reading %s", dir)
+	}
+
+	if err := registerTestDir(testsdir, prefix, children); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // RegisterExternalTests iterates over a directory, and finds subdirectories
 // that have exactly one executable binary.
 func RegisterExternalTests(dir string) error {
@@ -658,17 +678,7 @@ func RegisterExternalTests(dir string) error {
 	}
 	basename := fmt.Sprintf("ext.%s", filepath.Base(realdir))
 
-	testsdir := filepath.Join(dir, "tests/kola")
-	children, err := ioutil.ReadDir(testsdir)
-	if err != nil {
-		return errors.Wrapf(err, "reading %s", dir)
-	}
-
-	if err := registerTestDir(testsdir, basename, children); err != nil {
-		return err
-	}
-
-	return nil
+	return RegisterExternalTestsWithPrefix(dir, basename)
 }
 
 // getClusterSemVer returns the CoreOS semantic version via starting a