Install CSI driver on openshift-test start.

cmd/openshift-tests/csi.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/openshift/origin/test/extended/csi"
+
+	"k8s.io/kubernetes/test/e2e/storage/external"
+)
+
+const (
+	manifestEnvVar       = "TEST_CSI_DRIVER_FILES"
+	installDriversEnvVar = "TEST_INSTALL_CSI_DRIVERS"
+)
+
+// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES.
+func initCSITests(dryRun bool) error {
+	driverList := os.Getenv(installDriversEnvVar)
+	if driverList != "" {
+		drivers := strings.Split(driverList, ",")
+		for _, driver := range drivers {
+			manifestFile, err := csi.InstallCSIDriver(driver, dryRun)
+			if err != nil {
+				return fmt.Errorf("failed to install CSI driver from %q: %s", driver, err)
+			}
+			// Children processes need to see the newly introduced manifest,
+			// store it in TEST_CSI_DRIVER_FILES env. var for them.
+			manifestList := os.Getenv(manifestEnvVar)
+			if len(manifestList) > 0 {
+				manifestList += ","
+			}
+			manifestList += manifestFile
+			os.Setenv(manifestEnvVar, manifestList)
+		}
+	}
+
+	// Clear TEST_INSTALL_CSI_DRIVERS, we don't want the driver installed by children too.
+	os.Setenv(installDriversEnvVar, "")
+
+	manifestList := os.Getenv(manifestEnvVar)
+	if manifestList != "" {
+		manifests := strings.Split(manifestList, ",")
+		for _, manifest := range manifests {
+			if err := external.AddDriverDefinition(manifest); err != nil {
+				return fmt.Errorf("failed to load manifest from %q: %s", manifest, err)
+			}
+		}
+	}
+
+	return nil
+}

cmd/openshift-tests/openshift-tests.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"math/rand"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/onsi/ginkgo"
@@ -22,7 +21,6 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 	reale2e "k8s.io/kubernetes/test/e2e"
 	e2e "k8s.io/kubernetes/test/e2e/framework"
-	"k8s.io/kubernetes/test/e2e/storage/external"
 
 	"github.com/openshift/library-go/pkg/serviceability"
 	"github.com/openshift/origin/pkg/monitor"
@@ -103,6 +101,7 @@ func newRunCommand() *cobra.Command {
 	opt := &testginkgo.Options{
 		Suites: staticSuites,
 	}
+
 	cmd := &cobra.Command{
 		Use:   "run SUITE",
 		Short: "Run a test suite",
@@ -208,6 +207,7 @@ func newRunTestCommand() *cobra.Command {
 		Out:    os.Stdout,
 		ErrOut: os.Stderr,
 	}
+
 	cmd := &cobra.Command{
 		Use:   "run-test NAME",
 		Short: "Run a single test by name",
@@ -283,7 +283,7 @@ func initProvider(provider string, dryRun bool) error {
 	exutil.TestContext.MaxNodesToGather = 0
 	reale2e.SetViperConfig(os.Getenv("VIPERCONFIG"))
 
-	if err := initCSITests(); err != nil {
+	if err := initCSITests(dryRun); err != nil {
 		return err
 	}
 
@@ -337,19 +337,3 @@ func decodeProviderTo(provider string, testContext *e2e.TestContextType) error {
 	klog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig)
 	return nil
 }
-
-// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES.
-func initCSITests() error {
-	// TODO: replace with cmdline argument
-	driverList := os.Getenv("TEST_CSI_DRIVER_FILES")
-	if driverList == "" {
-		return nil
-	}
-	drivers := strings.Split(driverList, ",")
-	for _, driver := range drivers {
-		if err := external.AddDriverDefinition(driver); err != nil {
-			return fmt.Errorf("failed to load driver from %q: %s", driver, err)
-		}
-	}
-	return nil
-}

test/extended/csi/install.go

@@ -0,0 +1,94 @@
+package csi
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/openshift/origin/test/extended/testdata"
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+const (
+	csiBasePath         = "test/extended/testdata/csi"
+	defaultImageFormat  = "registry.svc.ci.openshift.org/origin/4.3:${component}"
+	imageFormatVariable = "IMAGE_FORMAT"
+)
+
+// InstallCSIDriver installs a CSI driver and defines its tests.
+// It applies "test/extended/csi/<driverName>/install-template.yaml" and
+// returns path to test manifest "test/extended/csi/<driverName>/manifest.yaml"
+func InstallCSIDriver(driverName string, dryRun bool) (string, error) {
+	// The driver name comes from an user and we want a nice error message instead
+	// of panic in FixturePath().
+	templatePath := filepath.Join(csiBasePath, driverName, "install-template.yaml")
+	if _, err := testdata.AssetInfo(templatePath); err != nil {
+		return "", fmt.Errorf("failed to install CSI driver %q: %s", driverName, err)
+	}
+	manifestPath := filepath.Join(csiBasePath, driverName, "manifest.yaml")
+	if _, err := testdata.AssetInfo(manifestPath); err != nil {
+		return "", fmt.Errorf("failed to install CSI driver %q: %s", driverName, err)
+	}
+
+	// Convert to array and cut "test/extended" for FixturePath()
+	templateFixturePath := strings.Split(templatePath, string(os.PathSeparator))[2:]
+	yamlPath, err := executeTemplate(exutil.FixturePath(templateFixturePath...))
+	defer os.Remove(yamlPath)
+	if err != nil {
+		return "", err
+	}
+
+	if !dryRun {
+		// Install the driver
+		oc := exutil.NewCLIWithoutNamespace("csi-install")
+		if err := oc.Run("apply").Args("-f", yamlPath).Execute(); err != nil {
+			return "", fmt.Errorf("failed to apply %s: %s", yamlPath, err)
+		}
+	}
+
+	// Cut "test/extended" for FixturePath()
+	manifestFixturePath := strings.Split(manifestPath, string(os.PathSeparator))[2:]
+	return exutil.FixturePath(manifestFixturePath...), nil
+}
+
+// ListCSIDrivers returns list of hardcoded CSI drivers, i.e. list of directories in "test/extended/csi".
+func ListCSIDrivers() ([]string, error) {
+	return testdata.AssetDir(csiBasePath)
+}
+
+// Executes given golang template file and returns path to resulting file.
+func executeTemplate(templatePath string) (string, error) {
+	tmpl, err := template.ParseFiles(templatePath)
+	if err != nil {
+		return "", err
+	}
+	yamlFile, err := ioutil.TempFile("", "openshift-tests-csi-*")
+	if err != nil {
+		return "", err
+	}
+	yamlPath := yamlFile.Name()
+
+	imageFormat := os.Getenv(imageFormatVariable)
+	if imageFormat == "" {
+		imageFormat = defaultImageFormat
+	}
+
+	variables := struct{ AttacherImage, ProvisionerImage, NodeDriverRegistrarImage, LivenessProbeImage, ImageFormat string }{
+		AttacherImage:            strings.ReplaceAll(imageFormat, "${component}", "csi-external-attacher"),
+		ProvisionerImage:         strings.ReplaceAll(imageFormat, "${component}", "csi-external-provisioner"),
+		NodeDriverRegistrarImage: strings.ReplaceAll(imageFormat, "${component}", "csi-node-driver-registrar"),
+		LivenessProbeImage:       strings.ReplaceAll(imageFormat, "${component}", "csi-livenessprobe"),
+		ImageFormat:              imageFormat,
+	}
+
+	err = tmpl.Execute(yamlFile, variables)
+	yamlFile.Close()
+	if err != nil {
+		os.Remove(yamlPath)
+		return "", err
+	}
+	return yamlPath, nil
+}