tests: Create a new test binary that can be used instead of extended.tests

Makefile

@@ -55,7 +55,7 @@ build-network:
 .PHONY: build-network
 
 build-extended-test:
-	hack/build-go.sh test/extended/extended.test
+	hack/build-go.sh cmd/openshift-tests
 .PHONY: build-extended-test
 
 build-integration-test: build-router-e2e-test

cmd/openshift-tests/e2e.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
+
+	"github.com/openshift/origin/pkg/test/ginkgo"
+
+	_ "github.com/openshift/origin/test/extended"
+)
+
+// staticSuites are all known test suites this binary should run
+var staticSuites = []*ginkgo.TestSuite{
+	{
+		Name: "openshift/conformance",
+		Description: templates.LongDesc(`
+		Tests that ensure an OpenShift cluster and components are working properly.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[Suite:openshift/conformance/")
+		},
+		Parallelism: 30,
+	},
+	{
+		Name: "openshift/conformance/parallel",
+		Description: templates.LongDesc(`
+		Only the portion of the openshift/conformance test suite that run in parallel.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[Suite:openshift/conformance/parallel")
+		},
+		Parallelism: 30,
+	},
+	{
+		Name: "openshift/conformance/serial",
+		Description: templates.LongDesc(`
+		Only the portion of the openshift/conformance test suite that run serially.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[Suite:openshift/conformance/serial")
+		},
+	},
+	{
+		Name: "kubernetes/conformance",
+		Description: templates.LongDesc(`
+		The default Kubernetes conformance suite.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[Suite:k8s]") && strings.Contains(name, "[Conformance]")
+		},
+		Parallelism: 30,
+	},
+	{
+		Name: "openshift/build",
+		Description: templates.LongDesc(`
+		Tests that exercise the OpenShift build functionality.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[Feature:Builds]")
+		},
+		Parallelism: 7,
+		// Jenkins tests can take upwards of 40 minutes
+		TestTimeout: 45 * time.Minute,
+	},
+	{
+		Name: "openshift/image-registry",
+		Description: templates.LongDesc(`
+		Tests that exercise the OpenShift image-registry functionality.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[registry]") && !strings.Contains(name, "[Local]")
+		},
+	},
+	{
+		Name: "openshift/image-ecosystem",
+		Description: templates.LongDesc(`
+		Tests that exercise language and tooling images shipped as part of OpenShift.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[image_ecosystem]") && !strings.Contains(name, "[Local]")
+		},
+		Parallelism: 7,
+		TestTimeout: 20 * time.Minute,
+	},
+	{
+		Name: "openshift/smoke-4",
+		Description: templates.LongDesc(`
+		Tests that verify a 4.X cluster (using the new operator based core) is ready. This
+		suite will be removed in favor of openshift/conformance once all functionality is
+		available.
+		`),
+		Matches: func(name string) bool {
+			return strings.Contains(name, "[Suite:openshift/smoke-4]")
+		},
+		Parallelism: 10,
+	},
+	{
+		Name: "openshift/all",
+		Description: templates.LongDesc(`
+		Run all tests.
+		`),
+		Matches: func(name string) bool { return true },
+	},
+	{
+		Name: "kubernetes/all",
+		Description: templates.LongDesc(`
+		Run all Kubernetes tests.
+		`),
+		Matches: func(name string) bool { return strings.Contains(name, "[k8s.io]") },
+	},
+}

cmd/openshift-tests/openshift-tests.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/onsi/gomega"
+
+	"github.com/golang/glog"
+	"github.com/onsi/ginkgo"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+
+	"github.com/openshift/origin/pkg/cmd/flagtypes"
+	testginkgo "github.com/openshift/origin/pkg/test/ginkgo"
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	root := &cobra.Command{
+		Long: templates.LongDesc(`
+		OpenShift Tests
+
+		This command verifies behavior of an OpenShift cluster by running remote tests against
+		the cluster API that exercise functionality. In general these tests may be disruptive
+		or require elevated privileges - see the descriptions of each test suite.
+		`),
+	}
+	flagtypes.GLog(root.PersistentFlags())
+
+	suites := staticSuites
+
+	suiteOpt := &testginkgo.Options{
+		DetectFlakes: 6,
+		Suites:       suites,
+	}
+	cmd := &cobra.Command{
+		Use:   "run SUITE",
+		Short: "Run a test suite",
+		Long: templates.LongDesc(`
+		Run a test suite against an OpenShift server
+
+		This command will run one of the following suites against a cluster identified by the current
+		KUBECONFIG file. See the suite description for more on what actions the suite will take.
+
+		If you specify the --dry-run argument, the names of each individual test that is part of the
+		suite will be printed, one per line. You may filter this list and pass it back to the run
+		command with the --file argument. You may also pipe a list of test names, one per line, on
+		standard input by passing "-f -".
+
+		`) + testginkgo.SuitesString(suites, "\n\nAvailable test suites:\n\n"),
+
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var exitErr error
+			var out, errOut io.Writer = os.Stdout, os.Stderr
+			if len(suiteOpt.OutFile) > 0 {
+				f, err := os.OpenFile(suiteOpt.OutFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640)
+				if err != nil {
+					return err
+				}
+				defer func() {
+					if exitErr != nil {
+						fmt.Fprintf(f, "error: %s", exitErr)
+					}
+					if err := f.Close(); err != nil {
+						fmt.Fprintf(os.Stderr, "error: Unable to close output file\n")
+					}
+				}()
+				out = io.MultiWriter(out, f)
+				errOut = io.MultiWriter(errOut, f)
+			}
+			suiteOpt.Out, suiteOpt.ErrOut = out, errOut
+
+			if exitErr = initProvider(suiteOpt.Provider); exitErr != nil {
+				return exitErr
+			}
+			os.Setenv("TEST_PROVIDER", suiteOpt.Provider)
+			exitErr = suiteOpt.Run(args)
+			return exitErr
+		},
+	}
+	cmd.Flags().BoolVar(&suiteOpt.DryRun, "dry-run", suiteOpt.DryRun, "Print the tests to run without executing them.")
+	cmd.Flags().StringVar(&suiteOpt.JUnitDir, "junit-dir", suiteOpt.JUnitDir, "The directory to write test reports to.")
+	cmd.Flags().StringVar(&suiteOpt.Provider, "provider", suiteOpt.Provider, "The cluster infrastructure provider. Will automatically default to the correct value.")
+	cmd.Flags().StringVarP(&suiteOpt.TestFile, "file", "f", suiteOpt.TestFile, "Create a suite from the newline-delimited test names in this file.")
+	cmd.Flags().StringVarP(&suiteOpt.OutFile, "output-file", "o", suiteOpt.OutFile, "Write all test output to this file.")
+	cmd.Flags().DurationVar(&suiteOpt.Timeout, "timeout", suiteOpt.Timeout, "Set the maximum time a test can run before being aborted. This is read from the suite by default, but will be 10 minutes otherwise.")
+	root.AddCommand(cmd)
+
+	testOpt := &testginkgo.TestOptions{}
+	cmd = &cobra.Command{
+		Use:   "run-test NAME",
+		Short: "Run a single test by name",
+		Long: templates.LongDesc(`
+		Execute a single test
+
+		This executes a single test by name. It is used by the run command during suite execution but may also
+		be used to test in isolation while developing new tests.
+		`),
+
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := initProvider(os.Getenv("TEST_PROVIDER")); err != nil {
+				return err
+			}
+			return testOpt.Run(args)
+		},
+	}
+	cmd.Flags().BoolVar(&testOpt.DryRun, "dry-run", testOpt.DryRun, "Print the test to run without executing them.")
+	root.AddCommand(cmd)
+
+	pflag.CommandLine = pflag.NewFlagSet("empty", pflag.ExitOnError)
+	flag.CommandLine = flag.NewFlagSet("empty", flag.ExitOnError)
+	exutil.InitStandardFlags()
+
+	if err := root.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func initProvider(provider string) error {
+	// record the exit error to the output file
+	if err := decodeProviderTo(provider, exutil.TestContext); err != nil {
+		return err
+	}
+	exutil.TestContext.AllowedNotReadyNodes = 100
+
+	exutil.AnnotateTestSuite()
+	exutil.InitTest()
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	// TODO: infer SSH keys from the cluster
+	return nil
+}
+
+func decodeProviderTo(provider string, testContext *e2e.TestContextType) error {
+	switch provider {
+	case "":
+		if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
+			if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
+				testContext.Provider = "local"
+			}
+		}
+		// TODO: detect which provider the cluster is running and use that as a default.
+	default:
+		var providerInfo struct{ Type string }
+		if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
+			return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
+		}
+		if len(providerInfo.Type) == 0 {
+			return fmt.Errorf("provider must be a JSON object with the 'type' key")
+		}
+		testContext.Provider = providerInfo.Type
+		if err := json.Unmarshal([]byte(provider), &testContext.CloudConfig); err != nil {
+			return fmt.Errorf("provider must decode into the cloud config object: %v", err)
+		}
+	}
+	glog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig)
+	return nil
+}

hack/build-cross.sh

@@ -75,10 +75,6 @@ os::build::build_binaries "${OS_IMAGE_COMPILE_TARGETS_LINUX[@]-}"
 OS_BUILD_PLATFORMS=("${platforms[@]+"${platforms[@]}"}")
 os::build::build_binaries "${OS_CROSS_COMPILE_TARGETS[@]}"
 
-# Build the test binaries for the host platform
-OS_BUILD_PLATFORMS=("${test_platforms[@]+"${test_platforms[@]}"}")
-os::build::build_binaries "${OS_TEST_TARGETS[@]}"
-
 if [[ "${OS_BUILD_RELEASE_ARCHIVES-}" != "n" ]]; then
   # Make the primary client/server release.
   OS_BUILD_PLATFORMS=("${platforms[@]+"${platforms[@]}"}")

hack/lib/constants.sh

@@ -38,6 +38,7 @@ readonly OS_IMAGE_COMPILE_TARGETS_LINUX=(
   cmd/template-service-broker
   cmd/openshift-node-config
   cmd/openshift-sdn
+  cmd/openshift-tests
   cmd/openshift
   vendor/k8s.io/kubernetes/cmd/hyperkube
 )
@@ -51,10 +52,6 @@ readonly OS_CROSS_COMPILE_TARGETS=(
 )
 readonly OS_CROSS_COMPILE_BINARIES=("${OS_CROSS_COMPILE_TARGETS[@]##*/}")
 
-readonly OS_TEST_TARGETS=(
-  test/extended/extended.test
-)
-
 readonly OS_GOVET_BLACKLIST=(
 )