Support for excluding subset of tests

pkg/apis/testharness/v1beta1/test_types.go

@@ -77,6 +77,10 @@ type TestSuite struct {
 	Suppress []string `json:"suppress"`
 
 	Config *RestConfig `json:"config,omitempty"`
+
+	// Test sub directories that needs to be excluded from the test run that may otherwise be included with `TestDirs`.
+	// Include all directories specified in `TestDirs` and exclude those specified in `TestExcludeDirs`.
+	TestExcludeDirs []string `json:"testExcludeDirs,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/test/harness.go

@@ -56,7 +56,7 @@ type Harness struct {
 }
 
 // LoadTests loads all of the tests in a given directory.
-func (h *Harness) LoadTests(dir string) ([]*Case, error) {
+func (h *Harness) LoadTests(dir string, testExcludes []string) ([]*Case, error) {
 	dir, err := filepath.Abs(dir)
 	if err != nil {
 		return nil, err
@@ -77,6 +77,11 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 			continue
 		}
 
+		// If test excludes are specified, check and add tests that are not excluded explicitly.
+		if isTestExcluded(file.Name(), testExcludes) {
+			h.T.Logf("skipping test dir %s as its excluded by excludeDirs", file.Name())
+			continue
+		}
 		tests = append(tests, &Case{
 			Timeout:            timeout,
 			Steps:              []*Step{},
@@ -360,12 +365,13 @@ func (h *Harness) RunTests() {
 	h.T.Log("running tests")
 
 	testDirs := h.testPreProcessing()
+	testExcludes := h.TestSuite.TestExcludeDirs
 
 	//todo: testsuite + testsuites (extend case to have what we need (need testdir here)
 	// TestSuite is a TestSuiteCollection and should be renamed for v1beta2
 	realTestSuite := make(map[string][]*Case)
 	for _, testDir := range testDirs {
-		tempTests, err := h.LoadTests(testDir)
+		tempTests, err := h.LoadTests(testDir, testExcludes)
 		if err != nil {
 			h.T.Fatal(err)
 		}
@@ -637,3 +643,18 @@ func (h *Harness) loadKindConfig(path string) (*kindConfig.Cluster, error) {
 	}
 	return cluster, nil
 }
+
+// isTestExcluded returns true if a test is excluded explicitly, false otherwise.
+func isTestExcluded(testDirName string, testExcludes []string) bool {
+	if len(testExcludes) <= 0 {
+		// Test excludes not specified. Include all tests by default.
+		return false
+	}
+	for _, testExclude := range testExcludes {
+		// TODO: support regex based comparison in future
+		if strings.Compare(testDirName, testExclude) == 0 {
+			return true
+		}
+	}
+	return false
+}