diff --git a/test/extended/setup.sh b/test/extended/setup.sh index 5ec867afdb1e..6822ccbd43aa 100644 --- a/test/extended/setup.sh +++ b/test/extended/setup.sh @@ -11,17 +11,22 @@ function os::test::extended::focus () { if [[ -n "${FOCUS:-}" ]]; then exitstatus=0 + local excludesFile="${ARTIFACT_DIR}/parallel_test_names.txt" + # first run anything that isn't explicitly declared [Serial], and matches the $FOCUS, in a parallel mode. os::log::info "Running parallel tests N=${PARALLEL_NODES:-} with focus ${FOCUS}" - TEST_REPORT_FILE_NAME=focus_parallel TEST_PARALLEL="${PARALLEL_NODES:-5}" os::test::extended::run -- -ginkgo.skip "\[Serial\]" -test.timeout 6h ${TEST_EXTENDED_ARGS-} || exitstatus=$? + TEST_REPORT_FILE_NAME=focus_parallel TEST_PARALLEL="${PARALLEL_NODES:-5}" \ + os::test::extended::run -- \ + -print-test-names-to "${excludesFile}" \ + -ginkgo.skip "\[Serial\]" \ + -test.timeout 6h ${TEST_EXTENDED_ARGS-} || exitstatus=$? # Then run everything that requires serial and matches the $FOCUS, serially. - # there is bit of overlap here because not all serial tests declare [Serial], so they might have run in the - # parallel section above. Hopefully your focus was precise enough to exclude them, and we should be adding - # the [Serial] tag to them as needed. os::log::info "" os::log::info "Running serial tests with focus ${FOCUS}" - TEST_REPORT_FILE_NAME=focus_serial os::test::extended::run -- -suite "serial.conformance.openshift.io" -test.timeout 6h ${TEST_EXTENDED_ARGS-} || exitstatus=$? + TEST_REPORT_FILE_NAME=focus_serial os::test::extended::run -- \ + -excludes-from-file "${excludesFile}" \ + -test.timeout 6h ${TEST_EXTENDED_ARGS-} || exitstatus=$? os::test::extended::merge_junit diff --git a/test/extended/util/reporter.go b/test/extended/util/reporter.go index 7f56e494dc01..5494d125f777 100644 --- a/test/extended/util/reporter.go +++ b/test/extended/util/reporter.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters/stenographer" "github.com/onsi/ginkgo/types" @@ -25,6 +26,8 @@ func NewSimpleReporter() *SimpleReporter { } } +var _ ginkgo.Reporter = &SimpleReporter{} + func (r *SimpleReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { fmt.Fprintf(r.Output, "=== SUITE %s (%d total specs, %d will run):\n", summary.SuiteDescription, summary.NumberOfTotalSpecs, summary.NumberOfSpecsThatWillBeRun) } @@ -124,3 +127,47 @@ func trimLocation(l types.CodeLocation) string { delimiter := "/openshift/origin/" return fmt.Sprintf("%q", l.FileName[strings.LastIndex(l.FileName, delimiter)+len(delimiter):]) } + +// TestNamesReporter prints completed (not skipped) test names to the given file. Each test name on single +// line. +type TestNamesReporter struct { + prefix string + out io.WriteCloser + closeOnSuiteEnd bool +} + +var _ ginkgo.Reporter = &TestNamesReporter{} + +// NewTestNamesReporterForFile creates a new TestNamesReporter for the given file path. +func NewTestNamesReporterForFile(filePath string) (*TestNamesReporter, error) { + out, err := os.Create(filePath) + if err != nil { + return nil, err + } + return &TestNamesReporter{ + out: out, + closeOnSuiteEnd: true, + }, nil +} + +func (r *TestNamesReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { + r.prefix = summary.SuiteDescription +} + +func (r *TestNamesReporter) SpecWillRun(specSummary *types.SpecSummary) {} + +func (r *TestNamesReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} + +func (r *TestNamesReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {} + +func (r *TestNamesReporter) SpecDidComplete(specSummary *types.SpecSummary) { + if specSummary.State != types.SpecStateSkipped { + r.out.Write([]byte(strings.Join(specSummary.ComponentTexts, " ") + "\n")) + } +} + +func (r *TestNamesReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { + if r.closeOnSuiteEnd { + r.out.Close() + } +} diff --git a/test/extended/util/test.go b/test/extended/util/test.go index 8c19e4db12ff..cb19e3864f3c 100644 --- a/test/extended/util/test.go +++ b/test/extended/util/test.go @@ -1,8 +1,10 @@ package util import ( + "bufio" "flag" "fmt" + "io" "os" "path" "path/filepath" @@ -35,10 +37,12 @@ import ( ) var ( - reportDir string - reportFileName string - syntheticSuite string - quiet bool + reportDir string + reportFileName string + syntheticSuite string + excludesFile string + quiet bool + printTestNamesTo string ) var TestContext *e2e.TestContextType = &e2e.TestContext @@ -55,6 +59,8 @@ func InitTest() { e2e.RegisterCommonFlags() e2e.RegisterClusterFlags() flag.StringVar(&syntheticSuite, "suite", "", "DEPRECATED: Optional suite selector to filter which tests are run. Use focus.") + flag.StringVar(&excludesFile, "excludes-from-file", "", "Path to file containing full test names to exclude. Each line may contain exactly one test name.") + flag.StringVar(&printTestNamesTo, "print-test-names-to", "", "Path to a file that will be filled with the names of tests that will have been run. The output can be used as the input for excludes-from-file.") extendedOutputDir := filepath.Join(os.TempDir(), "openshift-extended-tests") os.MkdirAll(extendedOutputDir, 0777) @@ -95,37 +101,63 @@ func InitTest() { } func ExecuteTest(t *testing.T, suite string) { - var r []ginkgo.Reporter - - if reportDir != "" { + if len(reportDir) > 0 { if err := os.MkdirAll(reportDir, 0755); err != nil { glog.Errorf("Failed creating report directory: %v", err) } defer e2e.CoreDump(reportDir) } - switch syntheticSuite { - case "parallel.conformance.openshift.io": - if len(config.GinkgoConfig.FocusString) > 0 { - config.GinkgoConfig.FocusString += "|" + if len(config.GinkgoConfig.FocusString) == 0 { + switch syntheticSuite { + case "parallel.conformance.openshift.io": + config.GinkgoConfig.FocusString = "\\[Suite:openshift/conformance/parallel\\]" + case "serial.conformance.openshift.io": + config.GinkgoConfig.FocusString = "\\[Suite:openshift/conformance/serial\\]" } - config.GinkgoConfig.FocusString = "\\[Suite:openshift/conformance/parallel\\]" - case "serial.conformance.openshift.io": - if len(config.GinkgoConfig.FocusString) > 0 { - config.GinkgoConfig.FocusString += "|" - } - config.GinkgoConfig.FocusString = "\\[Suite:openshift/conformance/serial\\]" } if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { config.GinkgoConfig.SkipString = "Skipped" } + annotateTests() + + if len(excludesFile) > 0 { + reExcludes, err := loadExcludesFromFile(excludesFile) + if err != nil { + FatalErr(err.Error()) + } + // This must be run after the test annotating because the created focus string matches the test names + // exactly. + focusNotExcludedTests(suite, reExcludes) + } + gomega.RegisterFailHandler(ginkgo.Fail) - if reportDir != "" { + var r []ginkgo.Reporter + + if len(reportDir) > 0 { r = append(r, reporters.NewJUnitReporter(path.Join(reportDir, fmt.Sprintf("%s_%02d.xml", reportFileName, config.GinkgoConfig.ParallelNode)))) } + if len(printTestNamesTo) > 0 { + reporter, err := NewTestNamesReporterForFile(printTestNamesTo) + if err != nil { + FatalErr(err.Error()) + } + r = append(r, reporter) + } + + if quiet { + r = append(r, NewSimpleReporter()) + ginkgo.RunSpecsWithCustomReporters(t, suite, r) + } else { + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, suite, r) + } +} + +// annotateTests suffixes each test name with a proper annotation (e.g. [Serial] or [Suite:...]). +func annotateTests() { ginkgo.WalkTests(func(name string, node types.TestNode) { isSerial := serialTestsFilter.MatchString(name) if isSerial { @@ -152,13 +184,44 @@ func ExecuteTest(t *testing.T, suite string) { node.SetText(node.Text() + " [Suite:k8s]") } }) +} - if quiet { - r = append(r, NewSimpleReporter()) - ginkgo.RunSpecsWithCustomReporters(t, suite, r) - } else { - ginkgo.RunSpecsWithDefaultAndCustomReporters(t, suite, r) +// focusNotExcludedTests builds and sets a new focus string containing names of all the tests that should be +// run. The tests must be matched neither by reExcludes nor by the current GinkgoConfig.SkipString. +func focusNotExcludedTests(suite string, reExcludes *regexp.Regexp) { + var ( + reFocus, reSkip *regexp.Regexp + err error + ) + + if reExcludes == nil { + return } + + if len(config.GinkgoConfig.SkipString) > 0 { + reSkip, err = regexp.Compile(config.GinkgoConfig.SkipString) + if err != nil { + FatalErr(fmt.Sprintf("failed to compile skip string %q: %v", config.GinkgoConfig.SkipString, err)) + } + } + if len(config.GinkgoConfig.FocusString) > 0 { + reFocus, err = regexp.Compile(config.GinkgoConfig.FocusString) + if err != nil { + FatalErr(fmt.Sprintf("failed to compile focus string %q: %v", config.GinkgoConfig.FocusString, err)) + } + } + + focusNames := []string{} + ginkgo.WalkTests(func(name string, node types.TestNode) { + if reExcludes.MatchString(name) || (reSkip != nil && reSkip.MatchString(name)) { + return + } + if reFocus != nil && reFocus.MatchString(name) { + focusNames = append(focusNames, regexp.QuoteMeta(fmt.Sprintf("%s %s", suite, name))) + } + }) + + config.GinkgoConfig.FocusString = fmt.Sprintf(`^(?:%s)$`, strings.Join(focusNames, "|")) } // TODO: Use either explicit tags (k8s.io) or https://github.com/onsi/ginkgo/pull/228 to implement this. @@ -492,3 +555,40 @@ func addRoleToE2EServiceAccounts(c authorizationclient.Interface, namespaces []k FatalErr(err) } } + +func loadExcludesFromFile(filePath string) (*regexp.Regexp, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + var ( + testNames []string + line string + readerErr error + ) + + reader := bufio.NewReader(file) + for readerErr == nil { + line, readerErr = reader.ReadString('\n') + + if readerErr != nil && readerErr != io.EOF { + return nil, readerErr + } + + line = strings.TrimSuffix(line, "\n") + + if len(line) == 0 { + continue + } + + testNames = append(testNames, regexp.QuoteMeta(line)) + } + + if len(testNames) == 0 { + return nil, nil + } + + return regexp.Compile(fmt.Sprintf("^(?:%s)$", strings.Join(testNames, "|"))) +}