diff --git a/test/e2e/v2/AGENTS.md b/test/e2e/v2/AGENTS.md index 4f7da4527e68..19c49e203fca 100644 --- a/test/e2e/v2/AGENTS.md +++ b/test/e2e/v2/AGENTS.md @@ -13,7 +13,7 @@ This is a Ginkgo v2 BDD test suite for validating hosted cluster control planes. The framework is organized into the following packages under `test/e2e/v2/`: -- `internal/` — Framework internals (test context, workload registry, fail handler, env var management). Do not add tests here. +- `internal/` — Framework internals (test context, workload registry, fail handler, env var management). Do not add e2e tests here; standard Go unit tests (`func TestXxx(t *testing.T)`) for internal functions are fine. - `tests/` — All standard v2 test files. Each file is feature-scoped with a top-level `Describe` and `Label`. The suite entry point is `suite_test.go`. - `util/` — Shared test utilities (pod exec helpers, metrics fetching) consumed by test files. Unlike `internal/`, these are importable by other packages. - `lifecycle/` — Platform-specific lifecycle helpers (e.g., Azure platform hooks). diff --git a/test/e2e/v2/internal/fail_handler.go b/test/e2e/v2/internal/fail_handler.go index d0f5aa144e36..218e38129c23 100644 --- a/test/e2e/v2/internal/fail_handler.go +++ b/test/e2e/v2/internal/fail_handler.go @@ -4,18 +4,37 @@ package internal import ( "slices" + "strings" . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" ) +// InformingLabel is the Ginkgo label that marks a test as informing. +const InformingLabel = "Informing" + +const informingSkipPrefix = "informing test failure: " + // InformingAwareFailHandler checks if the current spec has the "Informing" label. // If so, it skips the test with the failure message instead of failing the suite. func InformingAwareFailHandler(message string, callerSkip ...int) { labels := CurrentSpecReport().Labels() - if slices.Contains(labels, "Informing") { - // Skip marks test as skipped (visible in reports) without failing suite - Skip("informing test failure: " + message, callerSkip...) + if slices.Contains(labels, InformingLabel) { + Skip(informingSkipPrefix+message, callerSkip...) } - // For non-Informing tests, fail normally Fail(message, callerSkip...) } + +// IsInformingFailureSkip returns true if the spec was skipped by +// InformingAwareFailHandler due to an informing test failure. +func IsInformingFailureSkip(spec types.SpecReport) bool { + return spec.State == types.SpecStateSkipped && + slices.Contains(spec.Labels(), InformingLabel) && + strings.HasPrefix(spec.Failure.Message, informingSkipPrefix) +} + +// InformingFailureMessage extracts the original failure message from a spec +// that was skipped by InformingAwareFailHandler. +func InformingFailureMessage(spec types.SpecReport) string { + return strings.TrimPrefix(spec.Failure.Message, informingSkipPrefix) +} diff --git a/test/e2e/v2/internal/junit.go b/test/e2e/v2/internal/junit.go new file mode 100644 index 000000000000..017f4fe9b025 --- /dev/null +++ b/test/e2e/v2/internal/junit.go @@ -0,0 +1,102 @@ +//go:build e2ev2 + +package internal + +import ( + "encoding/xml" + + "github.com/onsi/ginkgo/v2/types" +) + +const lifecycleInforming = "informing" + +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []*JUnitTestSuite `xml:"testsuite"` +} + +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Name string `xml:"name,attr"` + NumTests int `xml:"tests,attr"` + NumSkipped int `xml:"skipped,attr"` + NumFailed int `xml:"failures,attr"` + Duration float64 `xml:"time,attr"` + TestCases []*JUnitTestCase `xml:"testcase"` +} + +type JUnitTestCase struct { + XMLName xml.Name `xml:"testcase"` + Name string `xml:"name,attr"` + Duration float64 `xml:"time,attr"` + Lifecycle string `xml:"lifecycle,attr,omitempty"` + Properties []*JUnitProperty `xml:"properties>property,omitempty"` + SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"` + FailureOutput *JUnitFailureOutput `xml:"failure,omitempty"` +} + +type JUnitProperty struct { + XMLName xml.Name `xml:"property"` + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +type JUnitSkipMessage struct { + XMLName xml.Name `xml:"skipped"` + Message string `xml:"message,attr,omitempty"` +} + +type JUnitFailureOutput struct { + XMLName xml.Name `xml:"failure"` + Message string `xml:"message,attr,omitempty"` + Output string `xml:",chardata"` +} + +// BuildInformingTestsLifecycleReport builds a JUnit test suite containing only informing test +// failures from the Ginkgo report. Informing failures converted to skips by +// InformingAwareFailHandler are re-emitted as failures with lifecycle="informing". +// ci-to-bigquery reads this attribute and populates the lifecycle column in +// BigQuery, making informing failures visible to Component Readiness. +func BuildInformingTestsLifecycleReport(suiteName string, specReports types.SpecReports) *JUnitTestSuites { + suite := &JUnitTestSuite{ + Name: suiteName + " [informing]", + } + + for _, spec := range specReports { + if !IsInformingFailureSkip(spec) { + continue + } + + msg := InformingFailureMessage(spec) + tc := &JUnitTestCase{ + Name: spec.FullText(), + Duration: spec.RunTime.Seconds(), + Lifecycle: lifecycleInforming, + Properties: []*JUnitProperty{ + {Name: "lifecycle", Value: lifecycleInforming}, + }, + FailureOutput: &JUnitFailureOutput{ + Message: msg, + Output: spec.Failure.Location.String(), + }, + } + + suite.TestCases = append(suite.TestCases, tc) + suite.NumTests++ + suite.NumFailed++ + } + + suite.Duration = sumDuration(suite.TestCases) + + return &JUnitTestSuites{ + Suites: []*JUnitTestSuite{suite}, + } +} + +func sumDuration(cases []*JUnitTestCase) float64 { + var total float64 + for _, tc := range cases { + total += tc.Duration + } + return total +} diff --git a/test/e2e/v2/internal/junit_test.go b/test/e2e/v2/internal/junit_test.go new file mode 100644 index 000000000000..5fee70af17e1 --- /dev/null +++ b/test/e2e/v2/internal/junit_test.go @@ -0,0 +1,275 @@ +//go:build e2ev2 + +package internal + +import ( + "encoding/xml" + "strings" + "testing" + "time" + + "github.com/onsi/ginkgo/v2/types" +) + +func TestBuildLifecycleReport(t *testing.T) { + tests := []struct { + name string + specs types.SpecReports + + wantTests int + wantFailed int + + // Per-testcase assertions, indexed by position in the output. + // Empty means only suite-level counts are checked. + wantCases []wantCase + }{ + { + name: "When an informing failure was converted to a skip, it should emit a failure", + specs: types.SpecReports{ + informingFailureSkip("should have custom labels", "expected labels to match"), + }, + wantTests: 1, + wantFailed: 1, + wantCases: []wantCase{ + { + lifecycle: lifecycleInforming, + failureMessage: "expected labels to match", + }, + }, + }, + { + name: "When an informing test passes, it should be excluded", + specs: types.SpecReports{ + informingPass("should have custom tolerations"), + }, + wantTests: 0, + }, + { + name: "When specs are non-informing, it should exclude them entirely", + specs: types.SpecReports{ + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "blocking pass", + State: types.SpecStatePassed, + RunTime: time.Second, + }, + { + LeafNodeType: types.NodeTypeIt, + LeafNodeLabels: []string{"blocking"}, + ContainerHierarchyTexts: []string{"Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "blocking fail", + State: types.SpecStateFailed, + RunTime: time.Second, + Failure: types.Failure{Message: "boom"}, + }, + }, + wantTests: 0, + }, + { + name: "When an informing test has a regular skip, it should be excluded", + specs: types.SpecReports{ + { + LeafNodeType: types.NodeTypeIt, + LeafNodeLabels: []string{InformingLabel}, + ContainerHierarchyTexts: []string{"Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "should skip", + State: types.SpecStateSkipped, + Failure: types.Failure{Message: "platform not supported"}, + }, + }, + wantTests: 0, + }, + { + name: "When a spec is a BeforeSuite node, it should exclude it", + specs: types.SpecReports{ + { + LeafNodeType: types.NodeTypeBeforeSuite, + LeafNodeLabels: []string{InformingLabel}, + ContainerHierarchyTexts: []string{}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "setup", + State: types.SpecStatePassed, + }, + }, + wantTests: 0, + }, + { + name: "When an informing test panics directly, it should be excluded", + specs: types.SpecReports{ + { + LeafNodeType: types.NodeTypeIt, + LeafNodeLabels: []string{InformingLabel}, + ContainerHierarchyTexts: []string{"Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "should not panic", + State: types.SpecStatePanicked, + RunTime: 500 * time.Millisecond, + Failure: types.Failure{ + Message: "runtime error: nil pointer", + Location: types.CodeLocation{FileName: "x_test.go", LineNumber: 7}, + }, + }, + }, + wantTests: 0, + }, + { + name: "When specs are mixed, it should include only informing failure skips", + specs: types.SpecReports{ + informingPass("informing pass"), + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "blocking pass", + State: types.SpecStatePassed, + RunTime: time.Second, + }, + informingFailureSkip("informing fail", "expected X"), + { + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: "blocking fail", + State: types.SpecStateFailed, + RunTime: time.Second, + Failure: types.Failure{Message: "bad"}, + }, + }, + wantTests: 1, + wantFailed: 1, + wantCases: []wantCase{ + {lifecycle: lifecycleInforming, failureMessage: "expected X"}, + }, + }, + { + name: "When the spec list is empty, it should produce an empty suite", + specs: types.SpecReports{}, + wantTests: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := BuildInformingTestsLifecycleReport("test-suite", tt.specs) + + if n := len(result.Suites); n != 1 { + t.Fatalf("expected 1 suite, got %d", n) + } + suite := result.Suites[0] + + if suite.NumTests != tt.wantTests { + t.Errorf("NumTests: got %d, want %d", suite.NumTests, tt.wantTests) + } + if suite.NumFailed != tt.wantFailed { + t.Errorf("NumFailed: got %d, want %d", suite.NumFailed, tt.wantFailed) + } + if n := len(suite.TestCases); n != tt.wantTests { + t.Fatalf("len(TestCases): got %d, want %d", n, tt.wantTests) + } + + for i, want := range tt.wantCases { + tc := suite.TestCases[i] + if tc.Lifecycle != want.lifecycle { + t.Errorf("case[%d] lifecycle: got %q, want %q", i, tc.Lifecycle, want.lifecycle) + } + if tc.FailureOutput == nil { + t.Fatalf("case[%d] expected failure output", i) + } + if want.failureMessage != "" && tc.FailureOutput.Message != want.failureMessage { + t.Errorf("case[%d] failure message: got %q, want %q", i, tc.FailureOutput.Message, want.failureMessage) + } + assertLifecycleProperty(t, i, tc, want.lifecycle) + } + }) + } +} + +func TestBuildLifecycleReport_XMLRoundTrip(t *testing.T) { + specs := types.SpecReports{ + informingFailureSkip("should emit lifecycle", "expected value"), + } + + result := BuildInformingTestsLifecycleReport("e2e", specs) + data, err := xml.MarshalIndent(result, "", " ") + if err != nil { + t.Fatalf("marshal: %v", err) + } + xmlStr := string(data) + + for _, want := range []string{ + `lifecycle="informing"`, + ` for informing failure:\n%s", xmlStr) + } + + var parsed JUnitTestSuites + if err := xml.Unmarshal(data, &parsed); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if n := len(parsed.Suites); n != 1 { + t.Fatalf("round-trip produced %d suites, want 1", n) + } + if n := len(parsed.Suites[0].TestCases); n != 1 { + t.Fatalf("round-trip produced %d cases, want 1", n) + } + tc := parsed.Suites[0].TestCases[0] + if tc.Lifecycle != lifecycleInforming { + t.Errorf("round-trip lifecycle: got %q, want %q", tc.Lifecycle, lifecycleInforming) + } +} + +// --- helpers --- + +type wantCase struct { + lifecycle string + failureMessage string +} + +func informingFailureSkip(leafText, originalMessage string) types.SpecReport { + return types.SpecReport{ + LeafNodeType: types.NodeTypeIt, + LeafNodeLabels: []string{InformingLabel}, + ContainerHierarchyTexts: []string{"[sig-hypershift] Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: leafText, + State: types.SpecStateSkipped, + RunTime: 2 * time.Second, + Failure: types.Failure{ + Message: informingSkipPrefix + originalMessage, + Location: types.CodeLocation{FileName: "test.go", LineNumber: 42}, + }, + } +} + +func informingPass(leafText string) types.SpecReport { + return types.SpecReport{ + LeafNodeType: types.NodeTypeIt, + LeafNodeLabels: []string{InformingLabel}, + ContainerHierarchyTexts: []string{"[sig-hypershift] Suite"}, + ContainerHierarchyLabels: [][]string{{}}, + LeafNodeText: leafText, + State: types.SpecStatePassed, + RunTime: time.Second, + } +} + +func assertLifecycleProperty(t *testing.T, idx int, tc *JUnitTestCase, expected string) { + t.Helper() + for _, p := range tc.Properties { + if p.Name == "lifecycle" && p.Value == expected { + return + } + } + t.Errorf("case[%d]: expected lifecycle property %q, not found", idx, expected) +} diff --git a/test/e2e/v2/tests/suite_test.go b/test/e2e/v2/tests/suite_test.go index 2ae79d315b94..b597ae4b968a 100644 --- a/test/e2e/v2/tests/suite_test.go +++ b/test/e2e/v2/tests/suite_test.go @@ -18,6 +18,10 @@ package tests import ( "context" + "encoding/xml" + "fmt" + "os" + "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" @@ -49,6 +53,35 @@ func TestE2EV2(t *testing.T) { RunSpecs(t, "hypershift-e2e") } +// ReportAfterSuite writes a supplemental JUnit file containing only informing +// tests with lifecycle="informing" on each . This is picked up by +// ci-to-bigquery and loaded into the ci_analysis_us.junit BigQuery table, +// making informing test failures visible to Component Readiness. +// +// TODO(CNTRLPLANE-3863): Replace this with OTE's built-in lifecycle JUnit +// emission once the test framework is ported to OTE. +var _ = ReportAfterSuite("Write lifecycle-aware JUnit", func(report Report) { + artifactDir := internal.GetEnvVarValue("ARTIFACT_DIR") + suites := internal.BuildInformingTestsLifecycleReport(report.SuiteDescription, report.SpecReports) + if len(suites.Suites) == 0 || len(suites.Suites[0].TestCases) == 0 { + return + } + + // The filename here is arbitrary; the CI system ingests all JUnit files written + // to the artifact location, so the only constraint to satisfy is that this name + // shouldn't conflict with the reports the main Ginkgo report writer produces. + const junitFilename = "junit_lifecycle_informing.xml" + + data, err := xml.MarshalIndent(suites, "", " ") + if err != nil { + Fail(fmt.Sprintf("failed to marshal lifecycle JUnit: %v", err)) + } + path := filepath.Join(artifactDir, junitFilename) + if err := os.WriteFile(path, append([]byte(xml.Header), data...), 0644); err != nil { + Fail(fmt.Sprintf("failed to write lifecycle JUnit to %s: %v", path, err)) + } +}) + var _ = BeforeSuite(func() { ctx := context.Background()