diff --git a/openshift-tests-plugin/pkg/plugin/plugin.go b/openshift-tests-plugin/pkg/plugin/plugin.go index 98c861a..c8c6e5f 100644 --- a/openshift-tests-plugin/pkg/plugin/plugin.go +++ b/openshift-tests-plugin/pkg/plugin/plugin.go @@ -134,6 +134,9 @@ func NewPlugin(name string) (*Plugin, error) { case PluginName10, PluginAlias10: p.id = PluginId10 p.SuiteName = PluginSuite10 + if suiteName := p.getSuiteName(PluginId10); suiteName != "" { + p.SuiteName = suiteName + } p.BlockerPlugins = []*Plugin{{name: PluginName05}} p.OTRunner = NewOpenShiftRunCommand("run", p.SuiteName) p.Timeout = 2 * time.Hour @@ -193,6 +196,20 @@ func (p *Plugin) PluginFullNameByName(name string) string { return fmt.Sprintf("%s-%s", id, name) } +// getSuiteName returns the suite name for the plugin. +func (p *Plugin) getSuiteName(id string) string { + switch id { + case PluginId10: + // Try to get from DEFAULT_SUITE_NAME, otherwise set const + suiteName := os.Getenv("DEFAULT_SUITE_NAME") + if suiteName == "" { + return PluginSuite10 + } + return suiteName + } + return "" +} + // Initialize resolve all dependencies before running the plugin. func (p *Plugin) Initialize() error { // TODO send a message to aggregator indicating for "initialization" state. @@ -476,11 +493,13 @@ func (p *Plugin) Run() error { threshold := 0 backoffSeconds := []int{1, 2, 4, 8} for { + // Exit the execution once the tests container/process has finished. if _, err := os.Stat(OpenShiftTestsDoneFile); err == nil { log.Info("Run: Detected done.") p.DoneControl = true break } else if errors.Is(err, os.ErrNotExist) { + // Keep waiting for the done file to be created (execution completed). sec := backoffSeconds[threshold%len(backoffSeconds)] log.Debugf("backoff waiting %d seconds for done file %s", sec, OpenShiftTestsDoneFile) time.Sleep(time.Duration(sec) * time.Second) @@ -511,6 +530,8 @@ func (p *Plugin) Done() { } // WatchForDone watches for the runtime (sonobuoy) done file. +// Done file signalize sonobuoy that the execution of plugin is done, +// and the plugin can start collecting the results and sending to the aggregator server. func (p *Plugin) WatchForDone() { defer p.Done() @@ -521,8 +542,10 @@ func (p *Plugin) WatchForDone() { log.Infof("Done file has been created at path %s\n", ResultsDoneFile) } -// RunReportProgress start the file/fifo scanner to report the progress, reading the -// data from the fifo, parsing it and sending to the aggregator server. +// RunReportProgress starts the file/fifo scanner to update status and progress. +// The scanner reads the data from the pipe file, parses it and updates the progress. +// The pipe file is created as output of the openshift-tests run command in the +// tests container/process. func (p *Plugin) RunReportProgress() { go func() { log.Info("Starting progress report reader...") @@ -623,6 +646,7 @@ func (p *Plugin) RunReportProgressUpgrade() { // RunDependencyWaiter runs the blocker plugin controller to ensure plugin/step // runs only after the previous plugin has been finished. +// The waiter ensures the DAG (Directed Acyclic Graph) of the workflows is respected. func (p *Plugin) RunDependencyWaiter() error { if len(p.BlockerPlugins) == 0 { return nil @@ -710,6 +734,14 @@ func (p *Plugin) RunDependencyWaiter() error { if pStatusBlocker.Status == "complete" || pStatusBlocker.Status == "failed" || podPhase == "Completed" { log.Infof("Plugin[%s] with status[%s] is in unblocker condition!", pluginBlocker, pStatusBlocker.Status) + + // Check if the blocker plugin failed and propagate failure to dependent plugins + // Exception: artifacts collector (99-openshift-artifacts-collector) should always run + if pStatusBlocker.Status == "failed" && p.ID() != PluginId99 { + log.Errorf("Blocker plugin[%s] failed. Propagating failure to dependent plugin[%s]", pluginBlocker, p.Name()) + return fmt.Errorf("blocker plugin %s failed, stopping execution of dependent plugin %s", pluginBlocker, p.Name()) + } + break } diff --git a/openshift-tests-plugin/pkg/plugin/plugin_test.go b/openshift-tests-plugin/pkg/plugin/plugin_test.go new file mode 100644 index 0000000..4700f13 --- /dev/null +++ b/openshift-tests-plugin/pkg/plugin/plugin_test.go @@ -0,0 +1,325 @@ +package plugin + +import ( + "os" + "testing" +) + +// TestGetSuiteName tests the getSuiteName method +func TestGetSuiteName(t *testing.T) { + tests := []struct { + name string + pluginID string + envValue string + setupEnv func() + cleanupEnv func() + expectedSuite string + }{ + { + name: "PluginId10 with DEFAULT_SUITE_NAME set", + pluginID: PluginId10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "kubernetes/conformance/parallel") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "kubernetes/conformance/parallel", + }, + { + name: "PluginId10 without DEFAULT_SUITE_NAME", + pluginID: PluginId10, + setupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + cleanupEnv: func() {}, + expectedSuite: PluginSuite10, + }, + { + name: "PluginId10 with empty DEFAULT_SUITE_NAME", + pluginID: PluginId10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: PluginSuite10, + }, + { + name: "PluginId05 should return empty string", + pluginID: PluginId05, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "some-value") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "", + }, + { + name: "PluginId20 should return empty string", + pluginID: PluginId20, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "some-value") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "", + }, + { + name: "PluginId80 should return empty string", + pluginID: PluginId80, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "some-value") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "", + }, + { + name: "PluginId99 should return empty string", + pluginID: PluginId99, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "some-value") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "", + }, + { + name: "Unknown plugin ID should return empty string", + pluginID: "999", + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "some-value") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "", + }, + { + name: "PluginId10 with custom suite name", + pluginID: PluginId10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "custom/conformance/suite") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "custom/conformance/suite", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup environment + tt.setupEnv() + defer tt.cleanupEnv() + + // Create a plugin instance (minimal initialization) + p := &Plugin{ + name: PluginName10, // Use valid plugin name for basic initialization + id: tt.pluginID, + } + + // Call getSuiteName + result := p.getSuiteName(tt.pluginID) + + // Verify result + if result != tt.expectedSuite { + t.Errorf("getSuiteName(%s) = %q, want %q", tt.pluginID, result, tt.expectedSuite) + } + }) + } +} + +// TestGetSuiteNameIntegration tests the integration of getSuiteName in NewPlugin +func TestGetSuiteNameIntegration(t *testing.T) { + tests := []struct { + name string + pluginName string + envValue string + setupEnv func() + cleanupEnv func() + expectedSuite string + wantErr bool + }{ + { + name: "Plugin10 uses getSuiteName with custom suite", + pluginName: PluginName10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "kubernetes/conformance/parallel") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "kubernetes/conformance/parallel", + wantErr: false, + }, + { + name: "Plugin10 uses default suite when env not set", + pluginName: PluginName10, + setupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + cleanupEnv: func() {}, + expectedSuite: PluginSuite10, + wantErr: false, + }, + { + name: "Plugin10 uses alias name with custom suite", + pluginName: PluginAlias10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "kubernetes/conformance/parallel") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "kubernetes/conformance/parallel", + wantErr: false, + }, + { + name: "Plugin05 does not use getSuiteName", + pluginName: PluginName05, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "should-be-ignored") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: PluginSuite05, + wantErr: false, + }, + { + name: "Plugin20 does not use getSuiteName", + pluginName: PluginName20, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "should-be-ignored") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: PluginSuite20, + wantErr: false, + }, + { + name: "Plugin80 does not use getSuiteName", + pluginName: PluginName80, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "should-be-ignored") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: PluginSuite80, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup environment + tt.setupEnv() + defer tt.cleanupEnv() + + // Create plugin using NewPlugin + plugin, err := NewPlugin(tt.pluginName) + + if tt.wantErr { + if err == nil { + t.Errorf("NewPlugin(%s) expected error, got nil", tt.pluginName) + } + return + } + + if err != nil { + t.Fatalf("NewPlugin(%s) unexpected error: %v", tt.pluginName, err) + } + + // Verify SuiteName was set correctly + if plugin.SuiteName != tt.expectedSuite { + t.Errorf("NewPlugin(%s).SuiteName = %q, want %q", tt.pluginName, plugin.SuiteName, tt.expectedSuite) + } + }) + } +} + +// TestGetSuiteNameConstantValues verifies the constant values used by getSuiteName +func TestGetSuiteNameConstantValues(t *testing.T) { + // Verify plugin constants are as expected + if PluginId10 != "10" { + t.Errorf("PluginId10 = %q, want %q", PluginId10, "10") + } + if PluginSuite10 != "kubernetes/conformance" { + t.Errorf("PluginSuite10 = %q, want %q", PluginSuite10, "kubernetes/conformance") + } +} + +// TestGetSuiteNameEdgeCases tests edge cases for getSuiteName +func TestGetSuiteNameEdgeCases(t *testing.T) { + tests := []struct { + name string + pluginID string + envValue string + setupEnv func() + cleanupEnv func() + expectedSuite string + }{ + { + name: "Very long suite name", + pluginID: PluginId10, + setupEnv: func() { + longName := "kubernetes/conformance/very/long/path/to/test/suite/that/exceeds/normal/length" + os.Setenv("DEFAULT_SUITE_NAME", longName) + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "kubernetes/conformance/very/long/path/to/test/suite/that/exceeds/normal/length", + }, + { + name: "Suite name with special characters", + pluginID: PluginId10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", "kubernetes/conformance-2.0_test") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: "kubernetes/conformance-2.0_test", + }, + { + name: "Suite name with whitespace (should preserve)", + pluginID: PluginId10, + setupEnv: func() { + os.Setenv("DEFAULT_SUITE_NAME", " kubernetes/conformance ") + }, + cleanupEnv: func() { + os.Unsetenv("DEFAULT_SUITE_NAME") + }, + expectedSuite: " kubernetes/conformance ", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setupEnv() + defer tt.cleanupEnv() + + p := &Plugin{ + name: PluginName10, + id: tt.pluginID, + } + + result := p.getSuiteName(tt.pluginID) + + if result != tt.expectedSuite { + t.Errorf("getSuiteName(%s) = %q, want %q", tt.pluginID, result, tt.expectedSuite) + } + }) + } +} diff --git a/openshift-tests-plugin/plugin/entrypoint-tests.sh b/openshift-tests-plugin/plugin/entrypoint-tests.sh index 6a21400..811439b 100755 --- a/openshift-tests-plugin/plugin/entrypoint-tests.sh +++ b/openshift-tests-plugin/plugin/entrypoint-tests.sh @@ -44,6 +44,12 @@ trap handle_error ERR --token="$(cat "${SA_TOKEN_PATH}")" \ --certificate-authority="${SA_CA_PATH}"; +# Extracting the suite list for each plugin (--dry-run). +# - openshift-tests-replay: skip +# - openshift-cluster-upgrade: gather suite list for upgrade plugin +# - openshift-kube-conformance: check if we have extracted k8s conformance tests from OTE (later 4.20 releases). +# If yes, use the extracted tests, otherwise use the default suite. +# - other plugins: gather suite list for plugin if [[ "${PLUGIN_NAME:-}" == "openshift-tests-replay" ]]; then echo "Skipping suite list for plugin ${PLUGIN_NAME:-}" @@ -55,10 +61,29 @@ elif [[ "${PLUGIN_NAME:-}" == "openshift-cluster-upgrade" ]] && [[ "${RUN_MODE:- ${CMD_OTESTS} ${OT_RUN_COMMAND:-run} ${SUITE_NAME:-${DEFAULT_SUITE_NAME-}} \ --to-image "${UPGRADE_RELEASES-}" \ --dry-run -o ${CTRL_SUITE_LIST} + elif [[ "${PLUGIN_NAME:-}" != "openshift-cluster-upgrade" ]]; then - echo "Gathering suite list for plugin ${PLUGIN_NAME:-} (stdin is redirected to ${CTRL_SUITE_LIST}.log)" - # shellcheck disable=SC2086 - ${CMD_OTESTS} ${OT_RUN_COMMAND:-run} ${SUITE_NAME:-${DEFAULT_SUITE_NAME-}} --dry-run -o ${CTRL_SUITE_LIST} >${CTRL_SUITE_LIST}.log + # For 10-openshift-kube-conformance plugin, we want to check if we have extracted k8s conformance tests from OTE, + # so that we can workaround the 4.20+ issue that suite kubernetes/conformance was removed. + # Check if we have extracted k8s conformance tests from OTE for kubernetes/conformance suite. + # The test list extraction is done in the init container of the plugin. Check the plugin manifest for more details. + K8S_CONFORMANCE_LIST="/tmp/shared/k8s-conformance-tests.list" + if [[ "${PLUGIN_NAME:-}" == "openshift-kube-conformance" ]] && [[ -f "${K8S_CONFORMANCE_LIST}" ]]; then + TEST_COUNT=$(wc -l < "${K8S_CONFORMANCE_LIST}") + if [[ $TEST_COUNT -gt 0 ]]; then + echo "Using extracted Kubernetes conformance tests from OTE (${TEST_COUNT} tests)" + cp "${K8S_CONFORMANCE_LIST}" "${CTRL_SUITE_LIST}" + echo "Tests extracted from k8s-tests-ext binary" > ${CTRL_SUITE_LIST}.log + else + echo "Warning: Extracted test list is empty, falling back to default suite" + # shellcheck disable=SC2086 + ${CMD_OTESTS} ${OT_RUN_COMMAND:-run} ${SUITE_NAME:-${DEFAULT_SUITE_NAME-}} --dry-run -o ${CTRL_SUITE_LIST} >${CTRL_SUITE_LIST}.log + fi + else + echo "Gathering suite list for plugin ${PLUGIN_NAME:-} (stdin is redirected to ${CTRL_SUITE_LIST}.log)" + # shellcheck disable=SC2086 + ${CMD_OTESTS} ${OT_RUN_COMMAND:-run} ${SUITE_NAME:-${DEFAULT_SUITE_NAME-}} --dry-run -o ${CTRL_SUITE_LIST} >${CTRL_SUITE_LIST}.log + fi else echo "Skipping suite list for plugin ${PLUGIN_NAME:-}" touch ${CTRL_SUITE_LIST} @@ -93,7 +118,7 @@ echo -e "\n\n\t>> Copying e2e artifacts to collector plugin..." oc cp -c plugin "${CTRL_SUITE_LIST}" opct/"${COLLECTOR_POD}":/tmp/sonobuoy/results/"${suite_file}" || true echo -e ">> Preparing e2e metatada..." - # must prefix with artifacts_ + # must set the filename prefix artifacts_ e2e_artifact_name="artifacts_e2e-metadata-${PLUGIN_NAME:-}.tar.gz" e2e_artifact="/tmp/${e2e_artifact_name}" tar cfzv "${e2e_artifact}" /tmp/shared/junit/* || true diff --git a/openshift-tests-plugin/plugin/platform.sh b/openshift-tests-plugin/plugin/platform.sh index b8dff32..69bb09b 100755 --- a/openshift-tests-plugin/plugin/platform.sh +++ b/openshift-tests-plugin/plugin/platform.sh @@ -21,7 +21,6 @@ os_log_info() { } export -f os_log_info - function setup_provider_azure() { os_log_info "[executor] setting provider configuration for [${PLATFORM_TYPE}]" @@ -44,8 +43,6 @@ EOF echo "${OPENSHIFT_TESTS_EXTRA_ARGS}" > /tmp/shared/platform-args } - - function setup_provider_gcp() { os_log_info "[executor] setting provider configuration for [${PLATFORM_TYPE}]"