diff --git a/cmd/openshift-tests/e2e.go b/cmd/openshift-tests/e2e.go index 5b406b981189..1482cc3dbd64 100644 --- a/cmd/openshift-tests/e2e.go +++ b/cmd/openshift-tests/e2e.go @@ -390,7 +390,7 @@ func isStandardEarlyOrLateTest(name string) bool { // suiteWithInitializedProviderPreSuite loads the provider info, but does not // exclude any tests specific to that provider. func suiteWithInitializedProviderPreSuite(opt *runOptions) error { - config, err := decodeProvider(opt.Provider, opt.DryRun, true) + config, err := decodeProvider(opt.Provider, opt.DryRun, true, nil) if err != nil { return err } @@ -401,13 +401,13 @@ func suiteWithInitializedProviderPreSuite(opt *runOptions) error { } // suiteWithProviderPreSuite ensures that the suite filters out tests from providers -// that aren't relevant (see getProviderMatchFn) by loading the provider info from the -// cluster or flags. +// that aren't relevant (see exutilcluster.ClusterConfig.MatchFn) by loading the +// provider info from the cluster or flags. func suiteWithProviderPreSuite(opt *runOptions) error { if err := suiteWithInitializedProviderPreSuite(opt); err != nil { return err } - opt.MatchFn = getProviderMatchFn(opt.config) + opt.MatchFn = opt.config.MatchFn() return nil } diff --git a/cmd/openshift-tests/openshift-tests.go b/cmd/openshift-tests/openshift-tests.go index 3deea3866612..76c5d83e1788 100644 --- a/cmd/openshift-tests/openshift-tests.go +++ b/cmd/openshift-tests/openshift-tests.go @@ -29,7 +29,7 @@ import ( testginkgo "github.com/openshift/origin/pkg/test/ginkgo" "github.com/openshift/origin/pkg/version" exutil "github.com/openshift/origin/test/extended/util" - "github.com/openshift/origin/test/extended/util/cloud" + "github.com/openshift/origin/test/extended/util/cluster" ) func main() { @@ -204,7 +204,7 @@ type runOptions struct { TestOptions []string // Shared by initialization code - config *cloud.ClusterConfiguration + config *cluster.ClusterConfiguration } func (opt *runOptions) AsEnv() []string { @@ -406,7 +406,7 @@ func newRunTestCommand() *cobra.Command { ginkgo.GlobalSuite().ClearBeforeSuiteNode() ginkgo.GlobalSuite().ClearAfterSuiteNode() - config, err := decodeProvider(os.Getenv("TEST_PROVIDER"), testOpt.DryRun, false) + config, err := decodeProvider(os.Getenv("TEST_PROVIDER"), testOpt.DryRun, false, nil) if err != nil { return err } diff --git a/cmd/openshift-tests/provider.go b/cmd/openshift-tests/provider.go index 5824ae10f3e0..84cd6fe5dcc2 100644 --- a/cmd/openshift-tests/provider.go +++ b/cmd/openshift-tests/provider.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "os" - "strings" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -13,7 +12,7 @@ import ( e2e "k8s.io/kubernetes/test/e2e/framework" exutil "github.com/openshift/origin/test/extended/util" - exutilcloud "github.com/openshift/origin/test/extended/util/cloud" + exutilcluster "github.com/openshift/origin/test/extended/util/cluster" // Initialize baremetal as a provider _ "github.com/openshift/origin/test/extended/util/baremetal" @@ -29,9 +28,7 @@ import ( _ "k8s.io/kubernetes/test/e2e/lifecycle" ) -type TestNameMatchesFunc func(name string) bool - -func initializeTestFramework(context *e2e.TestContextType, config *exutilcloud.ClusterConfiguration, dryRun bool) error { +func initializeTestFramework(context *e2e.TestContextType, config *exutilcluster.ClusterConfiguration, dryRun bool) error { // update context with loaded config context.Provider = config.ProviderName context.CloudConfig = e2e.CloudConfig{ @@ -67,51 +64,34 @@ func initializeTestFramework(context *e2e.TestContextType, config *exutilcloud.C return nil } -func getProviderMatchFn(config *exutilcloud.ClusterConfiguration) TestNameMatchesFunc { - // given the configuration we have loaded, skip tests that our provider, disconnected status, - // or our network plugin should exclude - var skips []string - skips = append(skips, fmt.Sprintf("[Skipped:%s]", config.ProviderName)) - for _, id := range config.NetworkPluginIDs { - skips = append(skips, fmt.Sprintf("[Skipped:Network/%s]", id)) - } - if config.Disconnected { - skips = append(skips, "[Skipped:Disconnected]") - } - - matchFn := func(name string) bool { - for _, skip := range skips { - if strings.Contains(name, skip) { - return false - } - } - return true - } - return matchFn -} - -func decodeProvider(provider string, dryRun, discover bool) (*exutilcloud.ClusterConfiguration, error) { +func decodeProvider(provider string, dryRun, discover bool, clusterState *exutilcluster.ClusterState) (*exutilcluster.ClusterConfiguration, error) { switch provider { case "none": - return &exutilcloud.ClusterConfiguration{ProviderName: "skeleton"}, nil + return &exutilcluster.ClusterConfiguration{ProviderName: "skeleton"}, nil case "": if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok { if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok { - return &exutilcloud.ClusterConfiguration{ProviderName: "local"}, nil + return &exutilcluster.ClusterConfiguration{ProviderName: "local"}, nil } } if dryRun { - return &exutilcloud.ClusterConfiguration{ProviderName: "skeleton"}, nil + return &exutilcluster.ClusterConfiguration{ProviderName: "skeleton"}, nil } fallthrough case "azure", "aws", "baremetal", "gce", "vsphere": - clientConfig, err := e2e.LoadConfig(true) - if err != nil { - return nil, err + if clusterState == nil { + clientConfig, err := e2e.LoadConfig(true) + if err != nil { + return nil, err + } + clusterState, err = exutilcluster.DiscoverClusterState(clientConfig) + if err != nil { + return nil, err + } } - config, err := exutilcloud.LoadConfig(clientConfig) + config, err := exutilcluster.LoadConfig(clusterState) if err != nil { return nil, err } @@ -122,58 +102,31 @@ func decodeProvider(provider string, dryRun, discover bool) (*exutilcloud.Cluste default: var providerInfo struct { - Type string - Disconnected bool - e2e.CloudConfig `json:",inline"` + Type string } if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil { - return nil, fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum, and decode into a cloud config object: %v", err) + return nil, fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err) } if len(providerInfo.Type) == 0 { return nil, fmt.Errorf("provider must be a JSON object with the 'type' key") } - // attempt to load the default config, then overwrite with any values from the passed - // object that can be overridden - var config *exutilcloud.ClusterConfiguration + var config *exutilcluster.ClusterConfiguration if discover { - if clientConfig, err := e2e.LoadConfig(true); err == nil { - config, _ = exutilcloud.LoadConfig(clientConfig) + if clusterState == nil { + if clientConfig, err := e2e.LoadConfig(true); err == nil { + clusterState, _ = exutilcluster.DiscoverClusterState(clientConfig) + } } - } - if config == nil { - config = &exutilcloud.ClusterConfiguration{ - ProviderName: providerInfo.Type, - ProjectID: providerInfo.ProjectID, - Region: providerInfo.Region, - Zone: providerInfo.Zone, - Zones: providerInfo.Zones, - NumNodes: providerInfo.NumNodes, - MultiMaster: providerInfo.MultiMaster, - MultiZone: providerInfo.MultiZone, - ConfigFile: providerInfo.ConfigFile, + if clusterState != nil { + config, _ = exutilcluster.LoadConfig(clusterState) } } else { - config.ProviderName = providerInfo.Type - if len(providerInfo.ProjectID) > 0 { - config.ProjectID = providerInfo.ProjectID - } - if len(providerInfo.Region) > 0 { - config.Region = providerInfo.Region - } - if len(providerInfo.Zone) > 0 { - config.Zone = providerInfo.Zone - } - if len(providerInfo.Zones) > 0 { - config.Zones = providerInfo.Zones - } - if len(providerInfo.ConfigFile) > 0 { - config.ConfigFile = providerInfo.ConfigFile - } - if providerInfo.NumNodes > 0 { - config.NumNodes = providerInfo.NumNodes - } + config = &exutilcluster.ClusterConfiguration{} + } + + if err := json.Unmarshal([]byte(provider), config); err != nil { + return nil, fmt.Errorf("provider must decode into the ClusterConfig object: %v", err) } - config.Disconnected = providerInfo.Disconnected return config, nil } } diff --git a/cmd/openshift-tests/provider_test.go b/cmd/openshift-tests/provider_test.go new file mode 100644 index 000000000000..2c558449f26e --- /dev/null +++ b/cmd/openshift-tests/provider_test.go @@ -0,0 +1,280 @@ +package main + +import ( + "os" + "testing" + + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + exutilcluster "github.com/openshift/origin/test/extended/util/cluster" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +var gcePlatform = &configv1.PlatformStatus{ + Type: configv1.GCPPlatformType, + GCP: &configv1.GCPPlatformStatus{ + ProjectID: "openshift-gce-devel-ci", + Region: "us-east1", + }, +} + +var awsPlatform = &configv1.PlatformStatus{ + Type: configv1.AWSPlatformType, + AWS: &configv1.AWSPlatformStatus{ + Region: "us-east-2", + }, +} + +var vspherePlatform = &configv1.PlatformStatus{ + Type: configv1.VSpherePlatformType, +} + +var noPlatform = &configv1.PlatformStatus{ + Type: configv1.NonePlatformType, +} + +var gceMasters = &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-1", + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "us-east1-a", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-2", + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "us-east1-b", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-3", + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "us-east1-c", + }, + }, + }, + }, +} + +var simpleMasters = &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-2", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-3", + }, + }, + }, +} + +var nonMasters = &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-2", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-3", + }, + }, + }, +} + +var sdnConfig = &operatorv1.NetworkSpec{ + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + Type: operatorv1.NetworkTypeOpenShiftSDN, + OpenShiftSDNConfig: &operatorv1.OpenShiftSDNConfig{}, + }, + ServiceNetwork: []string{"172.30.0.0/16"}, +} + +var multitenantConfig = &operatorv1.NetworkSpec{ + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + Type: operatorv1.NetworkTypeOpenShiftSDN, + OpenShiftSDNConfig: &operatorv1.OpenShiftSDNConfig{ + Mode: operatorv1.SDNModeMultitenant, + }, + }, + ServiceNetwork: []string{"172.30.0.0/16"}, +} + +var ovnKubernetesConfig = &operatorv1.NetworkSpec{ + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + Type: operatorv1.NetworkTypeOVNKubernetes, + }, + ServiceNetwork: []string{ + "172.30.0.0/16", + "fd02::/112", + }, +} + +var e2eTests = map[string]string{ + "everyone": "[Skipped:Wednesday]", + "not-gce": "[Skipped:gce]", + "not-aws": "[Skipped:aws]", + "not-sdn": "[Skipped:Network/OpenShiftSDN]", + "not-multitenant": "[Skipped:Network/OpenShiftSDN/Multitenant]", + "online": "[Skipped:Disconnected]", + "ipv4": "[Feature:Networking-IPv4]", + "ipv6": "[Feature:Networking-IPv6]", + "dual-stack": "[Feature:IPv6DualStackAlpha]", + "sctp": "[Feature:SCTPConnectivity]", +} + +func TestDecodeProvider(t *testing.T) { + var testCases = []struct { + name string + provider string + + discoveredPlatform *configv1.PlatformStatus + discoveredMasters *corev1.NodeList + discoveredNetwork *operatorv1.NetworkSpec + + expectedConfig string + runTests sets.String + }{ + { + name: "simple GCE", + provider: "", + discoveredPlatform: gcePlatform, + discoveredMasters: gceMasters, + discoveredNetwork: sdnConfig, + expectedConfig: `{"type":"gce","ProjectID":"openshift-gce-devel-ci","Region":"us-east1","Zone":"us-east1-a","NumNodes":3,"MultiMaster":true,"MultiZone":true,"Zones":["us-east1-a","us-east1-b","us-east1-c"],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"OpenShiftSDN","HasIPv4":true,"HasIPv6":false,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-aws", "not-multitenant", "online", "ipv4"), + }, + { + name: "GCE multitenant", + provider: "", + discoveredPlatform: gcePlatform, + discoveredMasters: gceMasters, + discoveredNetwork: multitenantConfig, + expectedConfig: `{"type":"gce","ProjectID":"openshift-gce-devel-ci","Region":"us-east1","Zone":"us-east1-a","NumNodes":3,"MultiMaster":true,"MultiZone":true,"Zones":["us-east1-a","us-east1-b","us-east1-c"],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"OpenShiftSDN","NetworkPluginMode":"Multitenant","HasIPv4":true,"HasIPv6":false,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-aws", "online", "ipv4"), + }, + { + name: "simple non-cloud", + provider: "", + discoveredPlatform: noPlatform, + discoveredMasters: simpleMasters, + discoveredNetwork: sdnConfig, + expectedConfig: `{"type":"skeleton","ProjectID":"","Region":"","Zone":"","NumNodes":3,"MultiMaster":true,"MultiZone":false,"Zones":[],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"OpenShiftSDN","HasIPv4":true,"HasIPv6":false,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-gce", "not-aws", "not-multitenant", "online", "ipv4"), + }, + { + name: "simple override", + provider: "vsphere", + discoveredPlatform: vspherePlatform, + discoveredMasters: simpleMasters, + discoveredNetwork: sdnConfig, + // NB: It does not actually use the passed-in Provider value + expectedConfig: `{"type":"skeleton","ProjectID":"","Region":"","Zone":"","NumNodes":3,"MultiMaster":true,"MultiZone":false,"Zones":[],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"OpenShiftSDN","HasIPv4":true,"HasIPv6":false,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-gce", "not-aws", "not-multitenant", "online", "ipv4"), + }, + { + name: "json simple override", + provider: `{"type": "openstack"}`, + discoveredPlatform: noPlatform, + discoveredMasters: simpleMasters, + discoveredNetwork: sdnConfig, + expectedConfig: `{"type":"openstack","ProjectID":"","Region":"","Zone":"","NumNodes":3,"MultiMaster":true,"MultiZone":false,"Zones":[],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"OpenShiftSDN","HasIPv4":true,"HasIPv6":false,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-gce", "not-aws", "not-multitenant", "online", "ipv4"), + }, + { + name: "complex override, dual-stack", + provider: `{"type":"aws","region":"us-east-2","zone":"us-east-2a","multimaster":false,"multizone":true}`, + discoveredPlatform: awsPlatform, + discoveredMasters: simpleMasters, + discoveredNetwork: ovnKubernetesConfig, + expectedConfig: `{"type":"aws","ProjectID":"","Region":"us-east-2","Zone":"us-east-2a","NumNodes":3,"MultiMaster":false,"MultiZone":true,"Zones":[],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"OVNKubernetes","HasIPv4":true,"HasIPv6":true,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-gce", "not-sdn", "not-multitenant", "online", "ipv4", "ipv6", "dual-stack"), + }, + { + name: "complex override without discovery", + provider: `{"type":"aws","region":"us-east-2","zone":"us-east-2a","multimaster":false,"multizone":true}`, + discoveredPlatform: nil, + expectedConfig: `{"type":"aws","ProjectID":"","Region":"us-east-2","Zone":"us-east-2a","NumNodes":0,"MultiMaster":false,"MultiZone":true,"Zones":null,"ConfigFile":"","Disconnected":false,"NetworkPlugin":"","HasIPv4":false,"HasIPv6":false,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-gce", "not-sdn", "not-multitenant", "online"), + }, + { + name: "disconnected", + provider: `{"type":"none","disconnected":true}`, + discoveredPlatform: noPlatform, + discoveredMasters: simpleMasters, + discoveredNetwork: ovnKubernetesConfig, + expectedConfig: `{"type":"none","ProjectID":"","Region":"","Zone":"","NumNodes":3,"MultiMaster":true,"MultiZone":false,"Zones":[],"ConfigFile":"","Disconnected":true,"NetworkPlugin":"OVNKubernetes","HasIPv4":true,"HasIPv6":true,"HasSCTP":false}`, + runTests: sets.NewString("everyone", "not-gce", "not-aws", "not-sdn", "not-multitenant", "ipv4", "ipv6", "dual-stack"), + }, + { + name: "override network plugin", + provider: `{"type":"aws","networkPlugin":"Calico","hasIPv4":false,"hasIPv6":true,"hasSCTP":true}`, + discoveredPlatform: awsPlatform, + discoveredMasters: simpleMasters, + discoveredNetwork: ovnKubernetesConfig, + expectedConfig: `{"type":"aws","ProjectID":"","Region":"us-east-2","Zone":"","NumNodes":3,"MultiMaster":true,"MultiZone":false,"Zones":[],"ConfigFile":"","Disconnected":false,"NetworkPlugin":"Calico","HasIPv4":false,"HasIPv6":true,"HasSCTP":true}`, + runTests: sets.NewString("everyone", "not-gce", "not-sdn", "not-multitenant", "online", "ipv6", "sctp"), + }, + } + + // Unset these to keep decodeProvider from returning "local" + os.Unsetenv("KUBE_SSH_USER") + os.Unsetenv("LOCAL_SSH_KEY") + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + discover := tc.discoveredPlatform != nil + var testState *exutilcluster.ClusterState + if discover { + testState = &exutilcluster.ClusterState{ + PlatformStatus: tc.discoveredPlatform, + Masters: tc.discoveredMasters, + NonMasters: nonMasters, + NetworkSpec: tc.discoveredNetwork, + } + } + config, err := decodeProvider(tc.provider, false, discover, testState) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + configJSON := config.ToJSONString() + if configJSON != tc.expectedConfig { + t.Fatalf("Generated config:\n%s\ndoes not match expected:\n%s\n", configJSON, tc.expectedConfig) + } + matchFn := config.MatchFn() + + runTests := sets.NewString() + for name, tags := range e2eTests { + if matchFn(name + " " + tags) { + runTests.Insert(name) + } + } + if !runTests.Equal(tc.runTests) { + t.Fatalf("Matched tests:\n%v\ndid not match expected:\n%v\n", runTests.List(), tc.runTests.List()) + } + }) + } +} diff --git a/go.mod b/go.mod index 0c28bc3b882d..b6e6ef58f327 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( k8s.io/kubelet v0.20.0 k8s.io/kubernetes v1.20.0 k8s.io/legacy-cloud-providers v0.20.0 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 sigs.k8s.io/yaml v1.2.0 ) diff --git a/test/extended/util/cloud/cloud.go b/test/extended/util/cloud/cloud.go deleted file mode 100644 index 264dbc478054..000000000000 --- a/test/extended/util/cloud/cloud.go +++ /dev/null @@ -1,149 +0,0 @@ -package cloud - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - - configv1 "github.com/openshift/api/config/v1" - configclient "github.com/openshift/client-go/config/clientset/versioned" - operatorclient "github.com/openshift/client-go/operator/clientset/versioned" - "github.com/openshift/origin/test/extended/util/azure" -) - -type ClusterConfiguration struct { - ProviderName string `json:"type"` - - // These fields chosen to match the e2e configuration we fill - ProjectID string - Region string - Zone string - NumNodes int - MultiMaster bool - MultiZone bool - Zones []string - ConfigFile string - - // Network-related configurations - Disconnected bool - NetworkPluginIDs []string -} - -func (c *ClusterConfiguration) ToJSONString() string { - out, err := json.Marshal(c) - if err != nil { - panic(err) - } - return string(out) -} - -// LoadConfig uses the cluster to setup the cloud provider config. -func LoadConfig(clientConfig *rest.Config) (*ClusterConfiguration, error) { - // LoadClientset but don't set the UserAgent to include the current test name because - // we don't run any test yet and this call panics - coreClient, err := clientset.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - configClient, err := configclient.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - operatorClient, err := operatorclient.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - - var networkPluginIDs []string - if networkConfig, err := operatorClient.OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{}); err == nil { - networkPluginIDs = append(networkPluginIDs, string(networkConfig.Spec.DefaultNetwork.Type)) - if networkConfig.Spec.DefaultNetwork.OpenShiftSDNConfig != nil && networkConfig.Spec.DefaultNetwork.OpenShiftSDNConfig.Mode != "" { - networkPluginIDs = append(networkPluginIDs, string(networkConfig.Spec.DefaultNetwork.Type)+"/"+string(networkConfig.Spec.DefaultNetwork.OpenShiftSDNConfig.Mode)) - } - - } - - infra, err := configClient.ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) - if err != nil { - return nil, err - } - p := infra.Status.PlatformStatus - if p == nil { - return nil, fmt.Errorf("status.platformStatus must be set") - } - if p.Type == configv1.NonePlatformType { - return &ClusterConfiguration{ - NetworkPluginIDs: networkPluginIDs, - }, nil - } - - masters, err := coreClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ - LabelSelector: "node-role.kubernetes.io/master=", - }) - if err != nil { - return nil, err - } - zones := sets.NewString() - for _, node := range masters.Items { - zones.Insert(node.Labels["failure-domain.beta.kubernetes.io/zone"]) - } - zones.Delete("") - - nonMasters, err := coreClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ - LabelSelector: "!node-role.kubernetes.io/master", - }) - if err != nil { - return nil, err - } - - config := &ClusterConfiguration{ - MultiMaster: len(masters.Items) > 1, - MultiZone: zones.Len() > 1, - Zones: zones.List(), - NetworkPluginIDs: networkPluginIDs, - } - if zones.Len() > 0 { - config.Zone = zones.List()[0] - } - if len(nonMasters.Items) == 0 { - config.NumNodes = len(nonMasters.Items) - } else { - config.NumNodes = len(masters.Items) - } - - switch { - case p.AWS != nil: - config.ProviderName = "aws" - config.Region = p.AWS.Region - - case p.GCP != nil: - config.ProviderName = "gce" - config.ProjectID = p.GCP.ProjectID - config.Region = p.GCP.Region - - case p.Azure != nil: - config.ProviderName = "azure" - - data, err := azure.LoadConfigFile() - if err != nil { - return nil, err - } - tmpFile, err := ioutil.TempFile("", "e2e-*") - if err != nil { - return nil, err - } - tmpFile.Close() - if err := ioutil.WriteFile(tmpFile.Name(), data, 0600); err != nil { - return nil, err - } - config.ConfigFile = tmpFile.Name() - } - - return config, nil -} diff --git a/test/extended/util/cluster/cluster.go b/test/extended/util/cluster/cluster.go new file mode 100644 index 000000000000..b4459b86b1e1 --- /dev/null +++ b/test/extended/util/cluster/cluster.go @@ -0,0 +1,234 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + utilnet "k8s.io/utils/net" + + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + configclient "github.com/openshift/client-go/config/clientset/versioned" + operatorclient "github.com/openshift/client-go/operator/clientset/versioned" + "github.com/openshift/origin/test/extended/util/azure" +) + +type ClusterConfiguration struct { + ProviderName string `json:"type"` + + // These fields (and the "type" tag for ProviderName) chosen to match + // upstream's e2e.CloudConfig. + ProjectID string + Region string + Zone string + NumNodes int + MultiMaster bool + MultiZone bool + Zones []string + ConfigFile string + + // Disconnected is set for test jobs without external internet connectivity + Disconnected bool + + // NetworkPlugin is the "official" plugin name + NetworkPlugin string + // NetworkPluginMode is an optional sub-identifier for the NetworkPlugin. + // (Currently it is only used for OpenShiftSDN.) + NetworkPluginMode string `json:",omitempty"` + + // HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific, + // and dual-stack-specific tests are run + HasIPv4 bool + HasIPv6 bool + + // HasSCTP determines whether SCTP connectivity tests can be run in the cluster + HasSCTP bool +} + +func (c *ClusterConfiguration) ToJSONString() string { + out, err := json.Marshal(c) + if err != nil { + panic(err) + } + return string(out) +} + +// ClusterState provides information about the cluster that is used to generate +// ClusterConfiguration +type ClusterState struct { + PlatformStatus *configv1.PlatformStatus + Masters *corev1.NodeList + NonMasters *corev1.NodeList + NetworkSpec *operatorv1.NetworkSpec +} + +// DiscoverClusterState creates a ClusterState based on a live cluster +func DiscoverClusterState(clientConfig *rest.Config) (*ClusterState, error) { + coreClient, err := clientset.NewForConfig(clientConfig) + if err != nil { + return nil, err + } + configClient, err := configclient.NewForConfig(clientConfig) + if err != nil { + return nil, err + } + operatorClient, err := operatorclient.NewForConfig(clientConfig) + if err != nil { + return nil, err + } + + state := &ClusterState{} + + infra, err := configClient.ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) + if err != nil { + return nil, err + } + state.PlatformStatus = infra.Status.PlatformStatus + if state.PlatformStatus == nil { + return nil, fmt.Errorf("status.platformStatus must be set") + } + + state.Masters, err = coreClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/master=", + }) + if err != nil { + return nil, err + } + + state.NonMasters, err = coreClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ + LabelSelector: "!node-role.kubernetes.io/master", + }) + if err != nil { + return nil, err + } + + networkConfig, err := operatorClient.OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{}) + if err != nil { + return nil, err + } + state.NetworkSpec = &networkConfig.Spec + + return state, nil +} + +// LoadConfig generates a ClusterConfiguration based on a detected or hard-coded ClusterState +func LoadConfig(state *ClusterState) (*ClusterConfiguration, error) { + zones := sets.NewString() + for _, node := range state.Masters.Items { + zones.Insert(node.Labels["failure-domain.beta.kubernetes.io/zone"]) + } + zones.Delete("") + + config := &ClusterConfiguration{ + MultiMaster: len(state.Masters.Items) > 1, + MultiZone: zones.Len() > 1, + Zones: zones.List(), + } + if zones.Len() > 0 { + config.Zone = zones.List()[0] + } + if len(state.NonMasters.Items) == 0 { + config.NumNodes = len(state.NonMasters.Items) + } else { + config.NumNodes = len(state.Masters.Items) + } + + switch { + case state.PlatformStatus.AWS != nil: + config.ProviderName = "aws" + config.Region = state.PlatformStatus.AWS.Region + + case state.PlatformStatus.GCP != nil: + config.ProviderName = "gce" + config.ProjectID = state.PlatformStatus.GCP.ProjectID + config.Region = state.PlatformStatus.GCP.Region + + case state.PlatformStatus.Azure != nil: + config.ProviderName = "azure" + + data, err := azure.LoadConfigFile() + if err != nil { + return nil, err + } + tmpFile, err := ioutil.TempFile("", "e2e-*") + if err != nil { + return nil, err + } + tmpFile.Close() + if err := ioutil.WriteFile(tmpFile.Name(), data, 0600); err != nil { + return nil, err + } + config.ConfigFile = tmpFile.Name() + } + + config.NetworkPlugin = string(state.NetworkSpec.DefaultNetwork.Type) + if state.NetworkSpec.DefaultNetwork.OpenShiftSDNConfig != nil && state.NetworkSpec.DefaultNetwork.OpenShiftSDNConfig.Mode != "" { + config.NetworkPluginMode = string(state.NetworkSpec.DefaultNetwork.OpenShiftSDNConfig.Mode) + } + + // Determine IP configuration + for _, cidr := range state.NetworkSpec.ServiceNetwork { + if utilnet.IsIPv6CIDRString(cidr) { + config.HasIPv6 = true + } else { + config.HasIPv4 = true + } + } + + // FIXME: detect SCTP availability; there's no explicit config for it, so we'd + // have to scan MachineConfig objects to figure this out? For now, callers can + // can just manually override with --provider... + + return config, nil +} + +// MatchFn returns a function that tests if a named function should be run based on +// the cluster configuration +func (c *ClusterConfiguration) MatchFn() func(string) bool { + var skips []string + skips = append(skips, fmt.Sprintf("[Skipped:%s]", c.ProviderName)) + + if c.NetworkPlugin != "" { + skips = append(skips, fmt.Sprintf("[Skipped:Network/%s]", c.NetworkPlugin)) + if c.NetworkPluginMode != "" { + skips = append(skips, fmt.Sprintf("[Skipped:Network/%s/%s]", c.NetworkPlugin, c.NetworkPluginMode)) + } + } + + if c.Disconnected { + skips = append(skips, "[Skipped:Disconnected]") + } + + if !c.HasIPv4 { + skips = append(skips, "[Feature:Networking-IPv4]") + } + if !c.HasIPv6 { + skips = append(skips, "[Feature:Networking-IPv6]") + } + if !c.HasIPv4 || !c.HasIPv6 { + // lack of "]" is intentional; this matches multiple tags + skips = append(skips, "[Feature:IPv6DualStack") + } + + if !c.HasSCTP { + skips = append(skips, "[Feature:SCTPConnectivity]") + } + + matchFn := func(name string) bool { + for _, skip := range skips { + if strings.Contains(name, skip) { + return false + } + } + return true + } + return matchFn +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b9175ee6106b..d0cfeb95e406 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2870,6 +2870,7 @@ k8s.io/mount-utils k8s.io/sample-apiserver/pkg/apis/wardle k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1 # k8s.io/utils v0.0.0-20201110183641-67b214c5f920 +## explicit k8s.io/utils/buffer k8s.io/utils/clock k8s.io/utils/exec