diff --git a/pkg/framework/framework.go b/pkg/framework/framework.go index ff705ce57..f844c3796 100644 --- a/pkg/framework/framework.go +++ b/pkg/framework/framework.go @@ -120,14 +120,14 @@ func LoadClientset() (*kubernetes.Clientset, error) { return kubernetes.NewForConfig(cfg) } -func IsStatusAvailable(client runtimeclient.Client, name string) bool { +func WaitForStatusAvailable(client runtimeclient.Client, name string) bool { key := types.NamespacedName{ Namespace: MachineAPINamespace, Name: name, } clusterOperator := &configv1.ClusterOperator{} - if err := wait.PollImmediate(RetryShort, WaitShort, func() (bool, error) { + if err := wait.PollImmediate(RetryShort, 10*time.Minute, func() (bool, error) { if err := client.Get(context.TODO(), key, clusterOperator); err != nil { klog.Errorf("error querying api for OperatorStatus object: %v, retrying...", err) return false, nil @@ -152,6 +152,27 @@ func IsStatusAvailable(client runtimeclient.Client, name string) bool { return true } +func IsStatusDegraded(client runtimeclient.Client, name string) (bool, error) { + key := types.NamespacedName{ + Namespace: MachineAPINamespace, + Name: name, + } + clusterOperator := &configv1.ClusterOperator{} + if err := client.Get(context.TODO(), key, clusterOperator); err != nil { + return false, fmt.Errorf("error querying api for OperatorStatus object: %v, retrying...", err) + } + + isAvailable := cov1helpers.IsStatusConditionTrue(clusterOperator.Status.Conditions, configv1.OperatorAvailable) + isDegraded := cov1helpers.IsStatusConditionTrue(clusterOperator.Status.Conditions, configv1.OperatorDegraded) + isProgressing := cov1helpers.IsStatusConditionTrue(clusterOperator.Status.Conditions, configv1.OperatorProgressing) + + klog.Infof("Condition: %q is %v, expected true", configv1.OperatorAvailable, isAvailable) + klog.Infof("Condition: %q is %v, expected true", configv1.OperatorDegraded, isDegraded) + klog.Infof("Condition: %q is %v, expected false", configv1.OperatorProgressing, isProgressing) + + return isAvailable && isDegraded && !isProgressing, nil +} + func WaitForValidatingWebhook(client runtimeclient.Client, name string) bool { key := types.NamespacedName{Name: name} webhook := &admissionregistrationv1beta1.ValidatingWebhookConfiguration{} diff --git a/pkg/operators/cluster-autoscaler-operator.go b/pkg/operators/cluster-autoscaler-operator.go index b6db4238a..baae60eda 100644 --- a/pkg/operators/cluster-autoscaler-operator.go +++ b/pkg/operators/cluster-autoscaler-operator.go @@ -92,6 +92,6 @@ var _ = Describe("[Feature:Operators] Cluster autoscaler cluster operator status var err error client, err := framework.LoadClient() Expect(err).NotTo(HaveOccurred()) - Expect(framework.IsStatusAvailable(client, "cluster-autoscaler")).To(BeTrue()) + Expect(framework.WaitForStatusAvailable(client, "cluster-autoscaler")).To(BeTrue()) }) }) diff --git a/pkg/operators/cluster-machine-approver.go b/pkg/operators/cluster-machine-approver.go index 0b19a0630..7322995bb 100644 --- a/pkg/operators/cluster-machine-approver.go +++ b/pkg/operators/cluster-machine-approver.go @@ -26,6 +26,6 @@ var _ = Describe("[Feature:Operators] Cluster Machine Approver Cluster Operator client, err := framework.LoadClient() Expect(err).NotTo(HaveOccurred()) - Expect(framework.IsStatusAvailable(client, cmaClusterOperator)).To(BeTrue()) + Expect(framework.WaitForStatusAvailable(client, cmaClusterOperator)).To(BeTrue()) }) }) diff --git a/pkg/operators/machine-api-operator.go b/pkg/operators/machine-api-operator.go index 3df02cafa..d48e081e3 100644 --- a/pkg/operators/machine-api-operator.go +++ b/pkg/operators/machine-api-operator.go @@ -1,12 +1,16 @@ package operators import ( + "context" "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + v1 "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) var ( @@ -73,6 +77,47 @@ var _ = Describe("[Feature:Operators] Machine API cluster operator status should It("be available", func() { client, err := framework.LoadClient() Expect(err).NotTo(HaveOccurred()) - Expect(framework.IsStatusAvailable(client, "machine-api")).To(BeTrue()) + Expect(framework.WaitForStatusAvailable(client, "machine-api")).To(BeTrue()) + }) + + It("be degraded when a pod owned by the operator is prevented from being available", func() { + c, err := framework.LoadClient() + Expect(err).NotTo(HaveOccurred()) + + // https://github.com/openshift/machine-api-operator/blob/d234cceb5de18b83aa0609d17db7d835f2d78973/pkg/operator/sync.go#L310-L313 + mAPIControllersLabels := map[string]string{ + "api": "clusterapi", + "k8s-app": "controller", + } + + Eventually(func() (bool, error) { + // get machine API controllers pods + podList := &v1.PodList{} + if err := c.List(context.TODO(), podList, client.MatchingLabels(mAPIControllersLabels)); err != nil { + klog.Errorf("failed to list pods: %v", err) + return false, nil + } + if len(podList.Items) < 1 { + klog.Errorf("list of pods is empty") + return false, nil + } + + // delete machine API controllers pods + for k := range podList.Items { + if err := c.Delete(context.Background(), &podList.Items[k]); err != nil { + klog.Errorf("failed to delete pod: %v", err) + return false, nil + } + } + + isStatusDegraded, err := framework.IsStatusDegraded(c, "machine-api") + if err != nil { + return false, nil + } + + return isStatusDegraded, nil + }, framework.WaitLong, framework.RetryShort).Should(BeTrue()) + + Expect(framework.WaitForStatusAvailable(c, "machine-api")).To(BeTrue()) }) })