*: add ManagementState support to controllers

Dockerfile

@@ -11,5 +11,4 @@ COPY --from=builder /go/src/github.com/openshift/cluster-etcd-operator/bindata/b
 COPY --from=builder /go/src/github.com/openshift/cluster-etcd-operator/cluster-etcd-operator /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/cluster-etcd-operator/manifests/ /manifests/
 
-#TODO uncomment below when we want to be part of CVO
-#LABEL io.openshift.release.operator true
+LABEL io.openshift.release.operator true

Dockerfile.rhel7

@@ -11,5 +11,4 @@ COPY --from=builder /go/src/github.com/openshift/cluster-etcd-operator/bindata/b
 COPY --from=builder /go/src/github.com/openshift/cluster-etcd-operator/cluster-etcd-operator /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/cluster-etcd-operator/manifests/ /manifests/
 
-#TODO uncomment below when we want to be part of CVO 
-#LABEL io.openshift.release.operator true
+LABEL io.openshift.release.operator true

manifests/0000_12_etcd-operator_01_operator.cr.yaml

@@ -5,4 +5,4 @@ metadata:
   annotations:
     release.openshift.io/create-only: "true"
 spec:
-  managementState: Managed
+  managementState: Unmanaged

manifests/0000_12_etcd-operator_06_deployment.yaml

@@ -7,6 +7,8 @@ metadata:
     app: etcd-operator
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: etcd-operator
@@ -20,7 +22,7 @@ spec:
       containers:
       - name: operator
         image: quay.io/openshift/cluster-etcd-operator:v4.0
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8443
           name: metrics
@@ -70,5 +72,16 @@ spec:
           secretName: etcd-client
       nodeSelector:
         node-role.kubernetes.io/master: ""
+      priorityClassName: "system-cluster-critical"
       tolerations:
-      - operator: Exists 
+      - key: "node-role.kubernetes.io/master"
+        operator: "Exists"
+        effect: "NoSchedule"
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

manifests/0000_12_etcd-operator_06_static_pod_demonset.yaml

@@ -45,7 +45,7 @@ spec:
         effect: NoSchedule
       containers:
       - image: "quay.io/openshift/cluster-etcd-operator:latest"
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: etcd-staticpod
         command: ["/usr/bin/cluster-etcd-operator"]
         args:

manifests/0000_12_etcd-operator_06_static_sync_demonset.yaml

@@ -45,7 +45,7 @@ spec:
         effect: NoSchedule
       containers:
       - image: "quay.io/openshift/cluster-etcd-operator:latest"
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: etcd-staticsync
         command: ["/usr/bin/cluster-etcd-operator"]
         args:

pkg/cmd/staticpodcontroller/staticpodcontroller.go

@@ -23,7 +23,11 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
 
-	operatorclient "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorversionedclient "github.com/openshift/client-go/operator/clientset/versioned"
+	operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions"
+	"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
+	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	mcfgclientset "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -97,11 +101,16 @@ func (s *podOpts) Run() error {
 	if err != nil {
 		return err
 	}
-	operatorClient, err := operatorclient.NewForConfig(clientConfig)
+	operatorConfigClient, err := operatorversionedclient.NewForConfig(clientConfig)
 	if err != nil {
 		return err
 	}
-	etcdOperatorClient := operatorClient.Etcds()
+
+	operatorConfigInformers := operatorv1informers.NewSharedInformerFactory(operatorConfigClient, 10*time.Minute)
+	operatorClient := &operatorclient.OperatorClient{
+		Informers: operatorConfigInformers,
+		Client:    operatorConfigClient.OperatorV1(),
+	}
 
 	kubeInformerFactory := informers.NewFilteredSharedInformerFactory(clientset, 0, etcdNamespace, nil)
 	nodeName := os.Getenv("NODE_NAME")
@@ -119,7 +128,7 @@ func (s *podOpts) Run() error {
 	eventRecorder := events.NewKubeRecorder(clientset.CoreV1().Events(etcdNamespace), "static-pod-controller-"+localEtcdName, controllerRef)
 
 	staticPodController := NewStaticPodController(
-		etcdOperatorClient,
+		operatorClient,
 		kubeInformerFactory,
 		clientset,
 		clientmc,
@@ -136,7 +145,7 @@ func (s *podOpts) Run() error {
 }
 
 type StaticPodController struct {
-	etcdOperatorClient                     operatorclient.EtcdInterface
+	operatorConfigClient                   v1helpers.OperatorClient
 	podInformer                            corev1informer.SecretInformer
 	kubeInformersForOpenshiftEtcdNamespace cache.SharedIndexInformer
 	clientset                              corev1client.Interface
@@ -149,19 +158,19 @@ type StaticPodController struct {
 }
 
 func NewStaticPodController(
-	etcdOperatorClient operatorclient.EtcdInterface,
+	operatorConfigClient v1helpers.OperatorClient,
 	kubeInformersForOpenshiftEtcdNamespace informers.SharedInformerFactory,
 	clientset corev1client.Interface,
 	clientmc mcfgclientset.Interface,
 	localEtcdName string,
 	eventRecorder events.Recorder,
 ) *StaticPodController {
 	c := &StaticPodController{
-		etcdOperatorClient: etcdOperatorClient,
-		eventRecorder:      eventRecorder.WithComponentSuffix("static-pod-controller-" + localEtcdName),
-		clientset:          clientset,
-		clientmc:           clientmc,
-		localEtcdName:      localEtcdName,
+		operatorConfigClient: operatorConfigClient,
+		eventRecorder:        eventRecorder.WithComponentSuffix("static-pod-controller-" + localEtcdName),
+		clientset:            clientset,
+		clientmc:             clientmc,
+		localEtcdName:        localEtcdName,
 
 		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StaticPodController"),
 	}
@@ -172,6 +181,21 @@ func NewStaticPodController(
 }
 
 func (c *StaticPodController) sync() error {
+	operatorSpec, _, _, err := c.operatorConfigClient.GetOperatorState()
+	if err != nil {
+		return err
+	}
+	switch operatorSpec.ManagementState {
+	case operatorv1.Managed:
+	case operatorv1.Unmanaged:
+		return nil
+	case operatorv1.Removed:
+		// TODO should we support removal?
+		return nil
+	default:
+		c.eventRecorder.Warningf("ManagementStateUnknown", "Unrecognized operator management state %q", operatorSpec.ManagementState)
+		return nil
+	}
 	pod, err := c.clientset.CoreV1().Pods(etcdNamespace).Get(c.localEtcdName, metav1.GetOptions{})
 	if err != nil {
 		klog.Infof("No Pod found in %s with name %s", etcdNamespace, c.localEtcdName)
@@ -234,13 +258,13 @@ func (c *StaticPodController) IsMemberRemove(name string) bool {
 
 func (c *StaticPodController) PendingMemberList() ([]ceoapi.Member, error) {
 	configPath := []string{"cluster", "pending"}
-	operatorSpec, err := c.etcdOperatorClient.Get("cluster", metav1.GetOptions{})
+	operatorSpec, _, _, err := c.operatorConfigClient.GetOperatorState()
 	if err != nil {
 		return nil, err
 	}
 
 	config := map[string]interface{}{}
-	if err := json.NewDecoder(bytes.NewBuffer(operatorSpec.Spec.ObservedConfig.Raw)).Decode(&config); err != nil {
+	if err := json.NewDecoder(bytes.NewBuffer(operatorSpec.ObservedConfig.Raw)).Decode(&config); err != nil {
 		klog.V(4).Infof("decode of existing config failed with error: %v", err)
 	}
 	data, exists, err := unstructured.NestedSlice(config, configPath...)

pkg/cmd/staticsynccontroller/staticsynccontroller.go

@@ -7,8 +7,13 @@ import (
 	"os"
 	"time"
 
+	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorversionedclient "github.com/openshift/client-go/operator/clientset/versioned"
+	operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions"
+	"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
 	"github.com/openshift/cluster-etcd-operator/pkg/version"
 	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"k8s.io/client-go/kubernetes"
@@ -79,6 +84,16 @@ func (s *syncOpts) Run() error {
 	if err != nil {
 		return err
 	}
+	operatorConfigClient, err := operatorversionedclient.NewForConfig(clientConfig)
+	if err != nil {
+		return err
+	}
+
+	operatorConfigInformers := operatorv1informers.NewSharedInformerFactory(operatorConfigClient, 10*time.Minute)
+	operatorClient := &operatorclient.OperatorClient{
+		Informers: operatorConfigInformers,
+		Client:    operatorConfigClient.OperatorV1(),
+	}
 
 	//TODO: util v6j
 	controllerRef, err := events.GetControllerReferenceForCurrentPod(clientset, etcdNamespace, nil)
@@ -91,6 +106,7 @@ func (s *syncOpts) Run() error {
 	kubeInformerFactory := informers.NewFilteredSharedInformerFactory(clientset, 0, etcdNamespace, nil)
 
 	staticSyncController := NewStaticSyncController(
+		operatorClient,
 		kubeInformerFactory,
 		eventRecorder,
 	)
@@ -104,6 +120,7 @@ func (s *syncOpts) Run() error {
 }
 
 type StaticSyncController struct {
+	operatorConfigClient                   v1helpers.OperatorClient
 	podInformer                            corev1informer.SecretInformer
 	kubeInformersForOpenshiftEtcdNamespace cache.SharedIndexInformer
 
@@ -113,19 +130,36 @@ type StaticSyncController struct {
 }
 
 func NewStaticSyncController(
+	operatorConfigClient v1helpers.OperatorClient,
 	kubeInformersForOpenshiftEtcdNamespace informers.SharedInformerFactory,
 	eventRecorder events.Recorder,
 ) *StaticSyncController {
 	c := &StaticSyncController{
-		eventRecorder: eventRecorder.WithComponentSuffix("resource-sync-controller-" + os.Getenv("NODE_NAME")),
-		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ResourceSyncController"),
+		operatorConfigClient: operatorConfigClient,
+		eventRecorder:        eventRecorder.WithComponentSuffix("resource-sync-controller-" + os.Getenv("NODE_NAME")),
+		queue:                workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ResourceSyncController"),
 	}
 	kubeInformersForOpenshiftEtcdNamespace.Core().V1().Secrets().Informer().AddEventHandler(c.eventHandler())
 
 	return c
 }
 
 func (c *StaticSyncController) sync() error {
+	operatorSpec, _, _, err := c.operatorConfigClient.GetOperatorState()
+	if err != nil {
+		return err
+	}
+	switch operatorSpec.ManagementState {
+	case operatorv1.Managed:
+	case operatorv1.Unmanaged:
+		return nil
+	case operatorv1.Removed:
+		// TODO should we support removal?
+		return nil
+	default:
+		c.eventRecorder.Warningf("ManagementStateUnknown", "Unrecognized operator management state %q", operatorSpec.ManagementState)
+		return nil
+	}
 	// if anything changes we copy
 	assets := [4]string{
 		"namespace",

pkg/operator/bootstrapteardown/teardown.go

@@ -6,8 +6,10 @@ import (
 	"k8s.io/client-go/rest"
 
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/clustermembercontroller"
 	cov1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
+	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/tools/cache"
@@ -18,7 +20,7 @@ import (
 )
 
 func TearDownBootstrap(config *rest.Config,
-	clusterMemberShipController *clustermembercontroller.ClusterMemberController) error {
+	clusterMemberShipController *clustermembercontroller.ClusterMemberController, operatorClient v1helpers.OperatorClient) error {
 	failing := configv1.ClusterStatusConditionType("Failing")
 	var lastError string
 	var err error
@@ -53,12 +55,23 @@ func TearDownBootstrap(config *rest.Config,
 		},
 	)
 
+	operatorSpec, _, _, err := operatorClient.GetOperatorState()
+	if err != nil {
+		return err
+	}
+
 	if err == nil {
 		klog.Infof("clusterversions is available, safe to remove bootstrap")
-		if clusterMemberShipController.IsMember("etcd-bootstrap") {
-			return clusterMemberShipController.RemoveBootstrap()
+		switch operatorSpec.ManagementState {
+		case operatorv1.Managed:
+			if clusterMemberShipController.IsMember("etcd-bootstrap") {
+				return clusterMemberShipController.RemoveBootstrap()
+			}
+			break
+		case operatorv1.Unmanaged:
+			break
 		}
-		return nil
+		// TODO handle default
 	}
 
 	return nil

pkg/operator/clustermembercontroller/clustermembercontroller.go

@@ -72,6 +72,45 @@ func NewClusterMemberController(
 }
 
 func (c *ClusterMemberController) sync() error {
+	operatorSpec, _, _, err := c.operatorConfigClient.GetOperatorState()
+	if err != nil {
+		return err
+	}
+	switch operatorSpec.ManagementState {
+	case operatorv1.Managed:
+	case operatorv1.Unmanaged:
+		condUpgradable := operatorv1.OperatorCondition{
+			Type:   operatorv1.OperatorStatusTypeUpgradeable,
+			Status: operatorv1.ConditionFalse,
+		}
+		condProgressing := operatorv1.OperatorCondition{
+			Type:   operatorv1.OperatorStatusTypeProgressing,
+			Status: operatorv1.ConditionFalse,
+		}
+		condAvailable := operatorv1.OperatorCondition{
+			Type:   operatorv1.OperatorStatusTypeAvailable,
+			Status: operatorv1.ConditionTrue,
+		}
+		condDegraded := operatorv1.OperatorCondition{
+			Type:   operatorv1.OperatorStatusTypeDegraded,
+			Status: operatorv1.ConditionFalse,
+		}
+		if _, _, updateError := v1helpers.UpdateStatus(c.operatorConfigClient,
+			v1helpers.UpdateConditionFn(condUpgradable),
+			v1helpers.UpdateConditionFn(condProgressing),
+			v1helpers.UpdateConditionFn(condDegraded),
+			v1helpers.UpdateConditionFn(condAvailable)); updateError != nil {
+			return updateError
+		}
+		return nil
+	case operatorv1.Removed:
+		// TODO should we support removal?
+		return nil
+	default:
+		c.eventRecorder.Warningf("ManagementStateUnknown", "Unrecognized operator management state %q", operatorSpec.ManagementState)
+		return nil
+	}
+
 	pods, err := c.clientset.CoreV1().Pods("openshift-etcd").List(metav1.ListOptions{LabelSelector: "k8s-app=etcd"})
 	if err != nil {
 		klog.Infof("No Pod found in openshift-etcd with label k8s-app=etcd")

pkg/operator/configobservation/configobservercontroller/observe_config_controller.go

@@ -52,6 +52,7 @@ func NewConfigObserver(
 
 				ResourceSync: resourceSyncer,
 				PreRunCachesSynced: append(configMapPreRunCacheSynced,
+					operatorClient.Informer().HasSynced,
 					operatorConfigInformers.Operator().V1().Etcds().Informer().HasSynced,
 
 					kubeInformersForNamespaces.InformersFor("openshift-etcd").Core().V1().Endpoints().Informer().HasSynced,