[fcos] Cherry-pick commits to enable cluster-etcd-operator

cmd/machine-config-operator/start.go

@@ -5,6 +5,9 @@ import (
 	"flag"
 
 	"github.com/golang/glog"
+	operatorclientset "github.com/openshift/client-go/operator/clientset/versioned"
+	operatorinformers "github.com/openshift/client-go/operator/informers/externalversions"
+	operatorv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
 	"github.com/openshift/machine-config-operator/cmd/common"
 	"github.com/openshift/machine-config-operator/internal/clients"
 	controllercommon "github.com/openshift/machine-config-operator/pkg/controller/common"
@@ -51,6 +54,14 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 	}
 	run := func(ctx context.Context) {
 		ctrlctx := controllercommon.CreateControllerContext(cb, ctx.Done(), componentNamespace)
+		operatorClient := cb.OperatorClientOrDie("operator-shared-informer")
+
+		etcdInformer, err := getEtcdInformer(operatorClient, ctrlctx.OperatorInformerFactory)
+		if err != nil {
+			// MCO pod needs to restart for transient apiserver errors
+			glog.Errorf("unable to query discovery API %#v", err)
+			controllercommon.WriteTerminationError(err)
+		}
 
 		controller := operator.New(
 			componentNamespace, componentName,
@@ -74,6 +85,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 			ctrlctx.ClientBuilder.APIExtClientOrDie(componentName),
 			ctrlctx.ClientBuilder.ConfigClientOrDie(componentName),
 			ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
+			etcdInformer,
 		)
 
 		ctrlctx.NamespacedInformerFactory.Start(ctrlctx.Stop)
@@ -82,6 +94,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 		ctrlctx.APIExtInformerFactory.Start(ctrlctx.Stop)
 		ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop)
 		ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Start(ctrlctx.Stop)
+		ctrlctx.OperatorInformerFactory.Start(ctrlctx.Stop)
 		close(ctrlctx.InformersStarted)
 
 		go controller.Run(2, ctrlctx.Stop)
@@ -103,3 +116,18 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 	})
 	panic("unreachable")
 }
+
+func getEtcdInformer(operatorClient operatorclientset.Interface, operatorSharedInformer operatorinformers.SharedInformerFactory) (operatorv1.EtcdInformer, error) {
+	operatorGroups, err := operatorClient.Discovery().ServerResourcesForGroupVersion("operator.openshift.io/v1")
+	if err != nil {
+		glog.Errorf("unable to get operatorGroups: %#v", err)
+		return nil, err
+	}
+
+	for _, o := range operatorGroups.APIResources {
+		if o.Kind == "Etcd" {
+			return operatorSharedInformer.Operator().V1().Etcds(), nil
+		}
+	}
+	return nil, nil
+}

cmd/setup-etcd-environment/run.go

@@ -193,7 +193,9 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 
 func (s *SetupEnv) getExportEnv() (map[string]string, error) {
 	// if etcd is managed by the operator populate ENV from configmap
-	if s.isEtcdScaling() {
+	// TODO: alaypatel07/hexfusion figure out a better way of implementing isScaling
+	// if s.isEtcdScaling() {
+	if inCluster() && !s.opts.bootstrapSRV {
 		return s.getOperatorEnv()
 	}
 	// initialize envs used to bootstrap etcd using SRV discovery or disaster recovery.
@@ -315,6 +317,7 @@ func (s *SetupEnv) getBootstrapEnv() (map[string]string, error) {
 // `cluster-etcd-operator` and returns true if yes. If false we can conclude that the member is already
 // part of the cluster or we have/are bootstrapping etcd using SRV discovery.
 func (s *SetupEnv) isEtcdScaling() bool {
+	// TODO: reimplement me!
 	if _, err := os.Stat(fmt.Sprintf("%s/member", s.etcdDataDir)); os.IsNotExist(err) && !s.opts.bootstrapSRV && inCluster() {
 		return true
 	}
@@ -339,6 +342,7 @@ func (s *SetupEnv) setClient() error {
 			return fmt.Errorf("error creating client: %v", err)
 		}
 		s.Client = client
+		return nil
 	}
 	glog.Infof("no client set")
 	return nil

install/0000_80_machine-config-operator_02_images.configmap.yaml

@@ -11,7 +11,7 @@ data:
       "etcd": "registry.svc.ci.openshift.org/openshift:etcd",
       "infraImage": "registry.svc.ci.openshift.org/openshift:pod",
       "kubeClientAgentImage": "registry.svc.ci.openshift.org/openshift:kube-client-agent",
-      "clusterEtcdOperatorImage": "",
+      "clusterEtcdOperatorImage": "registry.svc.ci.openshift.org/openshift:cluster-etcd-operator",
       "keepalivedImage": "registry.svc.ci.openshift.org/openshift:keepalived-ipfailover",
       "corednsImage": "registry.svc.ci.openshift.org/openshift:coredns",
       "mdnsPublisherImage": "registry.svc.ci.openshift.org/openshift:mdns-publisher",

install/image-references

@@ -31,6 +31,10 @@ spec:
     from:
       kind: DockerImage
       name: registry.svc.ci.openshift.org/openshift:kube-client-agent
+  - name: cluster-etcd-operator
+    from:
+      kind: DockerImage
+      name: registry.svc.ci.openshift.org/openshift:cluster-etcd-operator
   - name: cli
     from:
       kind: DockerImage

manifests/machineconfigcontroller/clusterrole.yaml

@@ -22,3 +22,6 @@ rules:
 - apiGroups: ["operator.openshift.io"]
   resources: ["imagecontentsourcepolicies"]
   verbs: ["get", "list", "watch"]
+- apiGroups: ["operator.openshift.io"]
+  resources: ["etcds"]
+  verbs: ["get", "list", "watch"]

pkg/operator/assets/bindata.go

@@ -867,6 +867,9 @@ rules:
 - apiGroups: ["operator.openshift.io"]
   resources: ["imagecontentsourcepolicies"]
   verbs: ["get", "list", "watch"]
+- apiGroups: ["operator.openshift.io"]
+  resources: ["etcds"]
+  verbs: ["get", "list", "watch"]
 `)
 
 func manifestsMachineconfigcontrollerClusterroleYamlBytes() ([]byte, error) {

pkg/operator/operator.go

@@ -8,6 +8,8 @@ import (
 	"github.com/golang/glog"
 
 	configclientset "github.com/openshift/client-go/config/clientset/versioned"
+	operatorv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
+	operatorlisterv1 "github.com/openshift/client-go/operator/listers/operator/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiextclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apiextinformersv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1beta1"
@@ -23,6 +25,7 @@ import (
 	coreclientsetv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	appslisterv1 "k8s.io/client-go/listers/apps/v1"
 	corelisterv1 "k8s.io/client-go/listers/core/v1"
+
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
@@ -79,6 +82,7 @@ type Operator struct {
 	clusterCmLister  corelisterv1.ConfigMapLister
 	proxyLister      configlistersv1.ProxyLister
 	oseKubeAPILister corelisterv1.ConfigMapLister
+	etcdLister       operatorlisterv1.EtcdLister
 
 	crdListerSynced                  cache.InformerSynced
 	deployListerSynced               cache.InformerSynced
@@ -95,6 +99,7 @@ type Operator struct {
 	clusterRoleBindingInformerSynced cache.InformerSynced
 	proxyListerSynced                cache.InformerSynced
 	oseKubeAPIListerSynced           cache.InformerSynced
+	etcdSynced                       cache.InformerSynced
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
 	queue workqueue.RateLimitingInterface
@@ -126,6 +131,7 @@ func New(
 	apiExtClient apiextclientset.Interface,
 	configClient configclientset.Interface,
 	oseKubeAPIInformer coreinformersv1.ConfigMapInformer,
+	etcdInformer operatorv1.EtcdInformer,
 ) *Operator {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(glog.Infof)
@@ -192,6 +198,17 @@ func New(
 	optr.infraListerSynced = infraInformer.Informer().HasSynced
 	optr.networkLister = networkInformer.Lister()
 	optr.networkListerSynced = networkInformer.Informer().HasSynced
+	if etcdInformer != nil {
+		optr.etcdLister = etcdInformer.Lister()
+		optr.etcdSynced = etcdInformer.Informer().HasSynced
+		etcdInformer.Informer().AddEventHandler(optr.eventHandler())
+	} else {
+		optr.etcdLister = nil
+		optr.etcdSynced = func() bool {
+			// if etcd is not part of CVO it will return true immediately
+			return true
+		}
+	}
 
 	optr.vStore.Set("operator", os.Getenv("RELEASE_VERSION"))
 
@@ -226,7 +243,8 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) {
 		optr.clusterRoleBindingInformerSynced,
 		optr.networkListerSynced,
 		optr.proxyListerSynced,
-		optr.oseKubeAPIListerSynced) {
+		optr.oseKubeAPIListerSynced,
+		optr.etcdSynced) {
 		glog.Error("failed to sync caches")
 		return
 	}

pkg/operator/sync.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
 	"github.com/openshift/machine-config-operator/lib/resourceapply"
 	"github.com/openshift/machine-config-operator/lib/resourceread"
 	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
@@ -206,6 +207,11 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error {
 		spec.CloudProviderConfig = cc
 	}
 
+	//TODO: alaypatel07 remove after cluster-etcd-operator deployed via CVO as Managed
+	if err = optr.setEtcdOperatorImage(&imgs); err != nil {
+		glog.Errorf("error setting etcd operator images: %#v", err)
+	}
+
 	spec.KubeAPIServerServingCAData = kubeAPIServerServingCABytes
 	spec.EtcdCAData = etcdCA
 	spec.EtcdMetricCAData = etcdMetricCA
@@ -743,6 +749,32 @@ func (optr *Operator) getGlobalConfig() (*configv1.Infrastructure, *configv1.Net
 	return infra, network, proxy, nil
 }
 
+func (optr *Operator) setEtcdOperatorImage(imgs *Images) error {
+	if optr.etcdLister == nil {
+		// if the resource is not found, i.e. it is not created by CVO
+		// which means cluster-etcd-operator images is not part of CVO
+		imgs.ControllerConfigImages.ClusterEtcdOperator = ""
+		glog.V(4).Info("etcd cr not found")
+		return nil
+	}
+	etcd, err := optr.etcdLister.Get("cluster")
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			imgs.ControllerConfigImages.ClusterEtcdOperator = ""
+			return nil
+		}
+		imgs.ControllerConfigImages.ClusterEtcdOperator = ""
+		return fmt.Errorf("error getting etcd CR: %#v", err)
+	}
+
+	if etcd.Spec.ManagementState == operatorv1.Unmanaged {
+		glog.V(4).Info("etcd cluster in unmanaged")
+		imgs.ControllerConfigImages.ClusterEtcdOperator = ""
+		return nil
+	}
+	return nil
+}
+
 func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *RenderConfigImages, apiServerURL string) *renderConfig {
 	return &renderConfig{
 		TargetNamespace:        tnamespace,

templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml

@@ -12,6 +12,18 @@ contents:
         k8s-app: etcd
     spec:
       initContainers:
+    {{if .Images.clusterEtcdOperatorImageKey}}
+      - name: wait-for-kube
+        image: "{{.Images.clusterEtcdOperatorImageKey}}"
+        command: ["/usr/bin/cluster-etcd-operator"]
+        args:
+        - "wait-for-kube"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: sa
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount/
+    {{end}}
       - name: discovery
         image: "{{.Images.setupEtcdEnvKey}}"
         command: ["/usr/bin/setup-etcd-environment"]