cmd/machine-config-operator/bootstrap: add cluster-etcd-operator flag

cmd/machine-config-operator/bootstrap.go

@@ -34,6 +34,7 @@ var (
 		keepalivedImage           string
 		kubeCAFile                string
 		kubeClientAgentImage      string
+		clusterEtcdOperatorImage  string
 		mcoImage                  string
 		mdnsPublisherImage        string
 		oauthProxyImage           string
@@ -63,6 +64,7 @@ func init() {
 	bootstrapCmd.MarkFlagRequired("etcd-image")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeClientAgentImage, "kube-client-agent-image", "", "Image for Kube Client Agent.")
 	bootstrapCmd.MarkFlagRequired("kube-client-agent-image")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.clusterEtcdOperatorImage, "cluster-etcd-operator-image", "", "Image for Cluster Etcd Operator. An empty string here means the cluster boots without CEO.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.infraImage, "infra-image", "", "Image for Infra Containers.")
 	bootstrapCmd.MarkFlagRequired("infra-image")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.configFile, "config-file", "", "ClusterConfig ConfigMap file.")
@@ -101,6 +103,7 @@ func runBootstrapCmd(cmd *cobra.Command, args []string) {
 			Etcd:                bootstrapOpts.etcdImage,
 			InfraImage:          bootstrapOpts.infraImage,
 			KubeClientAgent:     bootstrapOpts.kubeClientAgentImage,
+			ClusterEtcdOperator: bootstrapOpts.clusterEtcdOperatorImage,
 			Keepalived:          bootstrapOpts.keepalivedImage,
 			Coredns:             bootstrapOpts.corednsImage,
 			MdnsPublisher:       bootstrapOpts.mdnsPublisherImage,

install/0000_80_machine-config-operator_02_images.configmap.yaml

@@ -11,6 +11,7 @@ data:
       "etcd": "registry.svc.ci.openshift.org/openshift:etcd",
       "infraImage": "registry.svc.ci.openshift.org/openshift:pod",
       "kubeClientAgentImage": "registry.svc.ci.openshift.org/openshift:kube-client-agent",
+      "clusterEtcdOperatorImage": "",
       "keepalivedImage": "registry.svc.ci.openshift.org/openshift:keepalived-ipfailover",
       "corednsImage": "registry.svc.ci.openshift.org/openshift:coredns",
       "mdnsPublisherImage": "registry.svc.ci.openshift.org/openshift:mdns-publisher",

pkg/controller/template/constants.go

@@ -16,6 +16,9 @@ const (
 	// KubeClientAgentImageKey is the key that references the kube-client-agent image in the controller
 	KubeClientAgentImageKey string = "kubeClientAgentImageKey"
 
+	// ClusterEtcdOperatorImageKey is the key that references the cluster-etcd-operator image in the controller
+	ClusterEtcdOperatorImageKey string = "clusterEtcdOperatorImageKey"
+
 	// KeepalivedKey is the key that references the keepalived-ipfailover image in the controller
 	KeepalivedKey string = "keepalivedImage"
 

pkg/controller/template/render.go

@@ -327,6 +327,9 @@ func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error)
 	funcs["skip"] = skipMissing
 	funcs["etcdServerCertDNSNames"] = etcdServerCertDNSNames
 	funcs["etcdPeerCertDNSNames"] = etcdPeerCertDNSNames
+	funcs["etcdServerCertCommand"] = etcdServerCertCommand
+	funcs["etcdPeerCertCommand"] = etcdPeerCertCommand
+	funcs["etcdMetricCertCommand"] = etcdMetricCertCommand
 	funcs["cloudProvider"] = cloudProvider
 	funcs["cloudConfigFlag"] = cloudConfigFlag
 	tmpl, err := template.New(path).Funcs(funcs).Parse(string(b))
@@ -378,6 +381,90 @@ func etcdPeerCertDNSNames(cfg RenderConfig) (interface{}, error) {
 	return strings.Join(dnsNames, ","), nil
 }
 
+func etcdServerCertCommand(cfg RenderConfig) (interface{}, error) {
+	commands := []string{}
+	if cfg.Images[ClusterEtcdOperatorImageKey] == "" {
+		serverCertDNS, err := etcdServerCertDNSNames(cfg)
+		if err != nil {
+			return nil, err
+		}
+		commands = append(commands, []string{
+			"kube-client-agent \\",
+			"  request \\",
+			"  --kubeconfig=/etc/kubernetes/kubeconfig \\",
+			"  --orgname=system:etcd-servers \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			fmt.Sprintf("  --dnsnames=%s \\", serverCertDNS),
+			"  --commonname=system:etcd-server:${ETCD_DNS_NAME} \\",
+			"  --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \\",
+		}...)
+	} else {
+		commands = append(commands, []string{
+			"cluster-etcd-operator \\",
+			"  mount \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			"  --commonname=system:etcd-server:${ETCD_DNS_NAME} \\",
+		}...)
+	}
+	return commands, nil
+}
+
+func etcdPeerCertCommand(cfg RenderConfig) (interface{}, error) {
+	commands := []string{}
+	if cfg.Images[ClusterEtcdOperatorImageKey] == "" {
+		peerCertDNS, err := etcdPeerCertDNSNames(cfg)
+		if err != nil {
+			return nil, err
+		}
+		commands = append(commands, []string{
+			"kube-client-agent \\",
+			"  request \\",
+			"  --kubeconfig=/etc/kubernetes/kubeconfig \\",
+			"  --orgname=system:etcd-peers \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			fmt.Sprintf("  --dnsnames=%s \\", peerCertDNS),
+			"  --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\",
+			"  --ipaddrs=${ETCD_IPV4_ADDRESS} \\",
+		}...)
+	} else {
+		commands = append(commands, []string{
+			"cluster-etcd-operator \\",
+			"  mount \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			"  --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\",
+		}...)
+	}
+	return commands, nil
+}
+
+func etcdMetricCertCommand(cfg RenderConfig) (interface{}, error) {
+	commands := []string{}
+	if cfg.Images[ClusterEtcdOperatorImageKey] == "" {
+		metricCertDNS, err := etcdServerCertDNSNames(cfg)
+		if err != nil {
+			return nil, err
+		}
+		commands = append(commands, []string{
+			"kube-client-agent \\",
+			"  request \\",
+			"  --kubeconfig=/etc/kubernetes/kubeconfig \\",
+			"  --orgname=system:etcd-metrics \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			fmt.Sprintf("  --dnsnames=%s \\", metricCertDNS),
+			"  --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\",
+			"  --ipaddrs=${ETCD_IPV4_ADDRESS} \\",
+		}...)
+	} else {
+		commands = append(commands, []string{
+			"cluster-etcd-operator \\",
+			"  mount \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			"  --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\",
+		}...)
+	}
+	return commands, nil
+}
+
 func cloudProvider(cfg RenderConfig) (interface{}, error) {
 	switch cfg.Platform {
 	case platformAWS, platformAzure, platformOpenStack, platformVSphere:

pkg/operator/bootstrap.go

@@ -136,16 +136,17 @@ func RenderBootstrap(
 	spec.PullSecret = nil
 	spec.OSImageURL = imgs.MachineOSContent
 	spec.Images = map[string]string{
-		templatectrl.EtcdImageKey:            imgs.Etcd,
-		templatectrl.SetupEtcdEnvKey:         imgs.MachineConfigOperator,
-		templatectrl.GCPRoutesControllerKey:  imgs.MachineConfigOperator,
-		templatectrl.InfraImageKey:           imgs.InfraImage,
-		templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent,
-		templatectrl.KeepalivedKey:           imgs.Keepalived,
-		templatectrl.CorednsKey:              imgs.Coredns,
-		templatectrl.MdnsPublisherKey:        imgs.MdnsPublisher,
-		templatectrl.HaproxyKey:              imgs.Haproxy,
-		templatectrl.BaremetalRuntimeCfgKey:  imgs.BaremetalRuntimeCfg,
+		templatectrl.EtcdImageKey:                imgs.Etcd,
+		templatectrl.SetupEtcdEnvKey:             imgs.MachineConfigOperator,
+		templatectrl.GCPRoutesControllerKey:      imgs.MachineConfigOperator,
+		templatectrl.InfraImageKey:               imgs.InfraImage,
+		templatectrl.KubeClientAgentImageKey:     imgs.KubeClientAgent,
+		templatectrl.ClusterEtcdOperatorImageKey: imgs.ClusterEtcdOperator,
+		templatectrl.KeepalivedKey:               imgs.Keepalived,
+		templatectrl.CorednsKey:                  imgs.Coredns,
+		templatectrl.MdnsPublisherKey:            imgs.MdnsPublisher,
+		templatectrl.HaproxyKey:                  imgs.Haproxy,
+		templatectrl.BaremetalRuntimeCfgKey:      imgs.BaremetalRuntimeCfg,
 	}
 
 	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL)

pkg/operator/images.go

@@ -31,6 +31,7 @@ type ControllerConfigImages struct {
 	Etcd                string `json:"etcd"`
 	InfraImage          string `json:"infraImage"`
 	KubeClientAgent     string `json:"kubeClientAgentImage"`
+	ClusterEtcdOperator string `json:"clusterEtcdOperatorImage"`
 	Keepalived          string `json:"keepalivedImage"`
 	Coredns             string `json:"corednsImage"`
 	MdnsPublisher       string `json:"mdnsPublisherImage"`

pkg/operator/sync.go

@@ -213,16 +213,17 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error {
 	spec.PullSecret = &corev1.ObjectReference{Namespace: "openshift-config", Name: "pull-secret"}
 	spec.OSImageURL = imgs.MachineOSContent
 	spec.Images = map[string]string{
-		templatectrl.EtcdImageKey:            imgs.Etcd,
-		templatectrl.SetupEtcdEnvKey:         imgs.MachineConfigOperator,
-		templatectrl.GCPRoutesControllerKey:  imgs.MachineConfigOperator,
-		templatectrl.InfraImageKey:           imgs.InfraImage,
-		templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent,
-		templatectrl.KeepalivedKey:           imgs.Keepalived,
-		templatectrl.CorednsKey:              imgs.Coredns,
-		templatectrl.MdnsPublisherKey:        imgs.MdnsPublisher,
-		templatectrl.HaproxyKey:              imgs.Haproxy,
-		templatectrl.BaremetalRuntimeCfgKey:  imgs.BaremetalRuntimeCfg,
+		templatectrl.EtcdImageKey:                imgs.Etcd,
+		templatectrl.SetupEtcdEnvKey:             imgs.MachineConfigOperator,
+		templatectrl.GCPRoutesControllerKey:      imgs.MachineConfigOperator,
+		templatectrl.InfraImageKey:               imgs.InfraImage,
+		templatectrl.KubeClientAgentImageKey:     imgs.KubeClientAgent,
+		templatectrl.ClusterEtcdOperatorImageKey: imgs.ClusterEtcdOperator,
+		templatectrl.KeepalivedKey:               imgs.Keepalived,
+		templatectrl.CorednsKey:                  imgs.Coredns,
+		templatectrl.MdnsPublisherKey:            imgs.MdnsPublisher,
+		templatectrl.HaproxyKey:                  imgs.Haproxy,
+		templatectrl.BaremetalRuntimeCfgKey:      imgs.BaremetalRuntimeCfg,
 	}
 
 	// create renderConfig

templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml

@@ -19,14 +19,30 @@ contents:
         - "run"
         - "--discovery-srv={{.EtcdDiscoveryDomain}}"
         - "--output-file=/run/etcd/environment"
+        - "--bootstrap-srv={{if .Images.clusterEtcdOperatorImageKey}}{{false}}{{else}}{{true}}{{end}}"
         - "--v=4"
         securityContext:
           privileged: true
         volumeMounts:
         - name: discovery
           mountPath: /run/etcd/
+        - name: data-dir
+          mountPath: /var/lib/etcd/
+        - name: certs
+          mountPath: /etc/ssl/etcd/
+    {{if .Images.clusterEtcdOperatorImageKey}}
+        - name: sa
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount/
+    {{end}}
+        env:
+        - name: ETCD_DATA_DIR
+          value: "/var/lib/etcd"
+        - name: ETCD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
       - name: certs
-        image: "{{.Images.kubeClientAgentImageKey}}"
+        image: {{if .Images.clusterEtcdOperatorImageKey }}"{{.Images.clusterEtcdOperatorImageKey}}"{{else}}"{{.Images.kubeClientAgentImageKey}}"{{end}}
         command:
         - /bin/sh
         - -c
@@ -37,37 +53,16 @@ contents:
           source /run/etcd/environment
 
           [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \
-            -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \
-            kube-client-agent \
-              request \
-                --kubeconfig=/etc/kubernetes/kubeconfig \
-                --orgname=system:etcd-servers \
-                --assetsdir=/etc/ssl/etcd \
-                --dnsnames={{etcdServerCertDNSNames .}} \
-                --commonname=system:etcd-server:${ETCD_DNS_NAME} \
-                --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \
+            -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \{{range etcdServerCertCommand .}}
+            {{.}}{{end}}
 
           [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \
-            -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \
-            kube-client-agent \
-              request \
-                --kubeconfig=/etc/kubernetes/kubeconfig \
-                --orgname=system:etcd-peers \
-                --assetsdir=/etc/ssl/etcd \
-                --dnsnames={{etcdPeerCertDNSNames .}} \
-                --commonname=system:etcd-peer:${ETCD_DNS_NAME} \
-                --ipaddrs=${ETCD_IPV4_ADDRESS} \
+            -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \{{range etcdPeerCertCommand .}}
+            {{.}}{{end}}
 
           [ -e /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.crt -a \
-            -e /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.key ] || \
-            kube-client-agent \
-              request \
-                --kubeconfig=/etc/kubernetes/kubeconfig \
-                --orgname=system:etcd-metrics \
-                --assetsdir=/etc/ssl/etcd \
-                --dnsnames={{etcdServerCertDNSNames .}} \
-                --commonname=system:etcd-metric:${ETCD_DNS_NAME} \
-                --ipaddrs=${ETCD_IPV4_ADDRESS} \
+            -e /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.key ] || \{{range etcdMetricCertCommand .}}
+            {{.}}{{end}}
 
         securityContext:
           privileged: true
@@ -79,9 +74,53 @@ contents:
           mountPath: /etc/ssl/etcd/
         - name: kubeconfig
           mountPath: /etc/kubernetes/kubeconfig
+    {{if .Images.clusterEtcdOperatorImageKey}}
+        - name: sa
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount/
+   {{end}}
+   {{if .Images.clusterEtcdOperatorImageKey}}
+      - name: membership
+        image: "{{.Images.etcdKey}}"
+        command:
+          - /bin/sh
+          - -c
+          - |
+            #!/bin/sh
+            set -euxo pipefail
+
+            source /run/etcd/environment
+
+            export ETCDCTL_API=3 ETCDCTL_CACERT=/etc/ssl/etcd/ca.crt ETCDCTL_CERT=$(find /etc/ssl/ -name *peer*crt) ETCDCTL_KEY=$(find /etc/ssl/ -name *peer*key) \
+              ETCDCTL_ENDPOINTS="$ETCD_ENDPOINTS"
+
+            until $(etcdctl member list | grep $ETCD_DNS_NAME>/dev/null)
+            do
+              echo "waiting for $ETCD_DNS_NAME to be added to etcd membership"
+              sleep 2
+            done
+        securityContext:
+          privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+          - name: discovery
+            mountPath: /run/etcd/
+          - name: certs
+            mountPath: /etc/ssl/etcd/
+   {{end}}
       containers:
       - name: etcd-member
         image: "{{.Images.etcdKey}}"
+        readinessProbe:
+          exec:
+            command:
+              - /bin/sh
+              - -ec
+              - "lsof -n -i :2380 | grep LISTEN"
+          failureThreshold: 3
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          successThreshold: 1
+          timeoutSeconds: 5
         command:
         - /bin/sh
         - -c
@@ -197,3 +236,8 @@ contents:
       - name: conf
         hostPath:
           path: /etc/etcd
+   {{if .Images.clusterEtcdOperatorImageKey}}
+      - name: sa
+        hostPath:
+          path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token
+   {{end}}