diff --git a/cmd/machine-config-operator/bootstrap.go b/cmd/machine-config-operator/bootstrap.go index 2768e437ca..a41e64d815 100644 --- a/cmd/machine-config-operator/bootstrap.go +++ b/cmd/machine-config-operator/bootstrap.go @@ -34,6 +34,7 @@ var ( keepalivedImage string kubeCAFile string kubeClientAgentImage string + clusterEtcdOperatorImage string mcoImage string mdnsPublisherImage string oauthProxyImage string @@ -63,6 +64,7 @@ func init() { bootstrapCmd.MarkFlagRequired("etcd-image") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeClientAgentImage, "kube-client-agent-image", "", "Image for Kube Client Agent.") bootstrapCmd.MarkFlagRequired("kube-client-agent-image") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.clusterEtcdOperatorImage, "cluster-etcd-operator-image", "", "Image for Cluster Etcd Operator. An empty string here means the cluster boots without CEO.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.infraImage, "infra-image", "", "Image for Infra Containers.") bootstrapCmd.MarkFlagRequired("infra-image") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.configFile, "config-file", "", "ClusterConfig ConfigMap file.") @@ -101,6 +103,7 @@ func runBootstrapCmd(cmd *cobra.Command, args []string) { Etcd: bootstrapOpts.etcdImage, InfraImage: bootstrapOpts.infraImage, KubeClientAgent: bootstrapOpts.kubeClientAgentImage, + ClusterEtcdOperator: bootstrapOpts.clusterEtcdOperatorImage, Keepalived: bootstrapOpts.keepalivedImage, Coredns: bootstrapOpts.corednsImage, MdnsPublisher: bootstrapOpts.mdnsPublisherImage, diff --git a/install/0000_80_machine-config-operator_02_images.configmap.yaml b/install/0000_80_machine-config-operator_02_images.configmap.yaml index 31f1e684c4..038b6b6d18 100644 --- a/install/0000_80_machine-config-operator_02_images.configmap.yaml +++ b/install/0000_80_machine-config-operator_02_images.configmap.yaml @@ -11,6 +11,7 @@ data: "etcd": "registry.svc.ci.openshift.org/openshift:etcd", "infraImage": "registry.svc.ci.openshift.org/openshift:pod", "kubeClientAgentImage": "registry.svc.ci.openshift.org/openshift:kube-client-agent", + "clusterEtcdOperatorImage": "", "keepalivedImage": "registry.svc.ci.openshift.org/openshift:keepalived-ipfailover", "corednsImage": "registry.svc.ci.openshift.org/openshift:coredns", "mdnsPublisherImage": "registry.svc.ci.openshift.org/openshift:mdns-publisher", diff --git a/pkg/controller/template/constants.go b/pkg/controller/template/constants.go index 234bfb556d..7d98bb9d9b 100644 --- a/pkg/controller/template/constants.go +++ b/pkg/controller/template/constants.go @@ -16,6 +16,9 @@ const ( // KubeClientAgentImageKey is the key that references the kube-client-agent image in the controller KubeClientAgentImageKey string = "kubeClientAgentImageKey" + // ClusterEtcdOperatorImageKey is the key that references the cluster-etcd-operator image in the controller + ClusterEtcdOperatorImageKey string = "clusterEtcdOperatorImageKey" + // KeepalivedKey is the key that references the keepalived-ipfailover image in the controller KeepalivedKey string = "keepalivedImage" diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index d6a97ffad3..76a8e8a2d3 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -327,6 +327,9 @@ func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error) funcs["skip"] = skipMissing funcs["etcdServerCertDNSNames"] = etcdServerCertDNSNames funcs["etcdPeerCertDNSNames"] = etcdPeerCertDNSNames + funcs["etcdServerCertCommand"] = etcdServerCertCommand + funcs["etcdPeerCertCommand"] = etcdPeerCertCommand + funcs["etcdMetricCertCommand"] = etcdMetricCertCommand funcs["cloudProvider"] = cloudProvider funcs["cloudConfigFlag"] = cloudConfigFlag tmpl, err := template.New(path).Funcs(funcs).Parse(string(b)) @@ -378,6 +381,90 @@ func etcdPeerCertDNSNames(cfg RenderConfig) (interface{}, error) { return strings.Join(dnsNames, ","), nil } +func etcdServerCertCommand(cfg RenderConfig) (interface{}, error) { + commands := []string{} + if cfg.Images[ClusterEtcdOperatorImageKey] == "" { + serverCertDNS, err := etcdServerCertDNSNames(cfg) + if err != nil { + return nil, err + } + commands = append(commands, []string{ + "kube-client-agent \\", + " request \\", + " --kubeconfig=/etc/kubernetes/kubeconfig \\", + " --orgname=system:etcd-servers \\", + " --assetsdir=/etc/ssl/etcd \\", + fmt.Sprintf(" --dnsnames=%s \\", serverCertDNS), + " --commonname=system:etcd-server:${ETCD_DNS_NAME} \\", + " --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \\", + }...) + } else { + commands = append(commands, []string{ + "cluster-etcd-operator \\", + " mount \\", + " --assetsdir=/etc/ssl/etcd \\", + " --commonname=system:etcd-server:${ETCD_DNS_NAME} \\", + }...) + } + return commands, nil +} + +func etcdPeerCertCommand(cfg RenderConfig) (interface{}, error) { + commands := []string{} + if cfg.Images[ClusterEtcdOperatorImageKey] == "" { + peerCertDNS, err := etcdPeerCertDNSNames(cfg) + if err != nil { + return nil, err + } + commands = append(commands, []string{ + "kube-client-agent \\", + " request \\", + " --kubeconfig=/etc/kubernetes/kubeconfig \\", + " --orgname=system:etcd-peers \\", + " --assetsdir=/etc/ssl/etcd \\", + fmt.Sprintf(" --dnsnames=%s \\", peerCertDNS), + " --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\", + " --ipaddrs=${ETCD_IPV4_ADDRESS} \\", + }...) + } else { + commands = append(commands, []string{ + "cluster-etcd-operator \\", + " mount \\", + " --assetsdir=/etc/ssl/etcd \\", + " --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\", + }...) + } + return commands, nil +} + +func etcdMetricCertCommand(cfg RenderConfig) (interface{}, error) { + commands := []string{} + if cfg.Images[ClusterEtcdOperatorImageKey] == "" { + metricCertDNS, err := etcdServerCertDNSNames(cfg) + if err != nil { + return nil, err + } + commands = append(commands, []string{ + "kube-client-agent \\", + " request \\", + " --kubeconfig=/etc/kubernetes/kubeconfig \\", + " --orgname=system:etcd-metrics \\", + " --assetsdir=/etc/ssl/etcd \\", + fmt.Sprintf(" --dnsnames=%s \\", metricCertDNS), + " --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\", + " --ipaddrs=${ETCD_IPV4_ADDRESS} \\", + }...) + } else { + commands = append(commands, []string{ + "cluster-etcd-operator \\", + " mount \\", + " --assetsdir=/etc/ssl/etcd \\", + " --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\", + }...) + } + return commands, nil +} + func cloudProvider(cfg RenderConfig) (interface{}, error) { switch cfg.Platform { case platformAWS, platformAzure, platformOpenStack, platformVSphere: diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index 24553655a5..906feb8fb8 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -136,16 +136,17 @@ func RenderBootstrap( spec.PullSecret = nil spec.OSImageURL = imgs.MachineOSContent spec.Images = map[string]string{ - templatectrl.EtcdImageKey: imgs.Etcd, - templatectrl.SetupEtcdEnvKey: imgs.MachineConfigOperator, - templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, - templatectrl.InfraImageKey: imgs.InfraImage, - templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent, - templatectrl.KeepalivedKey: imgs.Keepalived, - templatectrl.CorednsKey: imgs.Coredns, - templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, - templatectrl.HaproxyKey: imgs.Haproxy, - templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, + templatectrl.EtcdImageKey: imgs.Etcd, + templatectrl.SetupEtcdEnvKey: imgs.MachineConfigOperator, + templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, + templatectrl.InfraImageKey: imgs.InfraImage, + templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent, + templatectrl.ClusterEtcdOperatorImageKey: imgs.ClusterEtcdOperator, + templatectrl.KeepalivedKey: imgs.Keepalived, + templatectrl.CorednsKey: imgs.Coredns, + templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, + templatectrl.HaproxyKey: imgs.Haproxy, + templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, } config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL) diff --git a/pkg/operator/images.go b/pkg/operator/images.go index a79aacbdc5..ec661c4ed9 100644 --- a/pkg/operator/images.go +++ b/pkg/operator/images.go @@ -31,6 +31,7 @@ type ControllerConfigImages struct { Etcd string `json:"etcd"` InfraImage string `json:"infraImage"` KubeClientAgent string `json:"kubeClientAgentImage"` + ClusterEtcdOperator string `json:"clusterEtcdOperatorImage"` Keepalived string `json:"keepalivedImage"` Coredns string `json:"corednsImage"` MdnsPublisher string `json:"mdnsPublisherImage"` diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 0b47600ab7..774e3cca26 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -213,16 +213,17 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error { spec.PullSecret = &corev1.ObjectReference{Namespace: "openshift-config", Name: "pull-secret"} spec.OSImageURL = imgs.MachineOSContent spec.Images = map[string]string{ - templatectrl.EtcdImageKey: imgs.Etcd, - templatectrl.SetupEtcdEnvKey: imgs.MachineConfigOperator, - templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, - templatectrl.InfraImageKey: imgs.InfraImage, - templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent, - templatectrl.KeepalivedKey: imgs.Keepalived, - templatectrl.CorednsKey: imgs.Coredns, - templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, - templatectrl.HaproxyKey: imgs.Haproxy, - templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, + templatectrl.EtcdImageKey: imgs.Etcd, + templatectrl.SetupEtcdEnvKey: imgs.MachineConfigOperator, + templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, + templatectrl.InfraImageKey: imgs.InfraImage, + templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent, + templatectrl.ClusterEtcdOperatorImageKey: imgs.ClusterEtcdOperator, + templatectrl.KeepalivedKey: imgs.Keepalived, + templatectrl.CorednsKey: imgs.Coredns, + templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, + templatectrl.HaproxyKey: imgs.Haproxy, + templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, } // create renderConfig diff --git a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml index 96f164887f..449f40a4a2 100644 --- a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml +++ b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml @@ -19,14 +19,30 @@ contents: - "run" - "--discovery-srv={{.EtcdDiscoveryDomain}}" - "--output-file=/run/etcd/environment" + - "--bootstrap-srv={{if .Images.clusterEtcdOperatorImageKey}}{{false}}{{else}}{{true}}{{end}}" - "--v=4" securityContext: privileged: true volumeMounts: - name: discovery mountPath: /run/etcd/ + - name: data-dir + mountPath: /var/lib/etcd/ + - name: certs + mountPath: /etc/ssl/etcd/ + {{if .Images.clusterEtcdOperatorImageKey}} + - name: sa + mountPath: /var/run/secrets/kubernetes.io/serviceaccount/ + {{end}} + env: + - name: ETCD_DATA_DIR + value: "/var/lib/etcd" + - name: ETCD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: certs - image: "{{.Images.kubeClientAgentImageKey}}" + image: {{if .Images.clusterEtcdOperatorImageKey }}"{{.Images.clusterEtcdOperatorImageKey}}"{{else}}"{{.Images.kubeClientAgentImageKey}}"{{end}} command: - /bin/sh - -c @@ -37,37 +53,16 @@ contents: source /run/etcd/environment [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \ - -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \ - kube-client-agent \ - request \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --orgname=system:etcd-servers \ - --assetsdir=/etc/ssl/etcd \ - --dnsnames={{etcdServerCertDNSNames .}} \ - --commonname=system:etcd-server:${ETCD_DNS_NAME} \ - --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \ + -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \{{range etcdServerCertCommand .}} + {{.}}{{end}} [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \ - -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \ - kube-client-agent \ - request \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --orgname=system:etcd-peers \ - --assetsdir=/etc/ssl/etcd \ - --dnsnames={{etcdPeerCertDNSNames .}} \ - --commonname=system:etcd-peer:${ETCD_DNS_NAME} \ - --ipaddrs=${ETCD_IPV4_ADDRESS} \ + -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \{{range etcdPeerCertCommand .}} + {{.}}{{end}} [ -e /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.crt -a \ - -e /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.key ] || \ - kube-client-agent \ - request \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --orgname=system:etcd-metrics \ - --assetsdir=/etc/ssl/etcd \ - --dnsnames={{etcdServerCertDNSNames .}} \ - --commonname=system:etcd-metric:${ETCD_DNS_NAME} \ - --ipaddrs=${ETCD_IPV4_ADDRESS} \ + -e /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.key ] || \{{range etcdMetricCertCommand .}} + {{.}}{{end}} securityContext: privileged: true @@ -79,9 +74,53 @@ contents: mountPath: /etc/ssl/etcd/ - name: kubeconfig mountPath: /etc/kubernetes/kubeconfig + {{if .Images.clusterEtcdOperatorImageKey}} + - name: sa + mountPath: /var/run/secrets/kubernetes.io/serviceaccount/ + {{end}} + {{if .Images.clusterEtcdOperatorImageKey}} + - name: membership + image: "{{.Images.etcdKey}}" + command: + - /bin/sh + - -c + - | + #!/bin/sh + set -euxo pipefail + + source /run/etcd/environment + + export ETCDCTL_API=3 ETCDCTL_CACERT=/etc/ssl/etcd/ca.crt ETCDCTL_CERT=$(find /etc/ssl/ -name *peer*crt) ETCDCTL_KEY=$(find /etc/ssl/ -name *peer*key) \ + ETCDCTL_ENDPOINTS="$ETCD_ENDPOINTS" + + until $(etcdctl member list | grep $ETCD_DNS_NAME>/dev/null) + do + echo "waiting for $ETCD_DNS_NAME to be added to etcd membership" + sleep 2 + done + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: discovery + mountPath: /run/etcd/ + - name: certs + mountPath: /etc/ssl/etcd/ + {{end}} containers: - name: etcd-member image: "{{.Images.etcdKey}}" + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - "lsof -n -i :2380 | grep LISTEN" + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 command: - /bin/sh - -c @@ -197,3 +236,8 @@ contents: - name: conf hostPath: path: /etc/etcd + {{if .Images.clusterEtcdOperatorImageKey}} + - name: sa + hostPath: + path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token + {{end}} \ No newline at end of file