Add etcd-quorum-guard manifests and doc

cmd/machine-config-operator/bootstrap.go

@@ -42,6 +42,7 @@ var (
 		infraImage           string
 		kubeClientAgentImage string
 		destinationDir       string
+		etcdQuorumGuardImage string
 	}
 )
 
@@ -66,6 +67,8 @@ func init() {
 	bootstrapCmd.MarkFlagRequired("etcd-image")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.setupEtcdEnvImage, "setup-etcd-env-image", "", "Image for Setup etcd Environment.")
 	bootstrapCmd.MarkFlagRequired("setup-etcd-env-image")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdQuorumGuardImage, "etcd-quorum-guard-image", "registry.svc.ci.openshift.org/openshift/origin-v4.0:base", "Image for etcd Quorum Guard.")
+	bootstrapCmd.MarkFlagRequired("etcd-quorum-guard-image")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeClientAgentImage, "kube-client-agent-image", "", "Image for Kube Client Agent.")
 	bootstrapCmd.MarkFlagRequired("kube-client-agent-image")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.infraImage, "infra-image", "", "Image for Infra Containers.")
@@ -90,6 +93,7 @@ func runBootstrapCmd(cmd *cobra.Command, args []string) {
 		MachineOSContent:        bootstrapOpts.oscontentImage,
 		Etcd:                    bootstrapOpts.etcdImage,
 		SetupEtcdEnv:            bootstrapOpts.setupEtcdEnvImage,
+		EtcdQuorumGuardImage:    "registry.svc.ci.openshift.org/openshift/origin-v4.0:base",
 		InfraImage:              bootstrapOpts.infraImage,
 		KubeClientAgent:         bootstrapOpts.kubeClientAgentImage,
 	}

cmd/machine-config-operator/start.go

@@ -65,6 +65,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 			ctrlctx.KubeNamespacedInformerFactory.Rbac().V1().ClusterRoles(),
 			ctrlctx.KubeNamespacedInformerFactory.Rbac().V1().ClusterRoleBindings(),
 			ctrlctx.KubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
+			ctrlctx.KubeNamespacedInformerFactory.Policy().V1beta1().PodDisruptionBudgets(),
 			ctrlctx.KubeInformerFactory.Core().V1().ConfigMaps(),
 			ctrlctx.ConfigInformerFactory.Config().V1().Infrastructures(),
 			ctrlctx.ConfigInformerFactory.Config().V1().Networks(),

docs/etcd-quorum-guard.md

@@ -0,0 +1,36 @@
+# etcd Quorum Guard
+
+The etcd Quorum Guard ensures that quorum is maintained for etcd for
+[OpenShift](https://openshift.io/).
+
+For the etcd cluster to remain usable, we must maintain quorum, which
+is a majority of all etcd members.  For example, an etcd cluster with
+3 members (i.e. a 3 master deployment) must have at least 2 healthy
+etcd members to meet the quorum limit.
+
+There are situations where 2 etcd members could be down at once:
+
+* a master has gone offline and the MachineConfig Controller (MCC)
+  tries to rollout a new MachineConfig (MC) by rebooting masters
+* the MCC is doing a MachineConfig rollout and doesn't wait for the
+  etcd on the previous master to become healthy again before rebooting
+  the next master
+
+The etcd Quorum Guard ensures that a drain on a master is not allowed
+to proceed if the reboot of the master would cause etcd quorum loss.
+It is implemented as a deployment, with one pod per master node.
+
+The etcd Quorum Guard checks the health of etcd by querying the health
+endpoint of etcd; if etcd reports itself unhealthy or is not present,
+the quorum guard reports itself not ready.  A disruption budget is
+used to allow no more than one unhealthy/missing quorum guard (and
+hence etcd).  If one etcd is already not healthy or missing, this
+disruption budget will act as a drain gate, not allowing an attempt to
+drain another node.
+
+This drain gate cannot protect against a second node failing due to
+e. g. hardware failure; it can only protect against an attempt to
+drain the node in preparation for taking it down.
+
+There is no user or administrator action necessary or available for
+the etcd Quorum Guard.

install/0000_30_machine-config-operator_02_images.configmap.yaml

@@ -12,5 +12,6 @@ data:
       "etcd": "registry.svc.ci.openshift.org/openshift/origin-v4.0:etcd",
       "setupEtcdEnv": "registry.svc.ci.openshift.org/openshift/origin-v4.0:setup-etcd-environment",
       "infraImage": "quay.io/openshift/origin-pod:v4.0",
-      "kubeClientAgentImage": "registry.svc.ci.openshift.org/openshift/origin-v4.0:kube-client-agent"
+      "kubeClientAgentImage": "registry.svc.ci.openshift.org/openshift/origin-v4.0:kube-client-agent",
+      "etcdQuorumGuardImage": "registry.svc.ci.openshift.org/openshift/origin-v4.0:base"
     }

install/image-references

@@ -27,6 +27,10 @@ spec:
     from:
       kind: DockerImage
       name: quay.io/openshift/origin-pod:v4.0
+  - name: etcd-quorum-guard
+    from:
+      kind: DockerImage
+      name: registry.svc.ci.openshift.org/openshift/origin-v4.0:base
   - name: setup-etcd-environment
     from:
       kind: DockerImage

lib/resourceapply/core.go

@@ -46,3 +46,94 @@ func ApplySecret(client coreclientv1.SecretsGetter, required *corev1.Secret) (*c
 	actual, err := client.Secrets(required.Namespace).Update(existing)
 	return actual, true, err
 }
+
+// ApplyConfigMap merges objectmeta, requires data
+func ApplyConfigMap(client coreclientv1.ConfigMapsGetter, required *corev1.ConfigMap) (*corev1.ConfigMap, bool, error) {
+	existing, err := client.ConfigMaps(required.Namespace).Get(required.Name, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		actual, err := client.ConfigMaps(required.Namespace).Create(required)
+		return actual, true, err
+	}
+	if err != nil {
+		return nil, false, err
+	}
+
+	modified := resourcemerge.BoolPtr(false)
+	existingCopy := existing.DeepCopy()
+
+	resourcemerge.EnsureObjectMeta(modified, &existingCopy.ObjectMeta, required.ObjectMeta)
+
+	var modifiedKeys []string
+	for existingCopyKey, existingCopyValue := range existingCopy.Data {
+		if requiredValue, ok := required.Data[existingCopyKey]; !ok || (existingCopyValue != requiredValue) {
+			modifiedKeys = append(modifiedKeys, "data."+existingCopyKey)
+		}
+	}
+	for requiredKey := range required.Data {
+		if _, ok := existingCopy.Data[requiredKey]; !ok {
+			modifiedKeys = append(modifiedKeys, "data."+requiredKey)
+		}
+	}
+
+	dataSame := len(modifiedKeys) == 0
+	if dataSame && !*modified {
+		return existingCopy, false, nil
+	}
+	existingCopy.Data = required.Data
+
+	actual, err := client.ConfigMaps(required.Namespace).Update(existingCopy)
+
+	return actual, true, err
+}
+
+// SyncConfigMap syncs ConfigMap opjectmeta from source namespace to destination
+func SyncConfigMap(client coreclientv1.ConfigMapsGetter, sourceNamespace, sourceName, targetNamespace, targetName string, ownerRefs []metav1.OwnerReference) (*corev1.ConfigMap, bool, error) {
+	source, err := client.ConfigMaps(sourceNamespace).Get(sourceName, metav1.GetOptions{})
+	sourceCopy := source.DeepCopy()
+
+	switch {
+	case apierrors.IsNotFound(err):
+		deleteErr := client.ConfigMaps(targetNamespace).Delete(targetName, nil)
+		if apierrors.IsNotFound(deleteErr) {
+			return nil, false, nil
+		}
+		if deleteErr == nil {
+			return nil, true, nil
+		}
+		return nil, false, deleteErr
+	case err != nil:
+		return nil, false, err
+	default:
+		sourceCopy.Namespace = targetNamespace
+		sourceCopy.Name = targetName
+		sourceCopy.ResourceVersion = ""
+		sourceCopy.OwnerReferences = ownerRefs
+		return ApplyConfigMap(client, sourceCopy)
+	}
+}
+
+// SyncConfigMap syncs Secret opjectmeta from source namespace to destination
+func SyncSecret(client coreclientv1.SecretsGetter, sourceNamespace, sourceName, targetNamespace, targetName string, ownerRefs []metav1.OwnerReference) (*corev1.Secret, bool, error) {
+	source, err := client.Secrets(sourceNamespace).Get(sourceName, metav1.GetOptions{})
+	sourceCopy := source.DeepCopy()
+
+	switch {
+	case apierrors.IsNotFound(err):
+		deleteErr := client.Secrets(targetNamespace).Delete(targetName, nil)
+		if apierrors.IsNotFound(deleteErr) {
+			return nil, false, nil
+		}
+		if deleteErr == nil {
+			return nil, true, nil
+		}
+		return nil, false, deleteErr
+	case err != nil:
+		return nil, false, err
+	default:
+		sourceCopy.Namespace = targetNamespace
+		sourceCopy.Name = targetName
+		sourceCopy.ResourceVersion = ""
+		sourceCopy.OwnerReferences = ownerRefs
+		return ApplySecret(client, sourceCopy)
+	}
+}

lib/resourceapply/policy.go

@@ -0,0 +1,30 @@
+package resourceapply
+
+import (
+	"github.com/openshift/machine-config-operator/lib/resourcemerge"
+	policyv1 "k8s.io/api/policy/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	policyclientv1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1"
+)
+
+// ApplyPodDisruptionBudget applies the required podDisruptionBudget to the cluster.
+func ApplyPodDisruptionBudget(client policyclientv1.PodDisruptionBudgetsGetter, required *policyv1.PodDisruptionBudget) (*policyv1.PodDisruptionBudget, bool, error) {
+	existing, err := client.PodDisruptionBudgets(required.Namespace).Get(required.Name, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		actual, err := client.PodDisruptionBudgets(required.Namespace).Create(required)
+		return actual, true, err
+	}
+	if err != nil {
+		return nil, false, err
+	}
+
+	modified := resourcemerge.BoolPtr(false)
+	resourcemerge.EnsurePodDisruptionBudget(modified, existing, *required)
+	if !*modified {
+		return existing, false, nil
+	}
+
+	actual, err := client.PodDisruptionBudgets(required.Namespace).Update(existing)
+	return actual, true, err
+}

lib/resourcemerge/policy.go

@@ -0,0 +1,16 @@
+package resourcemerge
+
+import (
+	policyv1 "k8s.io/api/policy/v1beta1"
+	"k8s.io/apimachinery/pkg/api/equality"
+)
+
+// EnsurePodDisruptionBudget ensures that the existing matches the required.
+// modified is set to true when existing had to be updated with required.
+func EnsurePodDisruptionBudget(modified *bool, existing *policyv1.PodDisruptionBudget, required policyv1.PodDisruptionBudget) {
+	EnsureObjectMeta(modified, &existing.ObjectMeta, required.ObjectMeta)
+	if !equality.Semantic.DeepEqual(existing.Spec, required.Spec) {
+		*modified = true
+		existing.Spec = required.Spec
+	}
+}

lib/resourceread/policy.go

@@ -0,0 +1,27 @@
+package resourceread
+
+import (
+	policyv1 "k8s.io/api/policy/v1beta1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+var (
+	policyScheme = runtime.NewScheme()
+	policyCodecs = serializer.NewCodecFactory(policyScheme)
+)
+
+func init() {
+	if err := policyv1.AddToScheme(policyScheme); err != nil {
+		panic(err)
+	}
+}
+
+// ReadPodDisruptionBudgetV1OrDie reads podDisruptionBudget object from bytes. Panics on error.
+func ReadPodDisruptionBudgetV1OrDie(objBytes []byte) *policyv1.PodDisruptionBudget {
+	requiredObj, err := runtime.Decode(policyCodecs.UniversalDecoder(policyv1.SchemeGroupVersion), objBytes)
+	if err != nil {
+		panic(err)
+	}
+	return requiredObj.(*policyv1.PodDisruptionBudget)
+}

manifests/etcdquorumguard/deployment.yaml

@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: etcd-quorum-guard
+  namespace: {{.TargetNamespace}}
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      k8s-app: etcd-quorum-guard
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: etcd-quorum-guard
+        k8s-app: etcd-quorum-guard
+    spec:
+      hostNetwork: true
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: k8s-app
+                operator: In
+                values:
+                - "etcd-quorum-guard"
+            topologyKey: kubernetes.io/hostname
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      priorityClassName: "system-cluster-critical"
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+        operator: Exists
+      - key: node.kubernetes.io/memory-pressure
+        effect: NoSchedule
+        operator: Exists
+      - key: node.kubernetes.io/disk-pressure
+        effect: NoSchedule
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        effect: NoExecute
+        operator: Exists
+      - key: node.kubernetes.io/unreachable
+        effect: NoExecute
+        operator: Exists
+      - key: node.kubernetes.io/unschedulable
+        effect: NoExecute
+        operator: Exists
+      - key: node-role.kubernetes.io/etcd
+        operator: Exists
+        effect: NoSchedule
+      containers:
+      - image: "{{.Images.EtcdQuorumGuardImage}}"
+        imagePullPolicy: IfNotPresent
+        name: etcd-quorum-guard-container
+        volumeMounts:
+        - mountPath: /mnt/kube
+          name: kubecerts
+        command:
+        - "/bin/sh"
+        args:
+        - "-c"
+        - |
+            declare -r croot=/mnt/kube
+            set -x
+            declare -r health_endpoint="https://127.0.0.1:2379/health"
+            declare -r cert="$(find $croot -name 'system:etcd-peer*.crt' -print -quit)"
+            declare -r key="${cert%.crt}.key"
+            declare -r cacert="$croot/ca.crt"
+            ls -lR "$croot"
+            ls -lRL "$croot"
+            while : ; do date; curl --max-time 2 --cert "${cert//:/\:}" --key "$key" --cacert "$cacert" "$health_endpoint"; sleep 5; done
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - |
+                declare -r croot=/mnt/kube
+                declare -r health_endpoint="https://127.0.0.1:2379/health"
+                declare -r cert="$(find $croot -name 'system:etcd-peer*.crt' -print -quit)"
+                declare -r key="${cert%.crt}.key"
+                declare -r cacert="$croot/ca.crt"
+                [[ -z $cert || -z $key ]] && exit 1
+                curl --max-time 2 --silent --cert "${cert//:/\:}" --key "$key" --cacert "$cacert" "$health_endpoint" |grep '{ *"health" *: *"true" *}'
+            initialDelaySecond: 5
+            periodSecond: 5
+        resources:
+          requests:
+            cpu: 10m
+            memory: 5Mi
+      volumes:
+      - name: kubecerts
+        hostPath:
+          path: /etc/kubernetes/static-pod-resources/etcd-member

manifests/etcdquorumguard/disruption-budget.yaml

@@ -0,0 +1,10 @@
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  namespace: {{.TargetNamespace}}
+  name: etcd-quorum-guard
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: etcd-quorum-guard

pkg/controller/template/constants.go

@@ -12,4 +12,7 @@ const (
 
 	// KubeClientAgentImageKey is the key that references the kube-client-agent image in the controller
 	KubeClientAgentImageKey string = "kubeClientAgentImage"
+
+	// EtcdQuorumGuardImageKey is the key that references the etcd-quorum-guard image
+	EtcdQuorumGuardImageKey string = "etcdQuorumGuardImage"
 )

pkg/controller/template/test_data/controller_config_aws.yaml

@@ -15,3 +15,4 @@ spec:
     setupEtcdEnv: image/setupEtcdEnv:1
     infraImage: image/infraImage:1
     kubeClientAgentImage: image/kubeClientAgentImage:1
+    etcdQuorumGuardImage: image/etcdQuorumGuardImage:1

pkg/controller/template/test_data/controller_config_libvirt.yaml

@@ -15,3 +15,4 @@ spec:
     setupEtcdEnv: image/setupEtcdEnv:1
     infraImage: image/infraImage:1
     kubeClientAgentImage: image/kubeClientAgentImage:1
+    etcdQuorumGuardImage: image/etcdQuorumGuardImage:1

pkg/controller/template/test_data/controller_config_none.yaml

@@ -15,3 +15,4 @@ spec:
     setupEtcdEnv: image/setupEtcdEnv:1
     infraImage: image/infraImage:1
     kubeClientAgentImage: image/kubeClientAgentImage:1
+    etcdQuorumGuardImage: image/etcdQuorumGuardImage:1