fixups: clustermembercontroller2, manifests and setup-env

bindata/etcd/pod.yaml

@@ -5,7 +5,8 @@ metadata:
   namespace: openshift-etcd
   labels:
     app: etcd
-    etcd: "not-true-yet"
+    k8s-app: etcd
+    etcd: "true"
     revision: "REVISION"
 spec:
   containers:
@@ -20,16 +21,50 @@ spec:
         #!/bin/sh
         set -euo pipefail
 
-        sleep 24h
+        ETCDCTL="etcdctl --cacert=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \
+                           --cert=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \
+                           --key=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \
+                           --endpoints=${ALL_ETCD_ENDPOINTS}"
+        ${ETCDCTL} member list
 
-        exit 0
+        echo "waiting for member $NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME..."
+        COUNT=30
+        while [ $COUNT -gt 0 ]; do
+          IS_MEMBER_PRESENT=$(${ETCDCTL} member list | grep -o "${NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME}.*:2380")
+          if [[ -n "${IS_MEMBER_PRESENT:-}" ]]; then
+            break
+          fi
+          sleep 1
+          let COUNT=$COUNT-1
+        done
 
-        # add logic here to confirm that we are part of the etcd members (the controller added us).
-        # this is probably a golang command that tries to confirm for two minutes before exiting
-        # and prints nothing except for the ETCD_INITIAL_CLUSTER
+        # if the member is not present after 30 seconds
+        if [ -z "$IS_MEMBER_PRESENT" ]; then
+          echo "member $NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME is not present after 30 seconds"
+          exit 1
+        fi
+        echo "member $NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME is present, continuing"
 
+        initial_cluster=""
+        member_output=$( ${ETCDCTL} member list | cut -d',' -f3 )
+        for endpoint_key in ${member_output}; do
+          endpoint=$(${ETCDCTL} member list | grep $endpoint_key | awk -F'[, ]' '{ print $7 }')
+          initial_cluster+="$endpoint_key=$endpoint,"
+          echo "adding $endpoint_key=$endpoint,"
+        done
+        # add this pod to the list
+        initial_cluster+="$NODE_NODE_ENVVAR_NAME_ETCD_NAME=https://$NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME:2380"
+        echo $initial_cluster
+
+        # at this point we know this member is added.  To support a transition, we must remove the old etcd pod.
+        # move it somewhere safe so we can retrieve it again later if something goes badly.
+        mv /etc/kubernetes/manifests/etcd-member.yaml /etc/kubernetes/etcd-backup-dir || true
+
+        export ETCD_INITIAL_CLUSTER="${initial_cluster}"
         export ETCD_NAME=${NODE_NODE_ENVVAR_NAME_ETCD_NAME}
+        env | grep ETCD | grep -v NODE
 
+        set -x
         exec etcd \
           --initial-advertise-peer-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2380 \
           --cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \
@@ -43,23 +78,40 @@ spec:
           --advertise-client-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2379 \
           --listen-client-urls=https://${LISTEN_ON_ALL_IPS}:2379 \
           --listen-peer-urls=https://${LISTEN_ON_ALL_IPS}:2380 \
-          --listen-metrics-urls=https://${LISTEN_ON_ALL_IPS}:9978
+          --listen-metrics-urls=https://${LISTEN_ON_ALL_IPS}:9978 ||  mv /etc/kubernetes/etcd-backup-dir/etcd-member.yaml /etc/kubernetes/manifests
     env:
 ${COMPUTED_ENV_VARS}
     resources:
       requests:
-        memory: 200Mi
-        cpu: 100m
+        memory: 600Mi
+        cpu: 300m
       limits:
-        memory: 200Mi
-        cpu: 100m
+        memory: 600Mi
+        cpu: 300m
+    readinessProbe:
+      exec:
+        command:
+          - /bin/sh
+          - -ec
+          - "lsof -n -i :2380 | grep LISTEN"
+      failureThreshold: 3
+      initialDelaySeconds: 3
+      periodSeconds: 5
+      successThreshold: 1
+      timeoutSeconds: 5
+    securityContext:
+      privileged: true
     volumeMounts:
-    - mountPath: /etc/kubernetes/static-pod-resources
-      name: resource-dir
-    - mountPath: /etc/kubernetes/static-pod-certs
-      name: cert-dir
-    - mountPath: /var/lib/etcd/
-      name: data-dir
+      - mountPath: /etc/kubernetes/manifests
+        name: static-pod-dir
+      - mountPath: /etc/kubernetes/etcd-backup-dir
+        name: etcd-backup-dir
+      - mountPath: /etc/kubernetes/static-pod-resources
+        name: resource-dir
+      - mountPath: /etc/kubernetes/static-pod-certs
+        name: cert-dir
+      - mountPath: /var/lib/etcd/
+        name: data-dir
   - name: etcd-metrics
     image: ${IMAGE}
     imagePullPolicy: IfNotPresent
@@ -71,14 +123,6 @@ ${COMPUTED_ENV_VARS}
         #!/bin/sh
         set -euo pipefail
 
-        sleep 24h
-
-        exit 0
-
-        # add logic here to confirm that we are part of the etcd members (the controller added us).
-        # this is probably a golang command that tries to confirm for two minutes before exiting
-        # and prints nothing except for the ETCD_INITIAL_CLUSTER
-
         export ETCD_NAME=${NODE_NODE_ENVVAR_NAME_ETCD_NAME}
 
         exec etcd grpc-proxy start \
@@ -100,6 +144,8 @@ ${COMPUTED_ENV_VARS}
       limits:
         memory: 200Mi
         cpu: 100m
+    securityContext:
+      privileged: true
     volumeMounts:
       - mountPath: /etc/kubernetes/static-pod-resources
         name: resource-dir
@@ -112,14 +158,20 @@ ${COMPUTED_ENV_VARS}
   tolerations:
   - operator: "Exists"
   volumes:
-  - hostPath:
-      path: /etc/kubernetes/static-pod-resources/etcd-pod-REVISION
-    name: resource-dir
-  - hostPath:
-      path: /etc/kubernetes/static-pod-resources/etcd-certs
-    name: cert-dir
-  - hostPath:
-      path: /var/lib/etcd
-      type: ""
-    name: data-dir
+    - hostPath:
+        path: /etc/kubernetes/manifests
+      name: static-pod-dir
+    - hostPath:
+        path: /etc/kubernetes/static-pod-resources/etcd-member
+      name: etcd-backup-dir
+    - hostPath:
+        path: /etc/kubernetes/static-pod-resources/etcd-pod-REVISION
+      name: resource-dir
+    - hostPath:
+        path: /etc/kubernetes/static-pod-resources/etcd-certs
+      name: cert-dir
+    - hostPath:
+        path: /var/lib/etcd
+        type: ""
+      name: data-dir
 

go.mod

@@ -8,14 +8,15 @@ require (
 	github.com/gorilla/mux v0.0.0-20191024121256-f395758b854c
 	github.com/jteeuwen/go-bindata v3.0.8-0.20151023091102-a0ff2567cfb7+incompatible
 	github.com/openshift/api v0.0.0-20200131223221-f2a771e1a90c
-	github.com/openshift/build-machinery-go v0.0.0-20200205161356-ef115f5adc73
+	github.com/openshift/build-machinery-go v0.0.0-20200210090402-3b072832771e
 	github.com/openshift/client-go v0.0.0-20200116152001-92a2713fa240
 	github.com/openshift/library-go v0.0.0-20200207150939-615337e1c3aa
 	github.com/prometheus/client_golang v1.1.0
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50
 	go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
+	google.golang.org/grpc v1.23.1
 	k8s.io/api v0.17.1
 	k8s.io/apimachinery v0.17.1
 	k8s.io/client-go v0.17.1

go.sum

@@ -314,6 +314,8 @@ github.com/openshift/api v0.0.0-20200131223221-f2a771e1a90c h1:6kb8UZix0DNEfZbys
 github.com/openshift/api v0.0.0-20200131223221-f2a771e1a90c/go.mod h1:fT6U/JfG8uZzemTRwZA2kBDJP5nWz7v05UHnty/D+pk=
 github.com/openshift/build-machinery-go v0.0.0-20200205161356-ef115f5adc73 h1:WCvABw620V2FqeNoRJWeuAATqGjsrzb0UQ3tL0RHcXw=
 github.com/openshift/build-machinery-go v0.0.0-20200205161356-ef115f5adc73/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
+github.com/openshift/build-machinery-go v0.0.0-20200210090402-3b072832771e h1:qlMmBDqBavn7p4Y22teVEkJCnU9YAwhABHeXanAirWE=
+github.com/openshift/build-machinery-go v0.0.0-20200210090402-3b072832771e/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/client-go v0.0.0-20200116152001-92a2713fa240 h1:XYfJWv2Ch+qInGLDEedHRtDsJwnxyU1L8U7SY56NcA8=
 github.com/openshift/client-go v0.0.0-20200116152001-92a2713fa240/go.mod h1:4riOwdj99Hd/q+iAcJZfNCsQQQMwURnZV6RL4WHYS5w=
 github.com/openshift/library-go v0.0.0-20200207150939-615337e1c3aa h1:3/i0Kzbt8TbC+QkZ3sZ9b6M64/CzEXa2fN2IoIKzXYs=

manifests/0000_12_etcd-operator_06_deployment.yaml

@@ -46,7 +46,7 @@ spec:
           name: etcd-client
         env:
         - name: IMAGE
-          value: quay.io/openshift/cluster-etcd-operator:v4.0
+          value: registry.svc.ci.openshift.org/openshift:etcd
         - name: OPERATOR_IMAGE
           value: quay.io/openshift/cluster-etcd-operator:v4.0
         - name: OPERATOR_IMAGE_VERSION

manifests/image-references

@@ -6,3 +6,7 @@ spec:
     from:
       kind: DockerImage
       name: quay.io/openshift/cluster-etcd-operator
+  - name: etcd
+    from:
+      kind: DockerImage
+      name: registry.svc.ci.openshift.org/openshift:etcd