WIP: cmd/setup-etcd-environment: add etcd pivot logic

cmd/setup-etcd-environment/run.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -14,7 +15,11 @@ import (
 	"github.com/golang/glog"
 	"github.com/openshift/machine-config-operator/pkg/version"
 	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog"
 )
 
 var (
@@ -29,9 +34,23 @@ var (
 		discoverySRV string
 		ifName       string
 		outputFile   string
+		pivot        bool
 	}
+	EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale"
 )
 
+type EtcdScaling struct {
+	Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
+	Members  []Member           `json:"members,omitempty"`
+}
+
+type Member struct {
+	ID         uint64   `json:"ID,omitempty"`
+	Name       string   `json:"name,omitempty"`
+	PeerURLS   []string `json:"peerURLs,omitempty"`
+	ClientURLS []string `json:"clientURLs,omitempty"`
+}
+
 func init() {
 	rootCmd.AddCommand(runCmd)
 	rootCmd.PersistentFlags().StringVar(&runOpts.discoverySRV, "discovery-srv", "", "DNS domain used to bootstrap initial etcd cluster.")
@@ -86,9 +105,54 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 		out = f
 	}
 
-	if err := writeEnvironmentFile(map[string]string{
-		"DISCOVERY_SRV": runOpts.discoverySRV,
-	}, out, true); err != nil {
+	export := make(map[string]string)
+	etcdName := os.Getenv("ETCD_NAME")
+	if etcdName == "" {
+		return fmt.Errorf("environment variable ETCD_NAME has no value")
+	}
+	var e EtcdScaling
+	if runOpts.pivot {
+		clientConfig, err := rest.InClusterConfig()
+		if err != nil {
+			panic(err.Error())
+		}
+		client, err := kubernetes.NewForConfig(clientConfig)
+		if err != nil {
+			return fmt.Errorf("error creating client: %v", err)
+		}
+		duration := 10 * time.Second
+		// wait forever for success and retry every duration interval
+		wait.PollInfinite(duration, func() (bool, error) {
+			result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("scaling-lock", metav1.GetOptions{})
+			if err != nil {
+				klog.Errorf("error creating client %v", err)
+				return false, nil
+			}
+			if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil {
+				klog.Errorf("error decoding result %v", err)
+				return false, nil
+			}
+			if e.Metadata.Name != etcdName {
+				klog.Errorf("could not find self in scaling-lock")
+				return false, nil
+			}
+			members := e.Members
+			if len(members) == 0 {
+				klog.Errorf("no members found in scaling-lock")
+				return false, nil
+			}
+			var memberList []string
+			for _, m := range members {
+				memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0]))
+			}
+			memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, ip))
+			export["INITIAL_CLUSTER"] = strings.Join(memberList, ",")
+			return true, nil
+		})
+	} else {
+		export["DISCOVERY_SRV"] = runOpts.discoverySRV
+	}
+	if err := writeEnvironmentFile(export, out, true); err != nil {
 		return err
 	}
 
@@ -137,6 +201,9 @@ func reverseLookupSelf(service, proto, name, self string) (string, error) {
 	}
 	selfTarget := ""
 	for _, srv := range srvs {
+		if isPivot(srv.Target) {
+			runOpts.pivot = true
+		}
 		glog.V(4).Infof("checking against %s", srv.Target)
 		addrs, err := net.LookupHost(srv.Target)
 		if err != nil {
@@ -156,10 +223,14 @@ func reverseLookupSelf(service, proto, name, self string) (string, error) {
 	return selfTarget, nil
 }
 
+func isPivot(target string) bool {
+	return strings.HasPrefix(target, "etcd-bootstrap")
+}
+
 func writeEnvironmentFile(m map[string]string, w io.Writer, export bool) error {
 	var buffer bytes.Buffer
 	for k, v := range m {
-		env := fmt.Sprintf("ETCD_%s=%s\n", k, v)
+		env := fmt.Sprintf("ETCD_%s=\"%s\"\n", k, v)
 		if export == true {
 			env = fmt.Sprintf("export %s", env)
 		}

templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml

@@ -23,6 +23,13 @@ contents:
         volumeMounts:
         - name: discovery
           mountPath: /run/etcd/
+        - name: sa
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount/
+        env:
+        - name: ETCD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
       - name: certs
         image: "{{.Images.kubeClientAgentImageKey}}"
         command:
@@ -105,6 +112,10 @@ contents:
             --listen-client-urls=https://0.0.0.0:2379 \
             --listen-peer-urls=https://0.0.0.0:2380 \
             --listen-metrics-urls=https://0.0.0.0:9978 \
+        lifecycle:
+          postStart:
+            exec:
+              command: ["/bin/sh", "-c", "sleep 5"]
         resources:
           requests:
             memory: 600Mi
@@ -129,6 +140,10 @@ contents:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: ETCD_INITIAL_CLUSTER
+          valueFrom:
+            fieldRef:
+               fieldPath: metadata.annotations['etcd.operator.openshift.io/scale']
         ports:
         - name: peer
           containerPort: 2380
@@ -157,6 +172,10 @@ contents:
             --cert-file /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.crt \
             --cacert /etc/ssl/etcd/ca.crt \
             --trusted-ca-file /etc/ssl/etcd/metric-ca.crt \
+        lifecycle:
+          postStart:
+            exec:
+              command: ["/bin/sh", "-c", "sleep 5"]
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - name: discovery
@@ -191,3 +210,6 @@ contents:
       - name: conf
         hostPath:
           path: /etc/etcd
+      - name: sa
+        hostPath:
+          path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token