setup-etcd-environment: support scaling from cluster-etcd-operator

cmd/setup-etcd-environment/run.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -14,9 +15,18 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/joho/godotenv"
+	ceoapi "github.com/openshift/cluster-etcd-operator/pkg/operator/api"
 	"github.com/openshift/machine-config-operator/pkg/version"
 	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+const (
+	EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale"
+	etcdInitialExisting      = "existing"
 )
 
 var (
@@ -53,6 +63,20 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 		return errors.New("--discovery-srv cannot be empty")
 	}
 
+	etcdName := os.Getenv("ETCD_NAME")
+	if etcdName == "" {
+		return fmt.Errorf("environment variable ETCD_NAME has no value")
+	}
+
+	etcdDataDir := os.Getenv("ETCD_DATA_DIR")
+	if etcdDataDir == "" {
+		return fmt.Errorf("environment variable ETCD_DATA_DIR has no value")
+	}
+
+	if !inCluster() {
+		glog.V(4).Infof("KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT contain no value, running in standalone mode.")
+	}
+
 	ips, err := ipAddrs()
 	if err != nil {
 		return err
@@ -62,7 +86,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 	var ip string
 	if err := wait.PollImmediate(30*time.Second, 5*time.Minute, func() (bool, error) {
 		for _, cand := range ips {
-			found, err := reverseLookupSelf("etcd-server-ssl", "tcp", runOpts.discoverySRV, cand)
+			found, err := reverseLookup("etcd-server-ssl", "tcp", runOpts.discoverySRV, cand, runOpts.bootstrapSRV)
 			if err != nil {
 				glog.Errorf("error looking up self for candidate IP %s: %v", cand, err)
 				continue
@@ -72,18 +96,27 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 				ip = cand
 				return true, nil
 			}
-			glog.V(4).Infof("no matching dns for %s", cand)
+			glog.V(4).Infof("no matching dns for %s in %s: %v", cand, runOpts.discoverySRV, err)
 		}
 		return false, nil
 	}); err != nil {
 		return fmt.Errorf("could not find self: %v", err)
 	}
 	glog.Infof("dns name is %s", dns)
 
-	// initialize envs used to bootstrap etcd
-	exportEnv, err := setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV)
-	if err != nil {
-		return err
+	var exportEnv map[string]string
+
+	if _, err := os.Stat(fmt.Sprintf("%s/member", etcdDataDir)); os.IsNotExist(err) && !runOpts.bootstrapSRV && inCluster() {
+		exportEnv, err = setExportEnv(etcdName, dns)
+		if err != nil {
+			return err
+		}
+	} else {
+		// initialize envs used to bootstrap etcd
+		exportEnv, err = setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV)
+		if err != nil {
+			return err
+		}
 	}
 
 	out := os.Stdout
@@ -96,6 +129,12 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 		out = f
 	}
 
+	if runOpts.bootstrapSRV {
+		exportEnv["DISCOVERY_SRV"] = runOpts.discoverySRV
+	} else {
+		exportEnv["NAME"] = etcdName
+	}
+
 	// enable etcd to run using s390 and s390x. Because these are not officially supported upstream
 	// etcd requires population of environment variable ETCD_UNSUPPORTED_ARCH at runtime.
 	// https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/supported-platform.md
@@ -114,6 +153,86 @@ func runRunCmd(cmd *cobra.Command, args []string) error {
 	}, out, false)
 }
 
+func setExportEnv(etcdName, dns string) (map[string]string, error) {
+	exportEnv := make(map[string]string)
+	duration := 10 * time.Second
+	wait.PollInfinite(duration, func() (bool, error) {
+		if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); os.IsNotExist(err) {
+			glog.Errorf("serviceaccount failed: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+
+	clientConfig, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err.Error())
+	}
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error creating client: %v", err)
+	}
+	var e ceoapi.EtcdScaling
+	// wait forever for success and retry every duration interval
+	wait.PollInfinite(duration, func() (bool, error) {
+		result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("member-config", metav1.GetOptions{})
+		if err != nil {
+			glog.Errorf("error creating client %v", err)
+			return false, nil
+		}
+		if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil {
+			glog.Errorf("error decoding result %v", err)
+			return false, nil
+		}
+		if e.Metadata.Name != etcdName {
+			glog.Errorf("could not find self in member-config")
+			return false, nil
+		}
+		members := e.Members
+		if len(members) == 0 {
+			glog.Errorf("no members found in member-config")
+			return false, nil
+		}
+		var memberList []string
+		for _, m := range members {
+			memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0]))
+		}
+		memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns))
+		exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",")
+		exportEnv["INITIAL_CLUSTER_STATE"] = etcdInitialExisting
+		return true, nil
+	})
+
+	ep, err := client.CoreV1().Endpoints("openshift-etcd").Get("etcd", metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	hostEtcdEndpoint, err := client.CoreV1().Endpoints("openshift-etcd").Get("host-etcd", metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if len(hostEtcdEndpoint.Subsets) != 1 {
+		return nil, fmt.Errorf("openshift-etcd/host-etcd endpoint subset length should be %d, found %d", 1, len(hostEtcdEndpoint.Subsets))
+	}
+
+	endpoints := make([]string, 0)
+	for _, member := range hostEtcdEndpoint.Subsets[0].Addresses {
+		if member.Hostname == "etcd-bootstrap" {
+			endpoints = append(endpoints, "https://etcd-bootstrap."+runOpts.discoverySRV+":2379")
+			break
+		}
+	}
+	if len(ep.Subsets) != 1 {
+		return nil, fmt.Errorf("openshift-etcd/etcd endpoint subset length should be %d, found %d", 1, len(ep.Subsets))
+	}
+	for _, s := range ep.Subsets[0].Addresses {
+		endpoints = append(endpoints, "https://"+s.IP+":2379")
+	}
+
+	exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",")
+	return exportEnv, nil
+}
+
 // setBootstrapEnv populates and returns a map based on envs from file.
 func setBootstrapEnv(envFile, discoverySRV string, bootstrapSRV bool) (map[string]string, error) {
 	bootstrapEnv := make(map[string]string)
@@ -167,29 +286,44 @@ func ipAddrs() ([]string, error) {
 	return ips, nil
 }
 
+func reverseLookup(service, proto, name, self string, bootstrapSRV bool) (string, error) {
+	if bootstrapSRV || inCluster() {
+		return reverseLookupSelf(service, proto, name, self)
+	}
+	return lookupTargetMatchSelf(fmt.Sprintf("etcd-bootstrap.%s", name), self)
+}
+
 // returns the target from the SRV record that resolves to self.
 func reverseLookupSelf(service, proto, name, self string) (string, error) {
 	_, srvs, err := net.LookupSRV(service, proto, name)
 	if err != nil {
 		return "", err
 	}
-	selfTarget := ""
 	for _, srv := range srvs {
 		glog.V(4).Infof("checking against %s", srv.Target)
-		addrs, err := net.LookupHost(srv.Target)
+		selfTarget, err := lookupTargetMatchSelf(srv.Target, self)
 		if err != nil {
-			return "", fmt.Errorf("could not resolve member %q", srv.Target)
+			return "", err
 		}
-
-		for _, addr := range addrs {
-			if addr == self {
-				selfTarget = strings.Trim(srv.Target, ".")
-				break
-			}
+		if selfTarget != "" {
+			return selfTarget, nil
 		}
 	}
-	if selfTarget == "" {
-		return "", fmt.Errorf("could not find self")
+	return "", fmt.Errorf("could not find self")
+}
+
+//
+func lookupTargetMatchSelf(target, self string) (string, error) {
+	addrs, err := net.LookupHost(target)
+	if err != nil {
+		return "", fmt.Errorf("could not resolve member %q", target)
+	}
+	selfTarget := ""
+	for _, addr := range addrs {
+		if addr == self {
+			selfTarget = strings.Trim(target, ".")
+			break
+		}
 	}
 	return selfTarget, nil
 }
@@ -208,3 +342,10 @@ func writeEnvironmentFile(m map[string]string, w io.Writer, export bool) error {
 	}
 	return nil
 }
+
+func inCluster() bool {
+	if os.Getenv("KUBERNETES_SERVICE_HOST") == "" || os.Getenv("KUBERNETES_SERVICE_PORT") == "" {
+		return false
+	}
+	return true
+}

go.mod

@@ -50,6 +50,7 @@ require (
 	github.com/openshift/api v3.9.1-0.20191014195513-c9253efc14f4+incompatible
 	github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0
 	github.com/openshift/cluster-api v0.0.0-20190923092624-4024de4fa64d
+	github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4
 	github.com/openshift/library-go v0.0.0-20191003152030-97c62d8a2901
 	github.com/openshift/runtime-utils v0.0.0-20191011150825-9169de69ebf6
 	github.com/pkg/errors v0.8.1

go.sum

@@ -685,6 +685,8 @@ github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0 h1:U0rtkdPj1lT
 github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk=
 github.com/openshift/cluster-api v0.0.0-20191004085540-83f32d3e7070 h1:Yw3cWSzWhZ9Kh8WqX6tOccVPs9BWUp4jDfb6I8AU/pg=
 github.com/openshift/cluster-api v0.0.0-20191004085540-83f32d3e7070/go.mod h1:mNsD1dsD4T57kV4/C6zTHke/Ro166xgnyyRZqkamiEU=
+github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4 h1:EDZ2v8AqMnXgeT3bECcbamUg8Haz0V/iXqDMlu94Mrw=
+github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4/go.mod h1:vcBAUefK8pQmTPQ3jlCemORXhnRnq3aTsfOSVeaWliY=
 github.com/openshift/imagebuilder v1.1.0 h1:oT704SkwMEzmIMU/+Uv1Wmvt+p10q3v2WuYMeFI18c4=
 github.com/openshift/imagebuilder v1.1.0/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo=
 github.com/openshift/kubernetes v1.16.0-beta.0.0.20190913145653-2bd9643cee5b h1:b3r/3odnWNdjYRrwvXiqxLFbPYFQ+m2GblFK6RwTC9Y=

templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml

@@ -74,7 +74,7 @@ contents:
           mountPath: /etc/ssl/etcd/
         - name: kubeconfig
           mountPath: /etc/kubernetes/kubeconfig
-    {{if .Images.clusterEtcdOperatorImageKey}}
+   {{if .Images.clusterEtcdOperatorImageKey}}
         - name: sa
           mountPath: /var/run/secrets/kubernetes.io/serviceaccount/
    {{end}}
@@ -242,4 +242,4 @@ contents:
       - name: sa
         hostPath:
           path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token
-   {{end}}
+   {{end}}