etcd boostrapping using dns

Dockerfile.setup-etcd-environment

@@ -0,0 +1,10 @@
+FROM golang:1.10.3 AS build-env
+
+COPY . /go/src/github.com/openshift/machine-config-operator
+WORKDIR /go/src/github.com/openshift/machine-config-operator
+RUN WHAT=setup-etcd-environment ./hack/build-go.sh
+
+FROM openshift/origin-base:v4.0.0
+COPY --from=build-env /go/src/github.com/openshift/machine-config-operator/_output/linux/amd64/setup-etcd-environment /bin/setup-etcd-environment
+
+ENTRYPOINT ["/bin/setup-etcd-environment"]

cmd/setup-etcd-environment/main.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+)
+
+const (
+	componentName = "etcd-setup-environment"
+)
+
+var (
+	rootOpts struct {
+		discoverySRV string
+		ifName       string
+		outputFile   string
+	}
+)
+
+func main() {
+	rootCmd := &cobra.Command{
+		Use:           componentName,
+		Short:         "Sets up the environment for etcd",
+		Long:          "",
+		RunE:          runRootCmd,
+		SilenceErrors: true,
+		SilenceUsage:  true,
+	}
+	rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
+	rootCmd.PersistentFlags().StringVar(&rootOpts.discoverySRV, "discovery-srv", "", "DNS domain used to bootstrap initial etcd cluster.")
+	rootCmd.PersistentFlags().StringVar(&rootOpts.ifName, "if-name", "eth0", "The network interface that should be used for getting local ip address.")
+	rootCmd.PersistentFlags().StringVar(&rootOpts.outputFile, "output-file", "", "file where the envs are written. If empty, prints to Stdout.")
+
+	if err := rootCmd.Execute(); err != nil {
+		glog.Exitf("Error executing %s: %v", componentName, err)
+	}
+}
+
+func runRootCmd(cmd *cobra.Command, args []string) error {
+	flag.Set("logtostderr", "true")
+	flag.Parse()
+
+	if rootOpts.discoverySRV == "" {
+		return errors.New("--discovery-srv cannot be empty")
+	}
+
+	ip, err := ipAddrForIf(rootOpts.ifName)
+	if err != nil {
+		return err
+	}
+	glog.Infof("ip addr is %s", ip)
+
+	var dns string
+	if err := wait.PollImmediate(1*time.Minute, 5*time.Minute, func() (bool, error) {
+		found, err := reverseLookupSelf("etcd-server-ssl", "tcp", rootOpts.discoverySRV, ip)
+		if err != nil {
+			glog.Errorf("error looking up self: %v", err)
+			return false, nil
+		}
+		if found != "" {
+			dns = found
+			return true, nil
+		}
+		return false, errors.New("found dns is invalid")
+	}); err != nil {
+		return fmt.Errorf("could not find self: %v", err)
+	}
+	glog.Infof("dns name is %s", dns)
+
+	out := os.Stdout
+	if rootOpts.outputFile != "" {
+		f, err := os.Create(rootOpts.outputFile)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		out = f
+	}
+
+	return writeEnvironmentFile(map[string]string{
+		"IPV4_ADDRESS": ip,
+		"DNS_NAME":     dns,
+	}, out)
+}
+
+func ipAddrForIf(ifname string) (string, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return "", err
+	}
+	for _, i := range ifaces {
+		if i.Name != ifname {
+			continue
+		}
+
+		addrs, err := i.Addrs()
+		if err != nil {
+			return "", err
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil {
+				continue
+			}
+			ip = ip.To4()
+			if ip == nil {
+				continue // not an ipv4 address
+			}
+			if !ip.IsGlobalUnicast() {
+				continue // we only want global unicast address
+			}
+			return ip.String(), nil
+		}
+	}
+	return "", fmt.Errorf("could not find ip address for %s", ifname)
+}
+
+// returns the target from the SRV record that resolves to self.
+func reverseLookupSelf(service, proto, name, self string) (string, error) {
+	_, srvs, err := net.LookupSRV(service, proto, name)
+	if err != nil {
+		return "", err
+	}
+	for _, srv := range srvs {
+		glog.V(4).Infof("checking against %s", srv.Target)
+		addrs, err := net.LookupHost(srv.Target)
+		if err != nil {
+			continue // don't care
+		}
+
+		for _, addr := range addrs {
+			if addr == self {
+				return strings.Trim(srv.Target, "."), nil
+			}
+		}
+	}
+	return "", fmt.Errorf("could not find self")
+}
+
+func writeEnvironmentFile(m map[string]string, w io.Writer) error {
+	var buffer bytes.Buffer
+	for k, v := range m {
+		buffer.WriteString(fmt.Sprintf("ETCD_%s=%s\n", k, v))
+	}
+	if _, err := buffer.WriteTo(w); err != nil {
+		return err
+	}
+	return nil
+}

hack/build-image.sh

@@ -63,5 +63,5 @@ fi
 for IMAGE_TO_BUILD in $TOBUILD; do
 	NAME="${IMAGE_TO_BUILD#Dockerfile.}"
 	set -x
-	podman build -t "${NAME}:${VERSION}" -f "${IMAGE_TO_BUILD}"
+	podman build -t "${NAME}:${VERSION}" -f "${IMAGE_TO_BUILD}" --no-cache
 done

manifests/machineconfigcontroller/controllerconfig.yaml

@@ -9,6 +9,5 @@ spec:
   clusterName: {{.ControllerConfig.ClusterName}}
   platform: {{.ControllerConfig.Platform}}
   baseDomain: {{.ControllerConfig.BaseDomain}}
-  etcdInitialCount: {{.ControllerConfig.EtcdInitialCount}}
   etcdCAData: {{.ControllerConfig.EtcdCAData | toString | b64enc}}
   rootCAData: {{.ControllerConfig.RootCAData | toString | b64enc}}

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -58,9 +58,6 @@ type MCOConfigSpec struct {
 	Platform string `json:"platform"`
 
 	BaseDomain string `json:"baseDomain"`
-
-	// Size of the initial etcd cluster.
-	EtcdInitialCount int `json:"etcdInitialCount"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -125,9 +122,6 @@ type ControllerConfigSpec struct {
 
 	BaseDomain string `json:"baseDomain"`
 
-	// Size of the initial etcd cluster.
-	EtcdInitialCount int `json:"etcdInitialCount"`
-
 	// CAs
 	EtcdCAData []byte `json:"etcdCAData"`
 	RootCAData []byte `json:"rootCAData"`

pkg/controller/template/render.go

@@ -265,7 +265,6 @@ func renderTemplate(config renderConfig, path string, b []byte) ([]byte, error)
 	funcs["skip"] = skipMissing
 	funcs["etcdServerCertDNSNames"] = etcdServerCertDNSNames
 	funcs["etcdPeerCertDNSNames"] = etcdPeerCertDNSNames
-	funcs["etcdInitialCluster"] = etcdInitialCluster
 	funcs["apiServerURL"] = apiServerURL
 	funcs["cloudProvider"] = cloudProvider
 	tmpl, err := template.New(path).Funcs(funcs).Parse(string(b))
@@ -294,53 +293,31 @@ func skipMissing(key string) (interface{}, error) {
 
 // Process the {{etcdPeerCertDNSNames}} and {{etcdServerCertDNSNames}}
 func etcdServerCertDNSNames(cfg renderConfig) (interface{}, error) {
-	if cfg.ClusterName == "" || cfg.BaseDomain == "" || cfg.EtcdInitialCount <= 0 {
+	if cfg.BaseDomain == "" {
 		return nil, fmt.Errorf("invalid configuration")
 	}
 
 	var dnsNames = []string{
 		"localhost",
-		"*.kube-etcd.kube-system.svc.cluster.local",
-		"kube-etcd-client.kube-system.svc.cluster.local",
 		"etcd.kube-system.svc",               // sign for the local etcd service name that cluster-network apiservers use to communicate
 		"etcd.kube-system.svc.cluster.local", // sign for the local etcd service name that cluster-network apiservers use to communicate
-	}
-
-	for i := 0; i < cfg.EtcdInitialCount; i++ {
-		dnsNames = append(dnsNames, fmt.Sprintf("%s-etcd-%d.%s", cfg.ClusterName, i, cfg.BaseDomain))
+		"${ETCD_DNS_NAME}",
 	}
 	return strings.Join(dnsNames, ","), nil
 }
 
 func etcdPeerCertDNSNames(cfg renderConfig) (interface{}, error) {
-	if cfg.ClusterName == "" || cfg.BaseDomain == "" || cfg.EtcdInitialCount <= 0 {
+	if cfg.BaseDomain == "" {
 		return nil, fmt.Errorf("invalid configuration")
 	}
 
 	var dnsNames = []string{
-		"*.kube-etcd.kube-system.svc.cluster.local",
-		"kube-etcd-client.kube-system.svc.cluster.local",
-	}
-
-	for i := 0; i < cfg.EtcdInitialCount; i++ {
-		dnsNames = append(dnsNames, fmt.Sprintf("%s-etcd-%d.%s", cfg.ClusterName, i, cfg.BaseDomain))
+		"${ETCD_DNS_NAME}",
+		cfg.BaseDomain, // https://github.com/etcd-io/etcd/blob/583763261f1c843e07c1bf7fea5fb4cfb684fe87/Documentation/op-guide/clustering.md#dns-discovery
 	}
 	return strings.Join(dnsNames, ","), nil
 }
 
-func etcdInitialCluster(cfg renderConfig) (interface{}, error) {
-	if cfg.ClusterName == "" || cfg.BaseDomain == "" || cfg.EtcdInitialCount <= 0 {
-		return nil, fmt.Errorf("invalid configuration")
-	}
-
-	var addresses []string
-	for i := 0; i < cfg.EtcdInitialCount; i++ {
-		endpoint := fmt.Sprintf("%s-etcd-%d.%s", cfg.ClusterName, i, cfg.BaseDomain)
-		addresses = append(addresses, fmt.Sprintf("%s=https://%s:2380", endpoint, endpoint))
-	}
-	return strings.Join(addresses, ","), nil
-}
-
 // generate apiserver url using cluster-name, basename
 func apiServerURL(cfg renderConfig) (interface{}, error) {
 	if cfg.ClusterName == "" || cfg.BaseDomain == "" {