From c611bec87d9d65312eb9d1f12fbeb76897b9359e Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Thu, 25 Jul 2019 16:45:26 -0400 Subject: [PATCH] cmd/setup-etcd-environment: add etcd pivot logic Signed-off-by: Sam Batschelet --- cmd/setup-etcd-environment/run.go | 79 ++++++++++++++++++- .../etc-kubernetes-manifests-etcd-member.yaml | 22 ++++++ 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index 63a1380421..b6192f7f0a 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "errors" "flag" "fmt" @@ -14,7 +15,11 @@ import ( "github.com/golang/glog" "github.com/openshift/machine-config-operator/pkg/version" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog" ) var ( @@ -29,9 +34,23 @@ var ( discoverySRV string ifName string outputFile string + pivot bool } + EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale" ) +type EtcdScaling struct { + Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` + Members []Member `json:"members,omitempty"` +} + +type Member struct { + ID uint64 `json:"ID,omitempty"` + Name string `json:"name,omitempty"` + PeerURLS []string `json:"peerURLs,omitempty"` + ClientURLS []string `json:"clientURLs,omitempty"` +} + func init() { rootCmd.AddCommand(runCmd) rootCmd.PersistentFlags().StringVar(&runOpts.discoverySRV, "discovery-srv", "", "DNS domain used to bootstrap initial etcd cluster.") @@ -86,9 +105,54 @@ func runRunCmd(cmd *cobra.Command, args []string) error { out = f } - if err := writeEnvironmentFile(map[string]string{ - "DISCOVERY_SRV": runOpts.discoverySRV, - }, out, true); err != nil { + export := make(map[string]string) + etcdName := os.Getenv("ETCD_NAME") + if etcdName == "" { + return fmt.Errorf("environment variable ETCD_NAME has no value") + } + var e EtcdScaling + if runOpts.pivot { + clientConfig, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return fmt.Errorf("error creating client: %v", err) + } + duration := 10 * time.Second + // wait forever for success and retry every duration interval + wait.PollInfinite(duration, func() (bool, error) { + result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("scaling-lock", metav1.GetOptions{}) + if err != nil { + klog.Errorf("error creating client %v", err) + return false, nil + } + if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil { + klog.Errorf("error decoding result %v", err) + return false, nil + } + if e.Metadata.Name != etcdName { + klog.Errorf("could not find self in scaling-lock") + return false, nil + } + members := e.Members + if len(members) == 0 { + klog.Errorf("no members found in scaling-lock") + return false, nil + } + var memberList []string + for _, m := range members { + memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) + } + memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, ip)) + export["INITIAL_CLUSTER"] = strings.Join(memberList, ",") + return true, nil + }) + } else { + export["DISCOVERY_SRV"] = runOpts.discoverySRV + } + if err := writeEnvironmentFile(export, out, true); err != nil { return err } @@ -137,6 +201,9 @@ func reverseLookupSelf(service, proto, name, self string) (string, error) { } selfTarget := "" for _, srv := range srvs { + if isPivot(srv.Target) { + runOpts.pivot = true + } glog.V(4).Infof("checking against %s", srv.Target) addrs, err := net.LookupHost(srv.Target) if err != nil { @@ -156,10 +223,14 @@ func reverseLookupSelf(service, proto, name, self string) (string, error) { return selfTarget, nil } +func isPivot(target string) bool { + return strings.HasPrefix(target, "etcd-bootstrap") +} + func writeEnvironmentFile(m map[string]string, w io.Writer, export bool) error { var buffer bytes.Buffer for k, v := range m { - env := fmt.Sprintf("ETCD_%s=%s\n", k, v) + env := fmt.Sprintf("ETCD_%s=\"%s\"\n", k, v) if export == true { env = fmt.Sprintf("export %s", env) } diff --git a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml index 1c5a1124be..d7e0b26e2a 100644 --- a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml +++ b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml @@ -23,6 +23,13 @@ contents: volumeMounts: - name: discovery mountPath: /run/etcd/ + - name: sa + mountPath: /var/run/secrets/kubernetes.io/serviceaccount/ + env: + - name: ETCD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: certs image: "{{.Images.kubeClientAgentImageKey}}" command: @@ -105,6 +112,10 @@ contents: --listen-client-urls=https://0.0.0.0:2379 \ --listen-peer-urls=https://0.0.0.0:2380 \ --listen-metrics-urls=https://0.0.0.0:9978 \ + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", "sleep 5"] resources: requests: memory: 600Mi @@ -129,6 +140,10 @@ contents: valueFrom: fieldRef: fieldPath: metadata.name + - name: ETCD_INITIAL_CLUSTER + valueFrom: + fieldRef: + fieldPath: metadata.annotations['etcd.operator.openshift.io/scale'] ports: - name: peer containerPort: 2380 @@ -157,6 +172,10 @@ contents: --cert-file /etc/ssl/etcd/system:etcd-metric:${ETCD_DNS_NAME}.crt \ --cacert /etc/ssl/etcd/ca.crt \ --trusted-ca-file /etc/ssl/etcd/metric-ca.crt \ + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", "sleep 5"] terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: discovery @@ -191,3 +210,6 @@ contents: - name: conf hostPath: path: /etc/etcd + - name: sa + hostPath: + path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token