From f68859ea2217e804fd95aee70e27fa6ea60d4e83 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 8 Sep 2019 19:44:50 -0400 Subject: [PATCH 1/8] setup-etcd-environment: add base scaleup logic Signed-off-by: Sam Batschelet --- cmd/setup-etcd-environment/run.go | 138 +++++++++++++++--- .../etc-kubernetes-manifests-etcd-member.yaml | 4 +- 2 files changed, 123 insertions(+), 19 deletions(-) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index 3b6af858d7..1f939e297d 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "errors" "flag" "fmt" @@ -16,9 +17,14 @@ import ( "github.com/joho/godotenv" "github.com/openshift/machine-config-operator/pkg/version" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) +const EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale" + var ( runCmd = &cobra.Command{ Use: "run", @@ -35,11 +41,23 @@ var ( } ) +type EtcdScaling struct { + Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` + Members []Member `json:"members,omitempty"` +} + +type Member struct { + ID uint64 `json:"ID,omitempty"` + Name string `json:"name,omitempty"` + PeerURLS []string `json:"peerURLs,omitempty"` + ClientURLS []string `json:"clientURLs,omitempty"` +} + func init() { rootCmd.AddCommand(runCmd) rootCmd.PersistentFlags().StringVar(&runOpts.discoverySRV, "discovery-srv", "", "DNS domain used to populate envs from SRV query.") rootCmd.PersistentFlags().StringVar(&runOpts.outputFile, "output-file", "", "file where the envs are written. If empty, prints to Stdout.") - rootCmd.PersistentFlags().BoolVar(&runOpts.bootstrapSRV, "bootstrap-srv", true, "use SRV discovery for bootstraping etcd cluster.") + rootCmd.PersistentFlags().BoolVar(&runOpts.bootstrapSRV, "srv-bootstrap", true, "use SRV discovery for bootstraping etcd cluster.") } func runRunCmd(cmd *cobra.Command, args []string) error { @@ -53,6 +71,20 @@ func runRunCmd(cmd *cobra.Command, args []string) error { return errors.New("--discovery-srv cannot be empty") } + etcdName := os.Getenv("ETCD_NAME") + if etcdName == "" { + return fmt.Errorf("environment variable ETCD_NAME has no value") + } + + etcdDataDir := os.Getenv("ETCD_DATA_DIR") + if etcdDataDir == "" { + return fmt.Errorf("environment variable ETCD_DATA_DIR has no value") + } + + if !inCluster() { + glog.V(4).Infof("KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT contain no value, running in standalone mode.") + } + ips, err := ipAddrs() if err != nil { return err @@ -62,7 +94,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { var ip string if err := wait.PollImmediate(30*time.Second, 5*time.Minute, func() (bool, error) { for _, cand := range ips { - found, err := reverseLookupSelf("etcd-server-ssl", "tcp", runOpts.discoverySRV, cand) + found, err := reverseLookup("etcd-server-ssl", "tcp", runOpts.discoverySRV, cand, runOpts.bootstrapSRV) if err != nil { glog.Errorf("error looking up self for candidate IP %s: %v", cand, err) continue @@ -80,10 +112,54 @@ func runRunCmd(cmd *cobra.Command, args []string) error { } glog.Infof("dns name is %s", dns) - // initialize envs used to bootstrap etcd - exportEnv, err := setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV) - if err != nil { - return err + exportEnv := make(map[string]string) + if _, err := os.Stat(fmt.Sprintf("%s/member", etcdDataDir)); os.IsNotExist(err) && !runOpts.bootstrapSRV && inCluster() { + duration := 10 * time.Second + wait.PollInfinite(duration, func() (bool, error) { + if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); os.IsNotExist(err) { + glog.Errorf("serviceaccount failed: %v", err) + return false, nil + } + return true, nil + }) + + clientConfig, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return fmt.Errorf("error creating client: %v", err) + } + var e EtcdScaling + // wait forever for success and retry every duration interval + wait.PollInfinite(duration, func() (bool, error) { + result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("member-config", metav1.GetOptions{}) + if err != nil { + glog.Errorf("error creating client %v", err) + return false, nil + } + if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil { + glog.Errorf("error decoding result %v", err) + return false, nil + } + if e.Metadata.Name != etcdName { + glog.Errorf("could not find self in member-config") + return false, nil + } + members := e.Members + if len(members) == 0 { + glog.Errorf("no members found in member-config") + return false, nil + } + var memberList []string + for _, m := range members { + memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) + } + memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns)) + exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",") + return true, nil + }) } out := os.Stdout @@ -96,6 +172,12 @@ func runRunCmd(cmd *cobra.Command, args []string) error { out = f } + if runOpts.bootstrapSRV { + exportEnv["DISCOVERY_SRV"] = runOpts.discoverySRV + } else { + exportEnv["NAME"] = etcdName + } + // enable etcd to run using s390 and s390x. Because these are not officially supported upstream // etcd requires population of environment variable ETCD_UNSUPPORTED_ARCH at runtime. // https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/supported-platform.md @@ -167,29 +249,44 @@ func ipAddrs() ([]string, error) { return ips, nil } +func reverseLookup(service, proto, name, self string, bootstrapSRV bool) (string, error) { + if bootstrapSRV || inCluster() { + return reverseLookupSelf(service, proto, name, self) + } + return lookupTargetMatchSelf(fmt.Sprintf("etcd-bootstrap.%s", name), self) +} + // returns the target from the SRV record that resolves to self. func reverseLookupSelf(service, proto, name, self string) (string, error) { _, srvs, err := net.LookupSRV(service, proto, name) if err != nil { return "", err } - selfTarget := "" for _, srv := range srvs { glog.V(4).Infof("checking against %s", srv.Target) - addrs, err := net.LookupHost(srv.Target) + selfTarget, err := lookupTargetMatchSelf(srv.Target, self) if err != nil { - return "", fmt.Errorf("could not resolve member %q", srv.Target) + return "", err } - - for _, addr := range addrs { - if addr == self { - selfTarget = strings.Trim(srv.Target, ".") - break - } + if selfTarget != "" { + return selfTarget, nil } } - if selfTarget == "" { - return "", fmt.Errorf("could not find self") + return "", fmt.Errorf("could not find self") +} + +// +func lookupTargetMatchSelf(target string, self string) (string, error) { + addrs, err := net.LookupHost(target) + if err != nil { + return "", fmt.Errorf("could not resolve member %q", target) + } + selfTarget := "" + for _, addr := range addrs { + if addr == self { + selfTarget = strings.Trim(target, ".") + break + } } return selfTarget, nil } @@ -208,3 +305,10 @@ func writeEnvironmentFile(m map[string]string, w io.Writer, export bool) error { } return nil } + +func inCluster() bool { + if os.Getenv("KUBERNETES_SERVICE_HOST") == "" || os.Getenv("KUBERNETES_SERVICE_PORT") == "" { + return false + } + return true +} diff --git a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml index d53c30e217..a6b9ee008f 100644 --- a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml +++ b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml @@ -74,7 +74,7 @@ contents: mountPath: /etc/ssl/etcd/ - name: kubeconfig mountPath: /etc/kubernetes/kubeconfig - {{if .Images.clusterEtcdOperatorImageKey}} + {{if .Images.clusterEtcdOperatorImageKey}} - name: sa mountPath: /var/run/secrets/kubernetes.io/serviceaccount/ {{end}} @@ -242,4 +242,4 @@ contents: - name: sa hostPath: path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token - {{end}} \ No newline at end of file + {{end}} From f43ecedb17ed377f47e50fd9eec7358924e2f6f7 Mon Sep 17 00:00:00 2001 From: Alay Patel Date: Tue, 17 Sep 2019 11:19:46 -0400 Subject: [PATCH 2/8] setup-etcd-environment: add logic to mount certs --- cmd/setup-etcd-environment/run.go | 89 +++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index 1f939e297d..812287a327 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -7,8 +7,12 @@ import ( "flag" "fmt" "io" + "io/ioutil" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" "net" "os" + "path" "runtime" "strings" "time" @@ -18,12 +22,14 @@ import ( "github.com/openshift/machine-config-operator/pkg/version" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) -const EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale" +const ( + EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale" + assetDir = "/etc/ssl/etcd" +) var ( runCmd = &cobra.Command{ @@ -44,6 +50,7 @@ var ( type EtcdScaling struct { Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` Members []Member `json:"members,omitempty"` + PodFQDN string `json:"podFQDN,omitempty"` } type Member struct { @@ -57,7 +64,7 @@ func init() { rootCmd.AddCommand(runCmd) rootCmd.PersistentFlags().StringVar(&runOpts.discoverySRV, "discovery-srv", "", "DNS domain used to populate envs from SRV query.") rootCmd.PersistentFlags().StringVar(&runOpts.outputFile, "output-file", "", "file where the envs are written. If empty, prints to Stdout.") - rootCmd.PersistentFlags().BoolVar(&runOpts.bootstrapSRV, "srv-bootstrap", true, "use SRV discovery for bootstraping etcd cluster.") + rootCmd.PersistentFlags().BoolVar(&runOpts.bootstrapSRV, "bootstrap-srv", true, "use SRV discovery for bootstraping etcd cluster.") } func runRunCmd(cmd *cobra.Command, args []string) error { @@ -152,12 +159,17 @@ func runRunCmd(cmd *cobra.Command, args []string) error { glog.Errorf("no members found in member-config") return false, nil } + err = waitForCerts(client, "openshift-etcd", etcdName, e.PodFQDN) + if err != nil { + return false, nil + } var memberList []string for _, m := range members { memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) } memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns)) exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",") + exportEnv["INITIAL_CLUSTER_STATE"] = "existing" return true, nil }) } @@ -312,3 +324,74 @@ func inCluster() bool { } return true } + +// TODO: Port this logic into Quay +func waitForCerts(client *kubernetes.Clientset, secretNamespace, podName, podFQDN string) error { + peerSecretName := podName + "-peer" + serverSecretName := podName + "-server" + metricSecretName := podName + "-metric" + peerCN := "system:etcd-peer:" + podFQDN + secretCN := "system:etcd-server:" + podFQDN + metricCN := "system:etcd-metric:" + podFQDN + + glog.Infof("getting peer secret %v/%v\n", secretNamespace, peerSecretName) + if err := writeSecret(client, peerSecretName, secretNamespace, peerCN); err != nil { + return err + } + + // wait for server certs + glog.Infof("getting server secret %v/%v\n", secretNamespace, serverSecretName) + if err := writeSecret(client, serverSecretName, secretNamespace, secretCN); err != nil { + return err + } + + // wait for server certs + glog.Infof("getting metric secret %v/%v\n", secretNamespace, metricSecretName) + if err := writeSecret(client, metricSecretName, secretNamespace, metricCN); err != nil { + return err + } + + return nil +} + +func writeSecret(client *kubernetes.Clientset, secretName, secretNamespace, cn string) error { + glog.Infof("getting secret %v/%v\n", secretNamespace, secretNamespace) + secret, err := client.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("error in getting secret %s/%s: %v", secretNamespace, secretName, err) + return err + } + glog.Infof("ensure secret keys %v/%v\n", secretNamespace, secretName) + err = ensureCertKeys(secret.Data) + if err != nil { + return err + } + + glog.Infof("writing secret to %v\n", cn) + err = writeToFile(secret, cn) + if err != nil { + return err + } + return nil +} + +func ensureCertKeys(data map[string][]byte) error { + if len(data["tls.crt"]) == 0 || len(data["tls.key"]) == 0 { + return fmt.Errorf("invalid secret data") + } + return nil +} + +func writeToFile(s *v1.Secret, commonName string) error { + // write out signed certificate to disk + certFile := path.Join(assetDir, commonName+".crt") + //fmt.Printf("%s", s.Data["tls.crt"]) + if err := ioutil.WriteFile(certFile, s.Data["tls.crt"], 0644); err != nil { + return fmt.Errorf("unable to write to %s: %v", certFile, err) + } + keyFile := path.Join(assetDir, commonName+".key") + if err := ioutil.WriteFile(keyFile, s.Data["tls.key"], 0644); err != nil { + return fmt.Errorf("unable to write to %s: %v", keyFile, err) + } + return nil +} From b67b2d40e3131c39fe0a57bb7ed895e6414080ba Mon Sep 17 00:00:00 2001 From: Alay Patel Date: Sat, 14 Sep 2019 11:46:13 -0400 Subject: [PATCH 3/8] *: use kubecsr mount, add readiness probe to etcd pod --- cmd/setup-etcd-environment/run.go | 80 +------------------------------ 1 file changed, 1 insertion(+), 79 deletions(-) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index 812287a327..af6807ef8c 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -7,12 +7,9 @@ import ( "flag" "fmt" "io" - "io/ioutil" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "net" "os" - "path" "runtime" "strings" "time" @@ -111,7 +108,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { ip = cand return true, nil } - glog.V(4).Infof("no matching dns for %s", cand) + glog.V(4).Infof("no matching dns for %s in %s: %v", cand, runOpts.discoverySRV, err) } return false, nil }); err != nil { @@ -159,10 +156,6 @@ func runRunCmd(cmd *cobra.Command, args []string) error { glog.Errorf("no members found in member-config") return false, nil } - err = waitForCerts(client, "openshift-etcd", etcdName, e.PodFQDN) - if err != nil { - return false, nil - } var memberList []string for _, m := range members { memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) @@ -324,74 +317,3 @@ func inCluster() bool { } return true } - -// TODO: Port this logic into Quay -func waitForCerts(client *kubernetes.Clientset, secretNamespace, podName, podFQDN string) error { - peerSecretName := podName + "-peer" - serverSecretName := podName + "-server" - metricSecretName := podName + "-metric" - peerCN := "system:etcd-peer:" + podFQDN - secretCN := "system:etcd-server:" + podFQDN - metricCN := "system:etcd-metric:" + podFQDN - - glog.Infof("getting peer secret %v/%v\n", secretNamespace, peerSecretName) - if err := writeSecret(client, peerSecretName, secretNamespace, peerCN); err != nil { - return err - } - - // wait for server certs - glog.Infof("getting server secret %v/%v\n", secretNamespace, serverSecretName) - if err := writeSecret(client, serverSecretName, secretNamespace, secretCN); err != nil { - return err - } - - // wait for server certs - glog.Infof("getting metric secret %v/%v\n", secretNamespace, metricSecretName) - if err := writeSecret(client, metricSecretName, secretNamespace, metricCN); err != nil { - return err - } - - return nil -} - -func writeSecret(client *kubernetes.Clientset, secretName, secretNamespace, cn string) error { - glog.Infof("getting secret %v/%v\n", secretNamespace, secretNamespace) - secret, err := client.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{}) - if err != nil { - glog.Errorf("error in getting secret %s/%s: %v", secretNamespace, secretName, err) - return err - } - glog.Infof("ensure secret keys %v/%v\n", secretNamespace, secretName) - err = ensureCertKeys(secret.Data) - if err != nil { - return err - } - - glog.Infof("writing secret to %v\n", cn) - err = writeToFile(secret, cn) - if err != nil { - return err - } - return nil -} - -func ensureCertKeys(data map[string][]byte) error { - if len(data["tls.crt"]) == 0 || len(data["tls.key"]) == 0 { - return fmt.Errorf("invalid secret data") - } - return nil -} - -func writeToFile(s *v1.Secret, commonName string) error { - // write out signed certificate to disk - certFile := path.Join(assetDir, commonName+".crt") - //fmt.Printf("%s", s.Data["tls.crt"]) - if err := ioutil.WriteFile(certFile, s.Data["tls.crt"], 0644); err != nil { - return fmt.Errorf("unable to write to %s: %v", certFile, err) - } - keyFile := path.Join(assetDir, commonName+".key") - if err := ioutil.WriteFile(keyFile, s.Data["tls.key"], 0644); err != nil { - return fmt.Errorf("unable to write to %s: %v", keyFile, err) - } - return nil -} From a56735d1ede127255a774a0caa79310146be0c15 Mon Sep 17 00:00:00 2001 From: Alay Patel Date: Wed, 9 Oct 2019 02:53:56 -0400 Subject: [PATCH 4/8] cmd/setup-etcd-env: set ENDPOINTS after it is populated --- cmd/setup-etcd-environment/run.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index af6807ef8c..cfe4c47c7c 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -117,6 +117,8 @@ func runRunCmd(cmd *cobra.Command, args []string) error { glog.Infof("dns name is %s", dns) exportEnv := make(map[string]string) + + endpoints := make([]string, 0) if _, err := os.Stat(fmt.Sprintf("%s/member", etcdDataDir)); os.IsNotExist(err) && !runOpts.bootstrapSRV && inCluster() { duration := 10 * time.Second wait.PollInfinite(duration, func() (bool, error) { @@ -165,6 +167,21 @@ func runRunCmd(cmd *cobra.Command, args []string) error { exportEnv["INITIAL_CLUSTER_STATE"] = "existing" return true, nil }) + + ep, err := client.CoreV1().Endpoints("openshift-etcd").Get("etcd", metav1.GetOptions{}) + if err != nil { + return err + } + if len(ep.Subsets) >= 1 { + if len(ep.Subsets[0].Addresses) == 0 { + endpoints = append(endpoints, "https://etcd-bootstrap."+runOpts.discoverySRV+":2379") + } else { + for _, s := range ep.Subsets[0].Addresses { + endpoints = append(endpoints, "https://"+s.IP+":2379") + } + } + } + exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",") } out := os.Stdout From d96a1cca54b03c19b36421542bc8bea8b1a97efd Mon Sep 17 00:00:00 2001 From: Alay Patel Date: Mon, 28 Oct 2019 12:56:35 -0400 Subject: [PATCH 5/8] cmd/setup-etcd-environment: remove etcd-bootstrap hard coded value --- cmd/setup-etcd-environment/run.go | 41 ++-- .../openshift/cluster-etcd-operator/LICENSE | 201 ++++++++++++++++++ .../pkg/operator/api/scaling.go | 67 ++++++ 3 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 vendor/github.com/openshift/cluster-etcd-operator/LICENSE create mode 100644 vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index cfe4c47c7c..d66ce254a2 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "k8s.io/apimachinery/pkg/util/wait" "net" "os" "runtime" @@ -16,9 +15,11 @@ import ( "github.com/golang/glog" "github.com/joho/godotenv" + ceoapi "github.com/openshift/cluster-etcd-operator/pkg/operator/api" "github.com/openshift/machine-config-operator/pkg/version" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -44,19 +45,6 @@ var ( } ) -type EtcdScaling struct { - Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` - Members []Member `json:"members,omitempty"` - PodFQDN string `json:"podFQDN,omitempty"` -} - -type Member struct { - ID uint64 `json:"ID,omitempty"` - Name string `json:"name,omitempty"` - PeerURLS []string `json:"peerURLs,omitempty"` - ClientURLS []string `json:"clientURLs,omitempty"` -} - func init() { rootCmd.AddCommand(runCmd) rootCmd.PersistentFlags().StringVar(&runOpts.discoverySRV, "discovery-srv", "", "DNS domain used to populate envs from SRV query.") @@ -137,7 +125,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("error creating client: %v", err) } - var e EtcdScaling + var e ceoapi.EtcdScaling // wait forever for success and retry every duration interval wait.PollInfinite(duration, func() (bool, error) { result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("member-config", metav1.GetOptions{}) @@ -172,15 +160,26 @@ func runRunCmd(cmd *cobra.Command, args []string) error { if err != nil { return err } - if len(ep.Subsets) >= 1 { - if len(ep.Subsets[0].Addresses) == 0 { + hostEtcdEndpoint, err := client.CoreV1().Endpoints("openshift-etcd").Get("host-etcd", metav1.GetOptions{}) + if err != nil { + return err + } + if len(hostEtcdEndpoint.Subsets) != 1 { + return fmt.Errorf("openshift-etcd/host-etcd endpoint subset length should be %d, found %d", 1, len(hostEtcdEndpoint.Subsets)) + } + for _, member := range hostEtcdEndpoint.Subsets[0].Addresses { + if member.Hostname == "etcd-bootstrap" { endpoints = append(endpoints, "https://etcd-bootstrap."+runOpts.discoverySRV+":2379") - } else { - for _, s := range ep.Subsets[0].Addresses { - endpoints = append(endpoints, "https://"+s.IP+":2379") - } + break } } + if len(ep.Subsets) != 1 { + return fmt.Errorf("openshift-etcd/etcd endpoint subset length should be %d, found %d", 1, len(ep.Subsets)) + } + for _, s := range ep.Subsets[0].Addresses { + endpoints = append(endpoints, "https://"+s.IP+":2379") + } + exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",") } diff --git a/vendor/github.com/openshift/cluster-etcd-operator/LICENSE b/vendor/github.com/openshift/cluster-etcd-operator/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/vendor/github.com/openshift/cluster-etcd-operator/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go b/vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go new file mode 100644 index 0000000000..fadccc7a36 --- /dev/null +++ b/vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go @@ -0,0 +1,67 @@ +package api + +import ( + v1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type EtcdScaling struct { + Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` + Members []Member `json:"members,omitempty"` + // deprecated pending removal + PodFQDN string `json:"podFQDN,omitempty"` +} + +type Member struct { + ID uint64 `json:"ID,omitempty"` + Name string `json:"name,omitempty"` + PeerURLS []string `json:"peerURLs,omitempty"` + ClientURLS []string `json:"clientURLs,omitempty"` + Conditions []MemberCondition `json:"conditions,omitempty"` +} + +type MemberCondition struct { + // type describes the current condition + Type MemberConditionType `json:"type"` + // status is the status of the condition (True, False, Unknown) + Status v1.ConditionStatus `json:"status"` + // timestamp for the last update to this condition + // +optional + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + // reason is the reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty"` + // message is a human-readable explanation containing details about + // the transition + // +optional + Message string `json:"message,omitempty"` +} + +type MemberConditionType string + +const ( + // Ready indicated the member is part of the cluster and endpoint is Ready + MemberReady MemberConditionType = "Ready" + // Unknown indicated the member condition is unknown and requires further observations to verify + MemberUnknown MemberConditionType = "Unknown" + // Degraded indicates the member pod is in a degraded state and should be restarted + MemberDegraded MemberConditionType = "Degraded" + // Remove indicates the member should be removed from the cluster + MemberRemove MemberConditionType = "Remove" + // MemberAdd is a member who is ready to join cluster but currently has not + MemberAdd MemberConditionType = "Add" +) + +func GetMemberCondition(status string) MemberConditionType { + switch { + case status == string(MemberReady): + return MemberReady + case status == string(MemberRemove): + return MemberRemove + case status == string(MemberUnknown): + return MemberUnknown + case status == string(MemberDegraded): + return MemberDegraded + } + return "" +} From 5741f892b8e2db31612e662fccd45da3df5f5ce7 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Tue, 29 Oct 2019 14:54:32 -0400 Subject: [PATCH 6/8] *: resolve verify failures --- cmd/setup-etcd-environment/run.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index d66ce254a2..9486fc4283 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -26,7 +26,7 @@ import ( const ( EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale" - assetDir = "/etc/ssl/etcd" + etcdInitialExisting = "existing" ) var ( @@ -152,7 +152,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { } memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns)) exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",") - exportEnv["INITIAL_CLUSTER_STATE"] = "existing" + exportEnv["INITIAL_CLUSTER_STATE"] = etcdInitialExisting return true, nil }) @@ -181,6 +181,12 @@ func runRunCmd(cmd *cobra.Command, args []string) error { } exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",") + } else { + // initialize envs used to bootstrap etcd + exportEnv, err = setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV) + if err != nil { + return err + } } out := os.Stdout @@ -297,7 +303,7 @@ func reverseLookupSelf(service, proto, name, self string) (string, error) { } // -func lookupTargetMatchSelf(target string, self string) (string, error) { +func lookupTargetMatchSelf(target, self string) (string, error) { addrs, err := net.LookupHost(target) if err != nil { return "", fmt.Errorf("could not resolve member %q", target) From e3163a5165bc4833d42488e6e74cbf1c7a12f0bc Mon Sep 17 00:00:00 2001 From: retroflexer Date: Sat, 2 Nov 2019 17:17:01 -0400 Subject: [PATCH 7/8] cmd/setup-etcd-environment: reduce cyclomatic complexity --- cmd/setup-etcd-environment/run.go | 154 ++++++++++++++++-------------- 1 file changed, 82 insertions(+), 72 deletions(-) diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index 9486fc4283..cc05152002 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -104,83 +104,13 @@ func runRunCmd(cmd *cobra.Command, args []string) error { } glog.Infof("dns name is %s", dns) - exportEnv := make(map[string]string) + var exportEnv map[string]string - endpoints := make([]string, 0) if _, err := os.Stat(fmt.Sprintf("%s/member", etcdDataDir)); os.IsNotExist(err) && !runOpts.bootstrapSRV && inCluster() { - duration := 10 * time.Second - wait.PollInfinite(duration, func() (bool, error) { - if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); os.IsNotExist(err) { - glog.Errorf("serviceaccount failed: %v", err) - return false, nil - } - return true, nil - }) - - clientConfig, err := rest.InClusterConfig() - if err != nil { - panic(err.Error()) - } - client, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return fmt.Errorf("error creating client: %v", err) - } - var e ceoapi.EtcdScaling - // wait forever for success and retry every duration interval - wait.PollInfinite(duration, func() (bool, error) { - result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("member-config", metav1.GetOptions{}) - if err != nil { - glog.Errorf("error creating client %v", err) - return false, nil - } - if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil { - glog.Errorf("error decoding result %v", err) - return false, nil - } - if e.Metadata.Name != etcdName { - glog.Errorf("could not find self in member-config") - return false, nil - } - members := e.Members - if len(members) == 0 { - glog.Errorf("no members found in member-config") - return false, nil - } - var memberList []string - for _, m := range members { - memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) - } - memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns)) - exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",") - exportEnv["INITIAL_CLUSTER_STATE"] = etcdInitialExisting - return true, nil - }) - - ep, err := client.CoreV1().Endpoints("openshift-etcd").Get("etcd", metav1.GetOptions{}) - if err != nil { - return err - } - hostEtcdEndpoint, err := client.CoreV1().Endpoints("openshift-etcd").Get("host-etcd", metav1.GetOptions{}) + exportEnv, err = setExportEnv(etcdName, dns) if err != nil { return err } - if len(hostEtcdEndpoint.Subsets) != 1 { - return fmt.Errorf("openshift-etcd/host-etcd endpoint subset length should be %d, found %d", 1, len(hostEtcdEndpoint.Subsets)) - } - for _, member := range hostEtcdEndpoint.Subsets[0].Addresses { - if member.Hostname == "etcd-bootstrap" { - endpoints = append(endpoints, "https://etcd-bootstrap."+runOpts.discoverySRV+":2379") - break - } - } - if len(ep.Subsets) != 1 { - return fmt.Errorf("openshift-etcd/etcd endpoint subset length should be %d, found %d", 1, len(ep.Subsets)) - } - for _, s := range ep.Subsets[0].Addresses { - endpoints = append(endpoints, "https://"+s.IP+":2379") - } - - exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",") } else { // initialize envs used to bootstrap etcd exportEnv, err = setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV) @@ -223,6 +153,86 @@ func runRunCmd(cmd *cobra.Command, args []string) error { }, out, false) } +func setExportEnv(etcdName, dns string) (map[string]string, error) { + exportEnv := make(map[string]string) + duration := 10 * time.Second + wait.PollInfinite(duration, func() (bool, error) { + if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); os.IsNotExist(err) { + glog.Errorf("serviceaccount failed: %v", err) + return false, nil + } + return true, nil + }) + + clientConfig, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, fmt.Errorf("error creating client: %v", err) + } + var e ceoapi.EtcdScaling + // wait forever for success and retry every duration interval + wait.PollInfinite(duration, func() (bool, error) { + result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("member-config", metav1.GetOptions{}) + if err != nil { + glog.Errorf("error creating client %v", err) + return false, nil + } + if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil { + glog.Errorf("error decoding result %v", err) + return false, nil + } + if e.Metadata.Name != etcdName { + glog.Errorf("could not find self in member-config") + return false, nil + } + members := e.Members + if len(members) == 0 { + glog.Errorf("no members found in member-config") + return false, nil + } + var memberList []string + for _, m := range members { + memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) + } + memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns)) + exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",") + exportEnv["INITIAL_CLUSTER_STATE"] = etcdInitialExisting + return true, nil + }) + + ep, err := client.CoreV1().Endpoints("openshift-etcd").Get("etcd", metav1.GetOptions{}) + if err != nil { + return nil, err + } + hostEtcdEndpoint, err := client.CoreV1().Endpoints("openshift-etcd").Get("host-etcd", metav1.GetOptions{}) + if err != nil { + return nil, err + } + if len(hostEtcdEndpoint.Subsets) != 1 { + return nil, fmt.Errorf("openshift-etcd/host-etcd endpoint subset length should be %d, found %d", 1, len(hostEtcdEndpoint.Subsets)) + } + + endpoints := make([]string, 0) + for _, member := range hostEtcdEndpoint.Subsets[0].Addresses { + if member.Hostname == "etcd-bootstrap" { + endpoints = append(endpoints, "https://etcd-bootstrap."+runOpts.discoverySRV+":2379") + break + } + } + if len(ep.Subsets) != 1 { + return nil, fmt.Errorf("openshift-etcd/etcd endpoint subset length should be %d, found %d", 1, len(ep.Subsets)) + } + for _, s := range ep.Subsets[0].Addresses { + endpoints = append(endpoints, "https://"+s.IP+":2379") + } + + exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",") + return exportEnv, nil +} + // setBootstrapEnv populates and returns a map based on envs from file. func setBootstrapEnv(envFile, discoverySRV string, bootstrapSRV bool) (map[string]string, error) { bootstrapEnv := make(map[string]string) From 37e68516031bead1bc4eefe086648c0f90efcb17 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Mon, 4 Nov 2019 06:41:52 -0500 Subject: [PATCH 8/8] vendor: add cluster-etcd-operator Signed-off-by: Sam Batschelet --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 2bcce3d0ab..9357dc3f1a 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/openshift/api v3.9.1-0.20191014195513-c9253efc14f4+incompatible github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0 github.com/openshift/cluster-api v0.0.0-20190923092624-4024de4fa64d + github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4 github.com/openshift/library-go v0.0.0-20191003152030-97c62d8a2901 github.com/openshift/runtime-utils v0.0.0-20191011150825-9169de69ebf6 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 3548d28420..3d4a5b65e1 100644 --- a/go.sum +++ b/go.sum @@ -685,6 +685,8 @@ github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0 h1:U0rtkdPj1lT github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= github.com/openshift/cluster-api v0.0.0-20191004085540-83f32d3e7070 h1:Yw3cWSzWhZ9Kh8WqX6tOccVPs9BWUp4jDfb6I8AU/pg= github.com/openshift/cluster-api v0.0.0-20191004085540-83f32d3e7070/go.mod h1:mNsD1dsD4T57kV4/C6zTHke/Ro166xgnyyRZqkamiEU= +github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4 h1:EDZ2v8AqMnXgeT3bECcbamUg8Haz0V/iXqDMlu94Mrw= +github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4/go.mod h1:vcBAUefK8pQmTPQ3jlCemORXhnRnq3aTsfOSVeaWliY= github.com/openshift/imagebuilder v1.1.0 h1:oT704SkwMEzmIMU/+Uv1Wmvt+p10q3v2WuYMeFI18c4= github.com/openshift/imagebuilder v1.1.0/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/openshift/kubernetes v1.16.0-beta.0.0.20190913145653-2bd9643cee5b h1:b3r/3odnWNdjYRrwvXiqxLFbPYFQ+m2GblFK6RwTC9Y=