diff --git a/cmd/setup-etcd-environment/run.go b/cmd/setup-etcd-environment/run.go index 3b6af858d7..cc05152002 100644 --- a/cmd/setup-etcd-environment/run.go +++ b/cmd/setup-etcd-environment/run.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "errors" "flag" "fmt" @@ -14,9 +15,18 @@ import ( "github.com/golang/glog" "github.com/joho/godotenv" + ceoapi "github.com/openshift/cluster-etcd-operator/pkg/operator/api" "github.com/openshift/machine-config-operator/pkg/version" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + EtcdScalingAnnotationKey = "etcd.operator.openshift.io/scale" + etcdInitialExisting = "existing" ) var ( @@ -53,6 +63,20 @@ func runRunCmd(cmd *cobra.Command, args []string) error { return errors.New("--discovery-srv cannot be empty") } + etcdName := os.Getenv("ETCD_NAME") + if etcdName == "" { + return fmt.Errorf("environment variable ETCD_NAME has no value") + } + + etcdDataDir := os.Getenv("ETCD_DATA_DIR") + if etcdDataDir == "" { + return fmt.Errorf("environment variable ETCD_DATA_DIR has no value") + } + + if !inCluster() { + glog.V(4).Infof("KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT contain no value, running in standalone mode.") + } + ips, err := ipAddrs() if err != nil { return err @@ -62,7 +86,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { var ip string if err := wait.PollImmediate(30*time.Second, 5*time.Minute, func() (bool, error) { for _, cand := range ips { - found, err := reverseLookupSelf("etcd-server-ssl", "tcp", runOpts.discoverySRV, cand) + found, err := reverseLookup("etcd-server-ssl", "tcp", runOpts.discoverySRV, cand, runOpts.bootstrapSRV) if err != nil { glog.Errorf("error looking up self for candidate IP %s: %v", cand, err) continue @@ -72,7 +96,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { ip = cand return true, nil } - glog.V(4).Infof("no matching dns for %s", cand) + glog.V(4).Infof("no matching dns for %s in %s: %v", cand, runOpts.discoverySRV, err) } return false, nil }); err != nil { @@ -80,10 +104,19 @@ func runRunCmd(cmd *cobra.Command, args []string) error { } glog.Infof("dns name is %s", dns) - // initialize envs used to bootstrap etcd - exportEnv, err := setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV) - if err != nil { - return err + var exportEnv map[string]string + + if _, err := os.Stat(fmt.Sprintf("%s/member", etcdDataDir)); os.IsNotExist(err) && !runOpts.bootstrapSRV && inCluster() { + exportEnv, err = setExportEnv(etcdName, dns) + if err != nil { + return err + } + } else { + // initialize envs used to bootstrap etcd + exportEnv, err = setBootstrapEnv(runOpts.outputFile, runOpts.discoverySRV, runOpts.bootstrapSRV) + if err != nil { + return err + } } out := os.Stdout @@ -96,6 +129,12 @@ func runRunCmd(cmd *cobra.Command, args []string) error { out = f } + if runOpts.bootstrapSRV { + exportEnv["DISCOVERY_SRV"] = runOpts.discoverySRV + } else { + exportEnv["NAME"] = etcdName + } + // enable etcd to run using s390 and s390x. Because these are not officially supported upstream // etcd requires population of environment variable ETCD_UNSUPPORTED_ARCH at runtime. // https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/supported-platform.md @@ -114,6 +153,86 @@ func runRunCmd(cmd *cobra.Command, args []string) error { }, out, false) } +func setExportEnv(etcdName, dns string) (map[string]string, error) { + exportEnv := make(map[string]string) + duration := 10 * time.Second + wait.PollInfinite(duration, func() (bool, error) { + if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); os.IsNotExist(err) { + glog.Errorf("serviceaccount failed: %v", err) + return false, nil + } + return true, nil + }) + + clientConfig, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, fmt.Errorf("error creating client: %v", err) + } + var e ceoapi.EtcdScaling + // wait forever for success and retry every duration interval + wait.PollInfinite(duration, func() (bool, error) { + result, err := client.CoreV1().ConfigMaps("openshift-etcd").Get("member-config", metav1.GetOptions{}) + if err != nil { + glog.Errorf("error creating client %v", err) + return false, nil + } + if err := json.Unmarshal([]byte(result.Annotations[EtcdScalingAnnotationKey]), &e); err != nil { + glog.Errorf("error decoding result %v", err) + return false, nil + } + if e.Metadata.Name != etcdName { + glog.Errorf("could not find self in member-config") + return false, nil + } + members := e.Members + if len(members) == 0 { + glog.Errorf("no members found in member-config") + return false, nil + } + var memberList []string + for _, m := range members { + memberList = append(memberList, fmt.Sprintf("%s=%s", m.Name, m.PeerURLS[0])) + } + memberList = append(memberList, fmt.Sprintf("%s=https://%s:2380", etcdName, dns)) + exportEnv["INITIAL_CLUSTER"] = strings.Join(memberList, ",") + exportEnv["INITIAL_CLUSTER_STATE"] = etcdInitialExisting + return true, nil + }) + + ep, err := client.CoreV1().Endpoints("openshift-etcd").Get("etcd", metav1.GetOptions{}) + if err != nil { + return nil, err + } + hostEtcdEndpoint, err := client.CoreV1().Endpoints("openshift-etcd").Get("host-etcd", metav1.GetOptions{}) + if err != nil { + return nil, err + } + if len(hostEtcdEndpoint.Subsets) != 1 { + return nil, fmt.Errorf("openshift-etcd/host-etcd endpoint subset length should be %d, found %d", 1, len(hostEtcdEndpoint.Subsets)) + } + + endpoints := make([]string, 0) + for _, member := range hostEtcdEndpoint.Subsets[0].Addresses { + if member.Hostname == "etcd-bootstrap" { + endpoints = append(endpoints, "https://etcd-bootstrap."+runOpts.discoverySRV+":2379") + break + } + } + if len(ep.Subsets) != 1 { + return nil, fmt.Errorf("openshift-etcd/etcd endpoint subset length should be %d, found %d", 1, len(ep.Subsets)) + } + for _, s := range ep.Subsets[0].Addresses { + endpoints = append(endpoints, "https://"+s.IP+":2379") + } + + exportEnv["ENDPOINTS"] = strings.Join(endpoints, ",") + return exportEnv, nil +} + // setBootstrapEnv populates and returns a map based on envs from file. func setBootstrapEnv(envFile, discoverySRV string, bootstrapSRV bool) (map[string]string, error) { bootstrapEnv := make(map[string]string) @@ -167,29 +286,44 @@ func ipAddrs() ([]string, error) { return ips, nil } +func reverseLookup(service, proto, name, self string, bootstrapSRV bool) (string, error) { + if bootstrapSRV || inCluster() { + return reverseLookupSelf(service, proto, name, self) + } + return lookupTargetMatchSelf(fmt.Sprintf("etcd-bootstrap.%s", name), self) +} + // returns the target from the SRV record that resolves to self. func reverseLookupSelf(service, proto, name, self string) (string, error) { _, srvs, err := net.LookupSRV(service, proto, name) if err != nil { return "", err } - selfTarget := "" for _, srv := range srvs { glog.V(4).Infof("checking against %s", srv.Target) - addrs, err := net.LookupHost(srv.Target) + selfTarget, err := lookupTargetMatchSelf(srv.Target, self) if err != nil { - return "", fmt.Errorf("could not resolve member %q", srv.Target) + return "", err } - - for _, addr := range addrs { - if addr == self { - selfTarget = strings.Trim(srv.Target, ".") - break - } + if selfTarget != "" { + return selfTarget, nil } } - if selfTarget == "" { - return "", fmt.Errorf("could not find self") + return "", fmt.Errorf("could not find self") +} + +// +func lookupTargetMatchSelf(target, self string) (string, error) { + addrs, err := net.LookupHost(target) + if err != nil { + return "", fmt.Errorf("could not resolve member %q", target) + } + selfTarget := "" + for _, addr := range addrs { + if addr == self { + selfTarget = strings.Trim(target, ".") + break + } } return selfTarget, nil } @@ -208,3 +342,10 @@ func writeEnvironmentFile(m map[string]string, w io.Writer, export bool) error { } return nil } + +func inCluster() bool { + if os.Getenv("KUBERNETES_SERVICE_HOST") == "" || os.Getenv("KUBERNETES_SERVICE_PORT") == "" { + return false + } + return true +} diff --git a/go.mod b/go.mod index 2bcce3d0ab..9357dc3f1a 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/openshift/api v3.9.1-0.20191014195513-c9253efc14f4+incompatible github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0 github.com/openshift/cluster-api v0.0.0-20190923092624-4024de4fa64d + github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4 github.com/openshift/library-go v0.0.0-20191003152030-97c62d8a2901 github.com/openshift/runtime-utils v0.0.0-20191011150825-9169de69ebf6 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 3548d28420..3d4a5b65e1 100644 --- a/go.sum +++ b/go.sum @@ -685,6 +685,8 @@ github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0 h1:U0rtkdPj1lT github.com/openshift/client-go v0.0.0-20191001081553-3b0e988f8cb0/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= github.com/openshift/cluster-api v0.0.0-20191004085540-83f32d3e7070 h1:Yw3cWSzWhZ9Kh8WqX6tOccVPs9BWUp4jDfb6I8AU/pg= github.com/openshift/cluster-api v0.0.0-20191004085540-83f32d3e7070/go.mod h1:mNsD1dsD4T57kV4/C6zTHke/Ro166xgnyyRZqkamiEU= +github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4 h1:EDZ2v8AqMnXgeT3bECcbamUg8Haz0V/iXqDMlu94Mrw= +github.com/openshift/cluster-etcd-operator v0.0.0-alpha.0.0.20191025163650-5854b5c48ce4/go.mod h1:vcBAUefK8pQmTPQ3jlCemORXhnRnq3aTsfOSVeaWliY= github.com/openshift/imagebuilder v1.1.0 h1:oT704SkwMEzmIMU/+Uv1Wmvt+p10q3v2WuYMeFI18c4= github.com/openshift/imagebuilder v1.1.0/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/openshift/kubernetes v1.16.0-beta.0.0.20190913145653-2bd9643cee5b h1:b3r/3odnWNdjYRrwvXiqxLFbPYFQ+m2GblFK6RwTC9Y= diff --git a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml index d53c30e217..a6b9ee008f 100644 --- a/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml +++ b/templates/master/00-master/_base/files/etc-kubernetes-manifests-etcd-member.yaml @@ -74,7 +74,7 @@ contents: mountPath: /etc/ssl/etcd/ - name: kubeconfig mountPath: /etc/kubernetes/kubeconfig - {{if .Images.clusterEtcdOperatorImageKey}} + {{if .Images.clusterEtcdOperatorImageKey}} - name: sa mountPath: /var/run/secrets/kubernetes.io/serviceaccount/ {{end}} @@ -242,4 +242,4 @@ contents: - name: sa hostPath: path: /etc/kubernetes/static-pod-resources/etcd-member/secrets/kubernetes.io/sa-token - {{end}} \ No newline at end of file + {{end}} diff --git a/vendor/github.com/openshift/cluster-etcd-operator/LICENSE b/vendor/github.com/openshift/cluster-etcd-operator/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/vendor/github.com/openshift/cluster-etcd-operator/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go b/vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go new file mode 100644 index 0000000000..fadccc7a36 --- /dev/null +++ b/vendor/github.com/openshift/cluster-etcd-operator/pkg/operator/api/scaling.go @@ -0,0 +1,67 @@ +package api + +import ( + v1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type EtcdScaling struct { + Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` + Members []Member `json:"members,omitempty"` + // deprecated pending removal + PodFQDN string `json:"podFQDN,omitempty"` +} + +type Member struct { + ID uint64 `json:"ID,omitempty"` + Name string `json:"name,omitempty"` + PeerURLS []string `json:"peerURLs,omitempty"` + ClientURLS []string `json:"clientURLs,omitempty"` + Conditions []MemberCondition `json:"conditions,omitempty"` +} + +type MemberCondition struct { + // type describes the current condition + Type MemberConditionType `json:"type"` + // status is the status of the condition (True, False, Unknown) + Status v1.ConditionStatus `json:"status"` + // timestamp for the last update to this condition + // +optional + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + // reason is the reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty"` + // message is a human-readable explanation containing details about + // the transition + // +optional + Message string `json:"message,omitempty"` +} + +type MemberConditionType string + +const ( + // Ready indicated the member is part of the cluster and endpoint is Ready + MemberReady MemberConditionType = "Ready" + // Unknown indicated the member condition is unknown and requires further observations to verify + MemberUnknown MemberConditionType = "Unknown" + // Degraded indicates the member pod is in a degraded state and should be restarted + MemberDegraded MemberConditionType = "Degraded" + // Remove indicates the member should be removed from the cluster + MemberRemove MemberConditionType = "Remove" + // MemberAdd is a member who is ready to join cluster but currently has not + MemberAdd MemberConditionType = "Add" +) + +func GetMemberCondition(status string) MemberConditionType { + switch { + case status == string(MemberReady): + return MemberReady + case status == string(MemberRemove): + return MemberRemove + case status == string(MemberUnknown): + return MemberUnknown + case status == string(MemberDegraded): + return MemberDegraded + } + return "" +}