diff --git a/Dockerfile.setup-etcd-environment b/Dockerfile.setup-etcd-environment new file mode 100644 index 0000000000..02575917fb --- /dev/null +++ b/Dockerfile.setup-etcd-environment @@ -0,0 +1,10 @@ +FROM golang:1.10.3 AS build-env + +COPY . /go/src/github.com/openshift/machine-config-operator +WORKDIR /go/src/github.com/openshift/machine-config-operator +RUN WHAT=setup-etcd-environment ./hack/build-go.sh + +FROM openshift/origin-base:v4.0.0 +COPY --from=build-env /go/src/github.com/openshift/machine-config-operator/_output/linux/amd64/setup-etcd-environment /bin/setup-etcd-environment + +ENTRYPOINT ["/bin/setup-etcd-environment"] diff --git a/cmd/setup-etcd-environment/main.go b/cmd/setup-etcd-environment/main.go new file mode 100644 index 0000000000..c4bd690b25 --- /dev/null +++ b/cmd/setup-etcd-environment/main.go @@ -0,0 +1,167 @@ +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "net" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/golang/glog" + "github.com/spf13/cobra" +) + +const ( + componentName = "etcd-setup-environment" +) + +var ( + rootOpts struct { + discoverySRV string + ifName string + outputFile string + } +) + +func main() { + rootCmd := &cobra.Command{ + Use: componentName, + Short: "Sets up the environment for etcd", + Long: "", + RunE: runRootCmd, + SilenceErrors: true, + SilenceUsage: true, + } + rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine) + rootCmd.PersistentFlags().StringVar(&rootOpts.discoverySRV, "discovery-srv", "", "DNS domain used to bootstrap initial etcd cluster.") + rootCmd.PersistentFlags().StringVar(&rootOpts.ifName, "if-name", "eth0", "The network interface that should be used for getting local ip address.") + rootCmd.PersistentFlags().StringVar(&rootOpts.outputFile, "output-file", "", "file where the envs are written. If empty, prints to Stdout.") + + if err := rootCmd.Execute(); err != nil { + glog.Exitf("Error executing %s: %v", componentName, err) + } +} + +func runRootCmd(cmd *cobra.Command, args []string) error { + flag.Set("logtostderr", "true") + flag.Parse() + + if rootOpts.discoverySRV == "" { + return errors.New("--discovery-srv cannot be empty") + } + + ip, err := ipAddrForIf(rootOpts.ifName) + if err != nil { + return err + } + glog.Infof("ip addr is %s", ip) + + var dns string + if err := wait.PollImmediate(1*time.Minute, 5*time.Minute, func() (bool, error) { + found, err := reverseLookupSelf("etcd-server-ssl", "tcp", rootOpts.discoverySRV, ip) + if err != nil { + glog.Errorf("error looking up self: %v", err) + return false, nil + } + if found != "" { + dns = found + return true, nil + } + return false, errors.New("found dns is invalid") + }); err != nil { + return fmt.Errorf("could not find self: %v", err) + } + glog.Infof("dns name is %s", dns) + + out := os.Stdout + if rootOpts.outputFile != "" { + f, err := os.Create(rootOpts.outputFile) + if err != nil { + return err + } + defer f.Close() + out = f + } + + return writeEnvironmentFile(map[string]string{ + "IPV4_ADDRESS": ip, + "DNS_NAME": dns, + }, out) +} + +func ipAddrForIf(ifname string) (string, error) { + ifaces, err := net.Interfaces() + if err != nil { + return "", err + } + for _, i := range ifaces { + if i.Name != ifname { + continue + } + + addrs, err := i.Addrs() + if err != nil { + return "", err + } + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + if ip == nil { + continue + } + ip = ip.To4() + if ip == nil { + continue // not an ipv4 address + } + if !ip.IsGlobalUnicast() { + continue // we only want global unicast address + } + return ip.String(), nil + } + } + return "", fmt.Errorf("could not find ip address for %s", ifname) +} + +// returns the target from the SRV record that resolves to self. +func reverseLookupSelf(service, proto, name, self string) (string, error) { + _, srvs, err := net.LookupSRV(service, proto, name) + if err != nil { + return "", err + } + for _, srv := range srvs { + glog.V(4).Infof("checking against %s", srv.Target) + addrs, err := net.LookupHost(srv.Target) + if err != nil { + continue // don't care + } + + for _, addr := range addrs { + if addr == self { + return strings.Trim(srv.Target, "."), nil + } + } + } + return "", fmt.Errorf("could not find self") +} + +func writeEnvironmentFile(m map[string]string, w io.Writer) error { + var buffer bytes.Buffer + for k, v := range m { + buffer.WriteString(fmt.Sprintf("ETCD_%s=%s\n", k, v)) + } + if _, err := buffer.WriteTo(w); err != nil { + return err + } + return nil +} diff --git a/docs/MachineConfigServer.md b/docs/MachineConfigServer.md index 52ce5de373..583442b295 100644 --- a/docs/MachineConfigServer.md +++ b/docs/MachineConfigServer.md @@ -22,20 +22,12 @@ MachineConfigServer serves Ignition at `/config/` endpoint. * If the server cannot find the machine pool requested in the URL, the server returns HTTP Status Code 404 with an empty response. -### Special parameter for master machine configuration - -The etcd members are co-located with the control plane on `master` machines. Bootstrapping an etcd cluster requires special code path that depends on the index of the machine in the pool. Therefore, the server needs to support `etcd_index` query parameter for `master` machine pools to serve the correct Ignition config. - ### Ignition config from MachineConfig MachineConfigServer serves the Ignition config defined in `spec.config` fields of the appropriate MachineConfig object. It performs the following extra actions on the Ignition config defined in the MachineConfig object before serving it: -* *etcd member index templating* - - The etcd unit files generated by MachineConfigController are go text templates with `etcd_index` variable. This variable needs to replaces with the index specified in `etcd_index` query. - * *Ignition file for MachineConfigDaemon* MachineConfigDaemon requires a file on disk (node annotations), to seed the `currentConfig` & `desiredConfig` annotations to its node object. The file is JSON object that contains the reference to `MachineConfig` object used to generate the Ignition config for the machine. @@ -58,77 +50,239 @@ It is recommended that the MachineConfigServer is run as a DaemonSet on all `mas Response: - **TODO(abhinavdahiya): add example response** +```json +{ + "ignition": { + "config": {}, + "security": { + "tls": {} + }, + "timeouts": {}, + "version": "2.2.0" + }, + "networkd": {}, + "passwd": {}, + "storage": { + "files": [ + { + "filesystem": "root", + "path": "/etc/containers/registries.conf", + "contents": { + "source": "data:,%5Bregistries.search%5D%0Aregistries%20%3D%20%5B'registry.access.redhat.com'%2C%20'docker.io'%5D%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/hosts", + "contents": { + "source": "data:,%23%20IPv4%20and%20IPv6%20localhost%20aliases%0A127.0.0.1%09localhost%0A%3A%3A1%09%09localhost%0A%0A%23%20Internal%20registry%20hack%0A10.3.0.25%20docker-registry.default.svc%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/sysconfig/crio-network", + "contents": { + "source": "data:,CRIO_NETWORK_OPTIONS%3D%22--cni-config-dir%3D%2Fetc%2Fkubernetes%2Fcni%2Fnet.d%20--cni-plugin-dir%3D%2Fvar%2Flib%2Fcni%2Fbin%22%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/kubelet.conf", + "contents": { + "source": "data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AreadOnlyPort%3A%2010255%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/docker/certs.d/docker-registry.default.svc:5000/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIDCTCCAfGgAwIBAgIBADANBgkqhkiG9w0BAQsFADAmMRIwEAYDVQQLEwlvcGVu%0Ac2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2EwHhcNMTgxMDI0MTc0NjE0WhcNMjgxMDIx%0AMTc0NjE0WjAmMRIwEAYDVQQLEwlvcGVuc2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2Ew%0AggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv8EgOZ%2BvexDJkpmEPuIVv%0ACJtvaJ9TEgpD4d0mN1N%2F2g0GWWP1sNM8lxztyA3mhahNkHLAYRScYjURKlaarXgo%0A0%2BnM2rEkkECn4o7TAetHmBd2%2FFgV3peTucVRIWV801QZMmP9vwCa4yPi2L8Ez37k%0A2RpepeeSVIvHARz7%2BHbMHu5cXauPRazSFko05P2y0VgvdhRzX6zm8DjppLQIHqTH%0AkvsIwEXwsQ8GjUnlqnYhDnI%2F1sTG3SVR3%2FbCobiq5N2JH9wKIfIt89KbNPfE7eH1%0AcTcsS1adPMnAVrviEYk9ukebd3pc9gDFUbxhEJLnMo815sy9O%2FyyrPG%2F3Xfjfn4Z%0AAgMBAAGjQjBAMA4GA1UdDwEB%2FwQEAwICpDAPBgNVHRMBAf8EBTADAQH%2FMB0GA1Ud%0ADgQWBBRRKkS2ZLQotJ2ft4o%2B1xf7hrM17DANBgkqhkiG9w0BAQsFAAOCAQEAj72Y%0AHILMf59%2Bcq%2BkHcwizFJk5dj%2FQaN5Bwe0wT1n%2FjneyV2ISzIC5NVbwcnP2DgZWVOT%0ArxA%2BIBuKH%2FXbjzaDpahgtnK1yqObjSAzsdz7DdstdpriqD0YjBQg23d5idrwyEep%0AF7%2FvdTfWjAZkDrszOCr%2BjWsrsCLUDiBf43u1B9RuuqCsl1bFVAHCK7Gj2cMBXJHd%0AjC4%2BOaZY4TUhmSZIi1nyiie79jMKRFiHtM1P%2BERljT4899faGoGbEHDlYn75HvQA%0AM1Yif0VCtzi%2B6xnKDZ5O3wvxctQTtmb9ayL11d1GT%2FOrM9II0UAtodIjpxBo%2BY7n%0Au4k%2BQSXwlOfqDSixwA%3D%3D%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIDCTCCAfGgAwIBAgIBADANBgkqhkiG9w0BAQsFADAmMRIwEAYDVQQLEwlvcGVu%0Ac2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2EwHhcNMTgxMDI0MTc0NjE0WhcNMjgxMDIx%0AMTc0NjE0WjAmMRIwEAYDVQQLEwlvcGVuc2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2Ew%0AggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv8EgOZ%2BvexDJkpmEPuIVv%0ACJtvaJ9TEgpD4d0mN1N%2F2g0GWWP1sNM8lxztyA3mhahNkHLAYRScYjURKlaarXgo%0A0%2BnM2rEkkECn4o7TAetHmBd2%2FFgV3peTucVRIWV801QZMmP9vwCa4yPi2L8Ez37k%0A2RpepeeSVIvHARz7%2BHbMHu5cXauPRazSFko05P2y0VgvdhRzX6zm8DjppLQIHqTH%0AkvsIwEXwsQ8GjUnlqnYhDnI%2F1sTG3SVR3%2FbCobiq5N2JH9wKIfIt89KbNPfE7eH1%0AcTcsS1adPMnAVrviEYk9ukebd3pc9gDFUbxhEJLnMo815sy9O%2FyyrPG%2F3Xfjfn4Z%0AAgMBAAGjQjBAMA4GA1UdDwEB%2FwQEAwICpDAPBgNVHRMBAf8EBTADAQH%2FMB0GA1Ud%0ADgQWBBRRKkS2ZLQotJ2ft4o%2B1xf7hrM17DANBgkqhkiG9w0BAQsFAAOCAQEAj72Y%0AHILMf59%2Bcq%2BkHcwizFJk5dj%2FQaN5Bwe0wT1n%2FjneyV2ISzIC5NVbwcnP2DgZWVOT%0ArxA%2BIBuKH%2FXbjzaDpahgtnK1yqObjSAzsdz7DdstdpriqD0YjBQg23d5idrwyEep%0AF7%2FvdTfWjAZkDrszOCr%2BjWsrsCLUDiBf43u1B9RuuqCsl1bFVAHCK7Gj2cMBXJHd%0AjC4%2BOaZY4TUhmSZIi1nyiie79jMKRFiHtM1P%2BERljT4899faGoGbEHDlYn75HvQA%0AM1Yif0VCtzi%2B6xnKDZ5O3wvxctQTtmb9ayL11d1GT%2FOrM9II0UAtodIjpxBo%2BY7n%0Au4k%2BQSXwlOfqDSixwA%3D%3D%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/machine-config-daemon/node-annotations.json", + "contents": { + "source": "data:,%7B%22machineconfiguration.openshift.io%2FcurrentConfig%22%3A%223aef043ad5aa416e240b6f207c5cd3b0%22%2C%22machineconfiguration.openshift.io%2FdesiredConfig%22%3A%223aef043ad5aa416e240b6f207c5cd3b0%22%7D", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/kubeconfig", + "contents": { + "source": "data:,clusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURDVENDQWZHZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFtTVJJd0VBWURWUVFMRXdsdmNHVnUKYzJocFpuUXhFREFPQmdOVkJBTVRCM0p2YjNRdFkyRXdIaGNOTVRneE1ESTBNVGMwTmpFMFdoY05Namd4TURJeApNVGMwTmpFMFdqQW1NUkl3RUFZRFZRUUxFd2x2Y0dWdWMyaHBablF4RURBT0JnTlZCQU1UQjNKdmIzUXRZMkV3CmdnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUN2OEVnT1ordmV4REprcG1FUHVJVnYKQ0p0dmFKOVRFZ3BENGQwbU4xTi8yZzBHV1dQMXNOTThseHp0eUEzbWhhaE5rSExBWVJTY1lqVVJLbGFhclhnbwowK25NMnJFa2tFQ240bzdUQWV0SG1CZDIvRmdWM3BlVHVjVlJJV1Y4MDFRWk1tUDl2d0NhNHlQaTJMOEV6MzdrCjJScGVwZWVTVkl2SEFSejcrSGJNSHU1Y1hhdVBSYXpTRmtvMDVQMnkwVmd2ZGhSelg2em04RGpwcExRSUhxVEgKa3ZzSXdFWHdzUThHalVubHFuWWhEbkkvMXNURzNTVlIzL2JDb2JpcTVOMkpIOXdLSWZJdDg5S2JOUGZFN2VIMQpjVGNzUzFhZFBNbkFWcnZpRVlrOXVrZWJkM3BjOWdERlVieGhFSkxuTW84MTVzeTlPL3l5clBHLzNYZmpmbjRaCkFnTUJBQUdqUWpCQU1BNEdBMVVkRHdFQi93UUVBd0lDcERBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWQKRGdRV0JCUlJLa1MyWkxRb3RKMmZ0NG8rMXhmN2hyTTE3REFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBajcyWQpISUxNZjU5K2NxK2tIY3dpekZKazVkai9RYU41QndlMHdUMW4vam5leVYySVN6SUM1TlZid2NuUDJEZ1pXVk9UCnJ4QStJQnVLSC9YYmp6YURwYWhndG5LMXlxT2JqU0F6c2R6N0Rkc3RkcHJpcUQwWWpCUWcyM2Q1aWRyd3lFZXAKRjcvdmRUZldqQVprRHJzek9DcitqV3Nyc0NMVURpQmY0M3UxQjlSdXVxQ3NsMWJGVkFIQ0s3R2oyY01CWEpIZApqQzQrT2FaWTRUVWhtU1pJaTFueWlpZTc5ak1LUkZpSHRNMVArRVJsalQ0ODk5ZmFHb0diRUhEbFluNzVIdlFBCk0xWWlmMFZDdHppKzZ4bktEWjVPM3d2eGN0UVR0bWI5YXlMMTFkMUdUL09yTTlJSTBVQXRvZElqcHhCbytZN24KdTRrK1FTWHdsT2ZxRFNpeHdBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo%3D%0A%20%20%20%20server%3A%20https%3A%2F%2Fadahiya-0-api.tt.testing%3A6443%0A%20%20name%3A%20adahiya-0%0Acontexts%3A%0A-%20context%3A%0A%20%20%20%20cluster%3A%20adahiya-0%0A%20%20%20%20user%3A%20kubelet%0A%20%20name%3A%20kubelet%0Acurrent-context%3A%20kubelet%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20kubelet%0A%20%20user%3A%0A%20%20%20%20client-certificate-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsRENDQW55Z0F3SUJBZ0lJUGRsdGdsVUN6SVV3RFFZSktvWklodmNOQVFFTEJRQXdKVEVSTUE4R0ExVUUKQ3hNSVltOXZkR3QxWW1VeEVEQU9CZ05WQkFNVEIydDFZbVV0WTJFd0hoY05NVGd4TURJME1UYzBOakUwV2hjTgpNVGd4TURJME1UZ3hOakUzV2pCNE1VSXdLUVlEVlFRS0V5SnplWE4wWlcwNmMyVnlkbWxqWldGalkyOTFiblJ6Ck9tdDFZbVV0YzNsemRHVnRNQlVHQTFVRUNoTU9jM2x6ZEdWdE9tMWhjM1JsY25NeE1qQXdCZ05WQkFNVEtYTjUKYzNSbGJUcHpaWEoyYVdObFlXTmpiM1Z1ZERwcmRXSmxMWE41YzNSbGJUcGtaV1poZFd4ME1JSUJJakFOQmdrcQpoa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQTAyZ2Yyd2FNNDlkUHlhL2N3aDVVZTlkbUhJd2JJNUlDCm5Ud1FLclJHQ0JSRnNJanQwOVdkRXI4NHBtTmpId0pMM1hZZHYrcy9PVkZHR3Vxekp5R2F2N0w2ZmJ1eHhmY1YKb1hWbm04SUM5bHJnMnU2eXB6eFZTQ0RIRW9CMUNqTlZGY094UlNmMk5TMWR1aTkzamU2UGVKWnBxRFZZWUszZAoycmt0Q2NoR0EzMGE0YzNIWHlKWVdxWjljbTY1eCt0b2hCb2g2ZEllWVVMcEVzbmtqSW5QSUh2eFc1L0hZZWExCkR4SnV2eTJxQlMvWmFoczA4d25MOXg1R1Q5emVVYkxsT0dwZlBqOWJmWHUza3phS2liSDZrR2JyanovdWVSZFEKYVZtdTZFMVp6cVJyOFg5YU1YZHVVTXUyRHY3ZEJGdjU0M29RSXQyYmZHR3Y0TjBWclBtV2FRSURBUUFCbzNVdwpjekFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvCkJBSXdBREFkQmdOVkhRNEVGZ1FVRGxIY2l5cnBHaWYrdU9pcGdvcDhXMTU1UDFrd0h3WURWUjBqQkJnd0ZvQVUKVVNwRXRtUzBLTFNkbjdlS1B0Y1grNGF6TmV3d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIQUhreUNKWHZJVwo3NC8yQ3NTbUFmTWduOHcwZVBOOU5KK0k4RVc1K1djSzNsY2NWNlZ5K0hrMXRDSWpXQ2RMOS85MjNPTlNzWlBoClZpU1UxRC8rY3FJSXcvbkt0U1lKTDB5elJCaWFZeHVmNXUyV0pVeUZTdGxBVnJReVlISE5tQXZ0NUlONHdlakIKMUFoRjl2YnJJM0sweVIzdjBiK1h6SEZiOWZvNm82YW51YmQ2QmZYMCs4bGxPbW5DZ3VvR1pyUUJzOXQ4NVlWYQpSQWNZellkRU9EOElkVlZNVlFSOWZ0NVhldmR0ekVnR0JPVExTaWhuZ0xIYzcwVmpTbmZMdmEzdDBNMmFhLzZGCmszUDNvVDVDRFRvZnVZeTRkVXRVM1V5Ync1MURYamVxdDdUWkRLYW4xeEJnZzEwdm1YUVdhQ3FiTEpqV0lFSngKNHdrTXduQ25TWDA9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K%0A%20%20%20%20client-key-data%3A%20LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBMDJnZjJ3YU00OWRQeWEvY3doNVVlOWRtSEl3Ykk1SUNuVHdRS3JSR0NCUkZzSWp0CjA5V2RFcjg0cG1Oakh3SkwzWFlkditzL09WRkdHdXF6SnlHYXY3TDZmYnV4eGZjVm9YVm5tOElDOWxyZzJ1NnkKcHp4VlNDREhFb0IxQ2pOVkZjT3hSU2YyTlMxZHVpOTNqZTZQZUpacHFEVllZSzNkMnJrdENjaEdBMzBhNGMzSApYeUpZV3FaOWNtNjV4K3RvaEJvaDZkSWVZVUxwRXNua2pJblBJSHZ4VzUvSFllYTFEeEp1dnkycUJTL1phaHMwCjh3bkw5eDVHVDl6ZVViTGxPR3BmUGo5YmZYdTNremFLaWJINmtHYnJqei91ZVJkUWFWbXU2RTFaenFScjhYOWEKTVhkdVVNdTJEdjdkQkZ2NTQzb1FJdDJiZkdHdjROMFZyUG1XYVFJREFRQUJBb0lCQUVqaTFsREtRbHJ2U2RmcwpaUDBjUGQ1d2xnanptUXU3ZEdGSGF2OStKY0wxVWsyWjkvMFg0YzZyMU5rdzNPUzlBdkQ0bnlzaTdTcFN4Z3ZUCnJTNnBuRlBKWGlscFE5SlA3TW84MHhyVldmWWJ3UGhhWVlmYytqNGk1dCtQSUVzREJhdTZTMnpmYVRoT1NzazkKUWtmUjN1OGhWSTRrempLTzN6VmdzSkYxMWdXdlJNeXc4b01UVER0aHhOTXU4NTU3MnQ2bHBlaEtPclFhcWgyKwpqcDlwYmEwNW9yazNCZVhvNk5MazliZVRDUjdibzZSZ2UyeC9Od2dadFlFc0xGb0NndG94VDl5eEYzNDdZTFk1CkpKRlowWFZ3aTJseTJ2b3I3bTJsSXNmVGxlQ3E4bldYUzRER005RHduTktWVVZZbVZmV2R1Ym4zcjlFNlh0VGIKR3Zuckc0RUNnWUVBNFMva1JsNC8xYlJYaGtUcGlKeWNhVHJCZk42cGZlcjZTajI2OUJRVVZ0Zk4rR0dicC94MwowNmJaUFUxK3BYTFBhYjI1aXpyMVAxbTBmaGxRSm55Qk9GdFFuU04zbm94eElCVlE5ZVhjV1FGQ1pqL3VXRzVMCkRtOFl6Mk1BWHE5K0djWXA3MDNLY2ZVNExPODJYWE55SUQrYURncHVMaTFDVFVERUFLS0FGamtDZ1lFQThGV0UKOHNvdm5sSXNITCtaaUJPQ3BYck9rbDlmWW0xRnJ2a0kzTUc5aHZ1V1RnNEorcHZScTFUZjA3U1crcXV0M3VZeQpYOEhvVE1SaE80SUZwZDl0TVdlWWovSnBoQ1FTcU1rcHlnc3hjM28wYzhzNFN2ZzlqOGo1SGFTb1o5UmZ3eWdiCmYzRHJDSDJQcjZleUVPRHErMTZ5cno0NkxiL0RRc1k4T3NUMEFiRUNnWUFndW1vdUJBSzVGNVhrOE4wVU90Yk0Kd0hwZ29LZjNvaEF3ZkJwUTRSNDNwUFBObHJvZHh5YlBQeCt4dGpLaTd6WFFBNEFWQ1VPZHFuYitJTVd5WWtRUgpvY3ZzbXJ3RzhoaDY5ajRuRHZwZ2dUdGFTdzVrRWR1Y3hHN1JyV3pmVmhnNHZNRlpnMi9aOGk3dzhPOXcwNWVSCnNreThuNjExenFRbFFEVjhkaUd4bVFLQmdRRGJ3dWR4OXkyNTBKdmpvZFByV1NQUzIwdi9EbFN6TlFaT0xBeE4KaUo4Y3lmc3ozcVNEVTI1VEE2WXorT05CempDTUxPU05LVXVZdnMzR1UydUV0Snd0Vy9SbVZCem1KdklsQXVWQwppaCtxMzJrTkpSdVJlaE1ZNG9YZzlFckZ2cTNlVDFOdG9qeFlwQy82U0JhTVZvNm9VbnlEd0J3RTcxL0dOR3lvCnRLWUcwUUtCZ0ZwYTY5anBmaW1wSTJJVTFTSmlqZkwxNFVWL09vRXBrUjFNMHU5cVFLZnNsM00xdzBmZ2huRVcKaEFYUUQzSTMxWnBkWkdYT0R6VzVJdUV5aExHaGJIWUgzejZldENJanFpNzdxWTlXbkJ3QjJwQ2Q3YzFuNVRCdwpNcElxV1BDbVQvSEpRUDAyYkhSZGlHSm55aURqZ3FpWnNYVmlrYk1yWjJCNjJqRHkvVVpmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg%3D%3D%0A", + "verification": {} + }, + "mode": 420 + } + ] + }, + "systemd": { + "units": [ + { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nWants=rpc-statd.service\n\n[Service]\nExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests\nEnvironmentFile=-/etc/kubernetes/kubelet-workaround\nEnvironmentFile=-/etc/kubernetes/kubelet-env\n\nExecStart=/usr/bin/hyperkube \\\n kubelet \\\n --config=/etc/kubernetes/kubelet.conf \\\n --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --container-runtime=remote \\\n --container-runtime-endpoint=/var/run/crio/crio.sock \\\n --allow-privileged \\\n --node-labels=node-role.kubernetes.io/worker \\\n --minimum-container-ttl-duration=6m0s \\\n --client-ca-file=/etc/kubernetes/ca.crt \\\n --cloud-provider= \\\n \\\n --anonymous-auth=false \\\n\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "kubelet.service" + } + ] + } +} +``` 2. Master machine with etcd member `etcd-1` Request: - GET `/config/master?etcd_index=1` + GET `/config/master` Response: ```json - { - "ignition": { - "config": {}, - "security": { - "tls": {} - }, - "timeouts": {} - }, - "networkd": {}, - "passwd": {}, - "storage": { - "files": [{ - "contents": { - "verification": {} - } - }, { - "filesystem": "root", - "path": "/etc/machine-config-daemon/node-annotations.json", - "contents": { - "source": "data:,%7B%22machineconfiguration.openshift.io%2FcurrentConfig%22%3A%22test-config%22%2C%22machineconfiguration.openshift.io%2FdesiredConfig%22%3A%22test-config%22%7D", - "verification": {} - }, - "mode": 420 - }, { - "filesystem": "root", - "path": "/etc/system/kubeconfig", - "contents": { - "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURkVENDQWwyZ0F3SUJBZ0lSQUs1aGNubDBBMGhrcnArU2tHdTg2ak13RFFZSktvWklodmNOQVFFTEJRQXcKVkRFdE1Dc0dBMVVFQ2hNa09UZzJaalZsWldJdE56WXpaUzB5WVRSbUxURTRaRFV0TTJWa01HRmxaRGszWkdRegpNUkV3RHdZRFZRUUxFd2gwWldOMGIyNXBZekVRTUE0R0ExVUVBeE1IEs5SUxiWGgvRGpzQUpFcWJCVnNIMTQ4SStwOWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg%3D%3D%0A%20%20%20%20server%3A%20https%3A%2F%2Ftest-system%3A443%0A%20%20name%3A%20mcs%0Acontexts%3A%0A-%20context%3A%0A%20%20%20%20cluster%3A%20mcs%0A%20%20%20%20user%3A%20admin%0A%20%20name%3A%20%22%22%0A-%20context%3A%0A%20%20%20%20cluster%3A%20mcs%0A%20%20%20%20namespace%3A%20kube-system%0A%20%20%20%20user%3A%20admin%0A%20%20name%3A%20kube-system%0Acurrent-context%3A%20kube-system%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20admin%0A%20%20user%3A%0A%20%20%20%20client-certificate-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhRENDQWxDZ0F3SUJBZ0lSQVA2RnYxK2RlSE9JNFRUTm5RNk5sUFl3RFFZSktvWklodmNOQVFFTEJRQXcKVkRFdE1Dc0dBMVVFQ2hNa05qRTBZemM1TldFdE9UTXpaQzB3WkRjM0xURTVOell0Tm1ReVpUazFNRFpoWTJSawpNUkV3RHdZRFZRUUxFd2hpYjI5MGEzVmlaVEVRTUE0R0ExVUVBeE1IYTNWaVpZUtpRmM4bzRMVVh1V1FhMUtleW5VCnpwWGROOUxpK0JJYnFDb0ZoZVpiTk5YM3hUSm5LNWFTOVVtWTliYXlERmhWOUl1RHVpWTBPWHB0TWxHQ243ZkIKaGh0YjFsUTcwaHV4Wno3OEJnSjZMU2UrcEt2eFBuaVpQZ1hFNlhCdmdkSmRjZU5ZZmJlUTFrbHoxRG1JSmIxYgpGSExNZVBxY2txMEkyaFJZYXJ5UDBlbVJjbUJqSDZpaWl1UnRBeTFUY1ZnTTBTZzhZbjFFSExCbnhiUm41TmpsCnZSU3c1MnRSUUtBbE5WeDEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo%3D%0A%20%20%20%20client-key-data%3A%20RWGdnT0huTmZYZDlxUnQ5TmdyVE5rMGh2Tzk0eApRYkxJUHNjM0dDTU1FQm8zZnlUait3ZkJTeFBydTg0Q3BRYjdxS3A2Q3FqcDNGSDBuT2l2ejNIU3JrRkF2cU5WClVpd001Q3ZFQWt0dTI5NU0zUE42YlNrQ2dZQTNLSi9pYUFqVW1IK3V2SjFXUFpMTnBtaDdEdmdxcmd6aEFoSHEKZ2V1NkpTNXpRYktyNkYvYmVmOVgxd1BLdmcxbDZ3WmVMc3cvZkNHMUY0c3U0NlVVY3ZDVlVRMm94eE40anc3SwozUWhiaWpxS3N5dmhPYzBtQ1FlemgzblRsOUtaVjJlOTllWUVvc0NIRU1ZekpMN3BiWW02SFpjajI5QjRsY21tCjM1bGhLd0tCZ1FDYitUQ2x6ZVFhakdWSWI0UXBtcjJ3R2xFamEzUytFTFNwUEVJZmNxUUdvWjcrMWdxMHRhOWsKT2Zvb0p2Y1NpVWZoSnJTbHJUSk1NTmFDT0ZUR0tZNVlKcGMrMzlndXY1YjI0aHVGZmdGeWJNT09aek1VekVWSwo2UHM1Z2hBQjh3c05oZTZHOHRBbytIZUduVU9DdE94YmlrMWp6RjkvQmJFaXE3Lzc2SDMyd0E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo%3D%0A", - "verification": {} - }, - "mode": 420 - }] - }, - "systemd": { - "units": [{ - "dropins": [{ - "contents": "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\n", - "name": "10-dockeropts.conf" - }], - "enabled": true, - "name": "docker.service" - }, { - "contents": "[Unit]\nDescription=etcd (System Application Container) TLS assets\nConditionFileNotEmpty=|!/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems.crt\nConditionFileNotEmpty=|!/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems.key\nConditionFileNotEmpty=|!/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems.crt\nConditionFileNotEmpty=|!/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems.key\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\n\nEnvironment=\"SIGNER_IMAGE=quay.io/coreos/kube-client-agent:678cc8e6841e2121ebfdb6e2db568fce290b67d6\"\n\nExecStart=/usr/bin/docker \\\n run \\\n --rm \\\n --volume /etc/ssl/etcd:/etc/ssl/etcd:rw \\\n \"${SIGNER_IMAGE}\" \\\n request \\\n --orgname=system:etcd-servers \\\n --cacrt=/etc/ssl/etcd/root-ca.crt \\\n --assetsdir=/etc/ssl/etcd \\\n --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \\\n --dnsnames=localhost,*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \\\n --commonname=system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems \\\n --ipaddrs=127.0.0.1 \\\n\nExecStart=/usr/bin/docker \\\n run \\\n --rm \\\n --volume /etc/ssl/etcd:/etc/ssl/etcd:rw \\\n \"${SIGNER_IMAGE}\" \\\n request \\\n --orgname=system:etcd-peers \\\n --cacrt=/etc/ssl/etcd/root-ca.crt \\\n --assetsdir=/etc/ssl/etcd \\\n --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \\\n --dnsnames=*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \\\n --commonname=system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems \\\n\nExecStart=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems.crt\nExecStart=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems.key\nExecStart=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems.crt\nExecStart=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems.key\n", - "enabled": true, - "name": "etcd-member-tls.service" - }, { - "contents": "[Unit]\nDescription=etcd (System Application Container)\nDocumentation=https://github.com/coreos/etcd\nRequires=etcd-member-tls.service\nAfter=etcd-member-tls.service\n\n[Service]\nRestart=on-failure\nRestartSec=10s\nTimeoutStartSec=0\nLimitNOFILE=40000\n\nEnvironment=\"ETCD_IMAGE=quay.io/coreos/etcd:v3.2.14\"\n\nExecStartPre=-/usr/bin/docker rm etcd-member\nExecStartPre=/usr/bin/mkdir --parents /var/lib/etcd\nExecStartPre=/usr/bin/mkdir --parents /run/etcd\nExecStartPre=/usr/bin/chown etcd /var/lib/etcd\nExecStartPre=/usr/bin/chown etcd /run/etcd\n\nExecStart= /usr/bin/bash -c \" \\\n /usr/bin/docker \\\n run \\\n --rm \\\n --name etcd-member \\\n --volume /run/systemd/system:/run/systemd/system:ro \\\n --volume /etc/ssl/certs:/etc/ssl/certs:ro \\\n --volume /etc/ssl/etcd:/etc/ssl/etcd:ro \\\n --volume /var/lib/etcd:/var/lib/etcd:rw \\\n --volume /etc/ssl/certs:/etc/ssl/certs:ro \\\n --env 'ETCD_NAME=%m' \\\n --env ETCD_DATA_DIR=/var/lib/etcd \\\n --network host \\\n --user=$(id --user etcd) \\\n '${ETCD_IMAGE}' \\\n /usr/local/bin/etcd \\\n --name=my-test-cluster-etcd-1.installer.team.coreos.systems \\\n --advertise-client-urls=https://my-test-cluster-etcd-1.installer.team.coreos.systems:2379 \\\n --cert-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems.crt \\\n --key-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-1.installer.team.coreos.systems.key \\\n --trusted-ca-file=/etc/ssl/etcd/ca.crt \\\n --client-cert-auth=true \\\n --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems.crt \\\n --peer-key-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-1.installer.team.coreos.systems.key \\\n --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \\\n --peer-client-cert-auth=true \\\n --initial-cluster='my-test-cluster-etcd-0.installer.team.coreos.systems=https://my-test-cluster-etcd-0.installer.team.coreos.systems:2380,my-test-cluster-etcd-1.installer.team.coreos.systems=https://my-test-cluster-etcd-1.installer.team.coreos.systems:2380,my-test-cluster-etcd-2.installer.team.coreos.systems=https://my-test-cluster-etcd-2.installer.team.coreos.systems:2380' \\\n --initial-advertise-peer-urls=https://my-test-cluster-etcd-1.installer.team.coreos.systems:2380 \\\n --listen-client-urls=https://0.0.0.0:2379 \\\n --listen-peer-urls=https://0.0.0.0:2380 \\\n \"\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "etcd-member.service" - }, { - "contents": "[Service]\nExecStart=/usr/bin/env bash -c \\\n \" \\\n if grep rhcos /etc/os-release \u003e /dev/null; \\\n then \\\n echo CGROUP_DRIVER_FLAG=--cgroup-driver=systemd \u003e /etc/kubernetes/kubelet-workaround; \\\n mount -o remount,rw /sys/fs/cgroup; \\\n ln --symbolic /sys/fs/cgroup/cpu,cpuacct /sys/fs/cgroup/cpuacct,cpu; \\\n fi \\\n \"\n", - "name": "kubelet-workaround.service" - }, { - "contents": "[Unit]\nDescription=Kubernetes Kubelet\nWants=rpc-statd.service\nRequires=docker.service kubelet-workaround.service\nAfter=docker.service kubelet-workaround.service\n\n[Service]\nEnvironment=\"KUBELET_IMAGE=openshift/origin-node\"\nEnvironmentFile=-/etc/kubernetes/kubelet-workaround\n\nExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir --parents /etc/kubernetes/checkpoint-secrets\nExecStartPre=/bin/mkdir --parents /etc/kubernetes/cni/net.d\nExecStartPre=/bin/mkdir --parents /run/kubelet\nExecStartPre=/bin/mkdir --parents /var/lib/cni\nExecStartPre=/bin/mkdir --parents /var/lib/kubelet/pki\n\nExecStartPre=/usr/bin/bash -c \"gawk '/certificate-authority-data/ {print $2}' /etc/kubernetes/kubeconfig | base64 --decode \u003e /etc/kubernetes/ca.crt\"\n\nExecStart=/usr/bin/docker \\\n run \\\n --rm \\\n --net host \\\n --pid host \\\n --privileged \\\n --volume /dev:/dev:rw \\\n --volume /sys:/sys:ro \\\n --volume /var/run:/var/run:rw \\\n --volume /var/lib/cni/:/var/lib/cni:rw \\\n --volume /var/lib/docker/:/var/lib/docker:rw \\\n --volume /var/lib/kubelet/:/var/lib/kubelet:shared \\\n --volume /var/log:/var/log:shared \\\n --volume /etc/kubernetes:/etc/kubernetes:ro \\\n --entrypoint /usr/bin/hyperkube \\\n \"${KUBELET_IMAGE}\" \\\n kubelet \\\n --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --rotate-certificates \\\n --cni-conf-dir=/etc/kubernetes/cni/net.d \\\n --cni-bin-dir=/var/lib/cni/bin \\\n --network-plugin=cni \\\n --lock-file=/var/run/lock/kubelet.lock \\\n --exit-on-lock-contention \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --allow-privileged \\\n --node-labels=node-role.kubernetes.io/etcd \\\n --minimum-container-ttl-duration=6m0s \\\n --cluster-dns=10.3.0.10 \\\n --cluster-domain=cluster.local \\\n --client-ca-file=/etc/kubernetes/ca.crt \\\n --cloud-provider=aws \\\n \\\n --anonymous-auth=false \\\n --register-with-taints=node-role.kubernetes.io/etcd=:NoSchedule \\\n $CGROUP_DRIVER_FLAG \\\n\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "kubelet.service" - }, { - "mask": true, - "name": "locksmith.service" - }] - } -} +{ + "ignition": { + "config": {}, + "security": { + "tls": {} + }, + "timeouts": {}, + "version": "2.2.0" + }, + "networkd": {}, + "passwd": {}, + "storage": { + "files": [{ + "filesystem": "root", + "path": "/etc/containers/registries.conf", + "contents": { + "source": "data:,%5Bregistries.search%5D%0Aregistries%20%3D%20%5B'registry.access.redhat.com'%2C%20'docker.io'%5D%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/hosts", + "contents": { + "source": "data:,%23%20IPv4%20and%20IPv6%20localhost%20aliases%0A127.0.0.1%09localhost%0A%3A%3A1%09%09localhost%0A%0A%23%20Internal%20registry%20hack%0A10.3.0.25%20docker-registry.default.svc%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/sysconfig/crio-network", + "contents": { + "source": "data:,CRIO_NETWORK_OPTIONS%3D%22--cni-config-dir%3D%2Fetc%2Fkubernetes%2Fcni%2Fnet.d%20--cni-plugin-dir%3D%2Fvar%2Flib%2Fcni%2Fbin%22%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/ssl/etcd/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIDKTCCAhGgAwIBAgIIGrsm9RgrvtQwDQYJKoZIhvcNAQELBQAwJjESMBAGA1UE%0ACxMJb3BlbnNoaWZ0MRAwDgYDVQQDEwdyb290LWNhMB4XDTE4MTAyNDE3NDYxNFoX%0ADTI4MTAyMTE3NDYxNFowHjENMAsGA1UECxMEZXRjZDENMAsGA1UEAxMEZXRjZDCC%0AASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALiSeGleHsZVeHZTlgc6qcNw%0AmlceIU5RK6TA3zHp%2ByaopILTn3oVyDozfQqF07H7RZ%2BMAaPFiOfhx6hh27SlvTIe%0AggT3vtVdPtjIsrJppEafdjc37ZDIAV%2BhAqXIAymOxG0IVHrozvbBjNBhr4DqgLpg%0AJyzPRXFdmGHo7Yi9eiR6CTv04cHUYJy7KbbIwT6AFsrW8daO8CmN4yX9pkGsDcvy%0Aordi5ZDgjpkPwhAlqQ7pn52WdELBaCY7Jv1h03inpuYQQpbVnIFxDylR%2FWeuDYz3%0A8%2BacfQ9ZAlVVMfUpqNgYvXvgq%2FEfY20QYYn76%2BZ7wQmNBXevvtU%2FKjUv2UCgoKEC%0AAwEAAaNjMGEwDgYDVR0PAQH%2FBAQDAgKkMA8GA1UdEwEB%2FwQFMAMBAf8wHQYDVR0O%0ABBYEFFEqRLZktCi0nZ%2B3ij7XF%2FuGszXsMB8GA1UdIwQYMBaAFFEqRLZktCi0nZ%2B3%0Aij7XF%2FuGszXsMA0GCSqGSIb3DQEBCwUAA4IBAQArUpdqaJY50u%2BOi39h1vSaliwY%0ABOkQ8xQno2Kkpxoet4FAO7vA2Zav8SEdt4bZdkydwEumNiqpMVrlz%2BpxTn%2BXgpCW%0AchwY2mZ1hlgiElXARPE%2FbJQesYMlogZP%2Bg%2FUcwJj8HJd%2F6d6j9Hsu8amhABjdk3G%0AHCH1h4vZKSz9opVpB1EzI1Y0Ls%2BTLBotpJJSHdRJZnWDNm%2Fjcs8ZnekcPB6RHwxK%0AeoHRt0ChmmdTbLg0FNZXpt4q%2F8zvRdxXmKW98MPhfrqcdCHb4ISjOECb2Mg2B5D6%0AK%2FDK22i8qZ5PgroB71sm4RQd1pup3yF02iZqfuuUzo1kJYawESfn816oounj%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/ssl/etcd/root-ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIDCTCCAfGgAwIBAgIBADANBgkqhkiG9w0BAQsFADAmMRIwEAYDVQQLEwlvcGVu%0Ac2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2EwHhcNMTgxMDI0MTc0NjE0WhcNMjgxMDIx%0AMTc0NjE0WjAmMRIwEAYDVQQLEwlvcGVuc2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2Ew%0AggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv8EgOZ%2BvexDJkpmEPuIVv%0ACJtvaJ9TEgpD4d0mN1N%2F2g0GWWP1sNM8lxztyA3mhahNkHLAYRScYjURKlaarXgo%0A0%2BnM2rEkkECn4o7TAetHmBd2%2FFgV3peTucVRIWV801QZMmP9vwCa4yPi2L8Ez37k%0A2RpepeeSVIvHARz7%2BHbMHu5cXauPRazSFko05P2y0VgvdhRzX6zm8DjppLQIHqTH%0AkvsIwEXwsQ8GjUnlqnYhDnI%2F1sTG3SVR3%2FbCobiq5N2JH9wKIfIt89KbNPfE7eH1%0AcTcsS1adPMnAVrviEYk9ukebd3pc9gDFUbxhEJLnMo815sy9O%2FyyrPG%2F3Xfjfn4Z%0AAgMBAAGjQjBAMA4GA1UdDwEB%2FwQEAwICpDAPBgNVHRMBAf8EBTADAQH%2FMB0GA1Ud%0ADgQWBBRRKkS2ZLQotJ2ft4o%2B1xf7hrM17DANBgkqhkiG9w0BAQsFAAOCAQEAj72Y%0AHILMf59%2Bcq%2BkHcwizFJk5dj%2FQaN5Bwe0wT1n%2FjneyV2ISzIC5NVbwcnP2DgZWVOT%0ArxA%2BIBuKH%2FXbjzaDpahgtnK1yqObjSAzsdz7DdstdpriqD0YjBQg23d5idrwyEep%0AF7%2FvdTfWjAZkDrszOCr%2BjWsrsCLUDiBf43u1B9RuuqCsl1bFVAHCK7Gj2cMBXJHd%0AjC4%2BOaZY4TUhmSZIi1nyiie79jMKRFiHtM1P%2BERljT4899faGoGbEHDlYn75HvQA%0AM1Yif0VCtzi%2B6xnKDZ5O3wvxctQTtmb9ayL11d1GT%2FOrM9II0UAtodIjpxBo%2BY7n%0Au4k%2BQSXwlOfqDSixwA%3D%3D%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/kubernetes/kubelet.conf", + "contents": { + "source": "data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AreadOnlyPort%3A%2010255%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/modules-load.d/bridge.conf", + "contents": { + "source": "data:,br_netfilter%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/docker/certs.d/docker-registry.default.svc:5000/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIDCTCCAfGgAwIBAgIBADANBgkqhkiG9w0BAQsFADAmMRIwEAYDVQQLEwlvcGVu%0Ac2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2EwHhcNMTgxMDI0MTc0NjE0WhcNMjgxMDIx%0AMTc0NjE0WjAmMRIwEAYDVQQLEwlvcGVuc2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2Ew%0AggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv8EgOZ%2BvexDJkpmEPuIVv%0ACJtvaJ9TEgpD4d0mN1N%2F2g0GWWP1sNM8lxztyA3mhahNkHLAYRScYjURKlaarXgo%0A0%2BnM2rEkkECn4o7TAetHmBd2%2FFgV3peTucVRIWV801QZMmP9vwCa4yPi2L8Ez37k%0A2RpepeeSVIvHARz7%2BHbMHu5cXauPRazSFko05P2y0VgvdhRzX6zm8DjppLQIHqTH%0AkvsIwEXwsQ8GjUnlqnYhDnI%2F1sTG3SVR3%2FbCobiq5N2JH9wKIfIt89KbNPfE7eH1%0AcTcsS1adPMnAVrviEYk9ukebd3pc9gDFUbxhEJLnMo815sy9O%2FyyrPG%2F3Xfjfn4Z%0AAgMBAAGjQjBAMA4GA1UdDwEB%2FwQEAwICpDAPBgNVHRMBAf8EBTADAQH%2FMB0GA1Ud%0ADgQWBBRRKkS2ZLQotJ2ft4o%2B1xf7hrM17DANBgkqhkiG9w0BAQsFAAOCAQEAj72Y%0AHILMf59%2Bcq%2BkHcwizFJk5dj%2FQaN5Bwe0wT1n%2FjneyV2ISzIC5NVbwcnP2DgZWVOT%0ArxA%2BIBuKH%2FXbjzaDpahgtnK1yqObjSAzsdz7DdstdpriqD0YjBQg23d5idrwyEep%0AF7%2FvdTfWjAZkDrszOCr%2BjWsrsCLUDiBf43u1B9RuuqCsl1bFVAHCK7Gj2cMBXJHd%0AjC4%2BOaZY4TUhmSZIi1nyiie79jMKRFiHtM1P%2BERljT4899faGoGbEHDlYn75HvQA%0AM1Yif0VCtzi%2B6xnKDZ5O3wvxctQTtmb9ayL11d1GT%2FOrM9II0UAtodIjpxBo%2BY7n%0Au4k%2BQSXwlOfqDSixwA%3D%3D%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/kubernetes/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIDCTCCAfGgAwIBAgIBADANBgkqhkiG9w0BAQsFADAmMRIwEAYDVQQLEwlvcGVu%0Ac2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2EwHhcNMTgxMDI0MTc0NjE0WhcNMjgxMDIx%0AMTc0NjE0WjAmMRIwEAYDVQQLEwlvcGVuc2hpZnQxEDAOBgNVBAMTB3Jvb3QtY2Ew%0AggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv8EgOZ%2BvexDJkpmEPuIVv%0ACJtvaJ9TEgpD4d0mN1N%2F2g0GWWP1sNM8lxztyA3mhahNkHLAYRScYjURKlaarXgo%0A0%2BnM2rEkkECn4o7TAetHmBd2%2FFgV3peTucVRIWV801QZMmP9vwCa4yPi2L8Ez37k%0A2RpepeeSVIvHARz7%2BHbMHu5cXauPRazSFko05P2y0VgvdhRzX6zm8DjppLQIHqTH%0AkvsIwEXwsQ8GjUnlqnYhDnI%2F1sTG3SVR3%2FbCobiq5N2JH9wKIfIt89KbNPfE7eH1%0AcTcsS1adPMnAVrviEYk9ukebd3pc9gDFUbxhEJLnMo815sy9O%2FyyrPG%2F3Xfjfn4Z%0AAgMBAAGjQjBAMA4GA1UdDwEB%2FwQEAwICpDAPBgNVHRMBAf8EBTADAQH%2FMB0GA1Ud%0ADgQWBBRRKkS2ZLQotJ2ft4o%2B1xf7hrM17DANBgkqhkiG9w0BAQsFAAOCAQEAj72Y%0AHILMf59%2Bcq%2BkHcwizFJk5dj%2FQaN5Bwe0wT1n%2FjneyV2ISzIC5NVbwcnP2DgZWVOT%0ArxA%2BIBuKH%2FXbjzaDpahgtnK1yqObjSAzsdz7DdstdpriqD0YjBQg23d5idrwyEep%0AF7%2FvdTfWjAZkDrszOCr%2BjWsrsCLUDiBf43u1B9RuuqCsl1bFVAHCK7Gj2cMBXJHd%0AjC4%2BOaZY4TUhmSZIi1nyiie79jMKRFiHtM1P%2BERljT4899faGoGbEHDlYn75HvQA%0AM1Yif0VCtzi%2B6xnKDZ5O3wvxctQTtmb9ayL11d1GT%2FOrM9II0UAtodIjpxBo%2BY7n%0Au4k%2BQSXwlOfqDSixwA%3D%3D%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/sysctl.d/bridge.conf", + "contents": { + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Anet.bridge.bridge-nf-call-arptables%20%3D%201%0A", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/machine-config-daemon/node-annotations.json", + "contents": { + "source": "data:,%7B%22machineconfiguration.openshift.io%2FcurrentConfig%22%3A%22be2ec8753b61b4ffcb0e1aca92d7936a%22%2C%22machineconfiguration.openshift.io%2FdesiredConfig%22%3A%22be2ec8753b61b4ffcb0e1aca92d7936a%22%7D", + "verification": {} + }, + "mode": 420 + }, { + "filesystem": "root", + "path": "/etc/kubernetes/kubeconfig", + "contents": { + "source": "data:,clusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURDVENDQWZHZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFtTVJJd0VBWURWUVFMRXdsdmNHVnUKYzJocFpuUXhFREFPQmdOVkJBTVRCM0p2YjNRdFkyRXdIaGNOTVRneE1ESTBNVGMwTmpFMFdoY05Namd4TURJeApNVGMwTmpFMFdqQW1NUkl3RUFZRFZRUUxFd2x2Y0dWdWMyaHBablF4RURBT0JnTlZCQU1UQjNKdmIzUXRZMkV3CmdnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUN2OEVnT1ordmV4REprcG1FUHVJVnYKQ0p0dmFKOVRFZ3BENGQwbU4xTi8yZzBHV1dQMXNOTThseHp0eUEzbWhhaE5rSExBWVJTY1lqVVJLbGFhclhnbwowK25NMnJFa2tFQ240bzdUQWV0SG1CZDIvRmdWM3BlVHVjVlJJV1Y4MDFRWk1tUDl2d0NhNHlQaTJMOEV6MzdrCjJScGVwZWVTVkl2SEFSejcrSGJNSHU1Y1hhdVBSYXpTRmtvMDVQMnkwVmd2ZGhSelg2em04RGpwcExRSUhxVEgKa3ZzSXdFWHdzUThHalVubHFuWWhEbkkvMXNURzNTVlIzL2JDb2JpcTVOMkpIOXdLSWZJdDg5S2JOUGZFN2VIMQpjVGNzUzFhZFBNbkFWcnZpRVlrOXVrZWJkM3BjOWdERlVieGhFSkxuTW84MTVzeTlPL3l5clBHLzNYZmpmbjRaCkFnTUJBQUdqUWpCQU1BNEdBMVVkRHdFQi93UUVBd0lDcERBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWQKRGdRV0JCUlJLa1MyWkxRb3RKMmZ0NG8rMXhmN2hyTTE3REFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBajcyWQpISUxNZjU5K2NxK2tIY3dpekZKazVkai9RYU41QndlMHdUMW4vam5leVYySVN6SUM1TlZid2NuUDJEZ1pXVk9UCnJ4QStJQnVLSC9YYmp6YURwYWhndG5LMXlxT2JqU0F6c2R6N0Rkc3RkcHJpcUQwWWpCUWcyM2Q1aWRyd3lFZXAKRjcvdmRUZldqQVprRHJzek9DcitqV3Nyc0NMVURpQmY0M3UxQjlSdXVxQ3NsMWJGVkFIQ0s3R2oyY01CWEpIZApqQzQrT2FaWTRUVWhtU1pJaTFueWlpZTc5ak1LUkZpSHRNMVArRVJsalQ0ODk5ZmFHb0diRUhEbFluNzVIdlFBCk0xWWlmMFZDdHppKzZ4bktEWjVPM3d2eGN0UVR0bWI5YXlMMTFkMUdUL09yTTlJSTBVQXRvZElqcHhCbytZN24KdTRrK1FTWHdsT2ZxRFNpeHdBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo%3D%0A%20%20%20%20server%3A%20https%3A%2F%2Fadahiya-0-api.tt.testing%3A6443%0A%20%20name%3A%20adahiya-0%0Acontexts%3A%0A-%20context%3A%0A%20%20%20%20cluster%3A%20adahiya-0%0A%20%20%20%20user%3A%20kubelet%0A%20%20name%3A%20kubelet%0Acurrent-context%3A%20kubelet%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20kubelet%0A%20%20user%3A%0A%20%20%20%20client-certificate-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsRENDQW55Z0F3SUJBZ0lJUGRsdGdsVUN6SVV3RFFZSktvWklodmNOQVFFTEJRQXdKVEVSTUE4R0ExVUUKQ3hNSVltOXZkR3QxWW1VeEVEQU9CZ05WQkFNVEIydDFZbVV0WTJFd0hoY05NVGd4TURJME1UYzBOakUwV2hjTgpNVGd4TURJME1UZ3hOakUzV2pCNE1VSXdLUVlEVlFRS0V5SnplWE4wWlcwNmMyVnlkbWxqWldGalkyOTFiblJ6Ck9tdDFZbVV0YzNsemRHVnRNQlVHQTFVRUNoTU9jM2x6ZEdWdE9tMWhjM1JsY25NeE1qQXdCZ05WQkFNVEtYTjUKYzNSbGJUcHpaWEoyYVdObFlXTmpiM1Z1ZERwcmRXSmxMWE41YzNSbGJUcGtaV1poZFd4ME1JSUJJakFOQmdrcQpoa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQTAyZ2Yyd2FNNDlkUHlhL2N3aDVVZTlkbUhJd2JJNUlDCm5Ud1FLclJHQ0JSRnNJanQwOVdkRXI4NHBtTmpId0pMM1hZZHYrcy9PVkZHR3Vxekp5R2F2N0w2ZmJ1eHhmY1YKb1hWbm04SUM5bHJnMnU2eXB6eFZTQ0RIRW9CMUNqTlZGY094UlNmMk5TMWR1aTkzamU2UGVKWnBxRFZZWUszZAoycmt0Q2NoR0EzMGE0YzNIWHlKWVdxWjljbTY1eCt0b2hCb2g2ZEllWVVMcEVzbmtqSW5QSUh2eFc1L0hZZWExCkR4SnV2eTJxQlMvWmFoczA4d25MOXg1R1Q5emVVYkxsT0dwZlBqOWJmWHUza3phS2liSDZrR2JyanovdWVSZFEKYVZtdTZFMVp6cVJyOFg5YU1YZHVVTXUyRHY3ZEJGdjU0M29RSXQyYmZHR3Y0TjBWclBtV2FRSURBUUFCbzNVdwpjekFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvCkJBSXdBREFkQmdOVkhRNEVGZ1FVRGxIY2l5cnBHaWYrdU9pcGdvcDhXMTU1UDFrd0h3WURWUjBqQkJnd0ZvQVUKVVNwRXRtUzBLTFNkbjdlS1B0Y1grNGF6TmV3d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIQUhreUNKWHZJVwo3NC8yQ3NTbUFmTWduOHcwZVBOOU5KK0k4RVc1K1djSzNsY2NWNlZ5K0hrMXRDSWpXQ2RMOS85MjNPTlNzWlBoClZpU1UxRC8rY3FJSXcvbkt0U1lKTDB5elJCaWFZeHVmNXUyV0pVeUZTdGxBVnJReVlISE5tQXZ0NUlONHdlakIKMUFoRjl2YnJJM0sweVIzdjBiK1h6SEZiOWZvNm82YW51YmQ2QmZYMCs4bGxPbW5DZ3VvR1pyUUJzOXQ4NVlWYQpSQWNZellkRU9EOElkVlZNVlFSOWZ0NVhldmR0ekVnR0JPVExTaWhuZ0xIYzcwVmpTbmZMdmEzdDBNMmFhLzZGCmszUDNvVDVDRFRvZnVZeTRkVXRVM1V5Ync1MURYamVxdDdUWkRLYW4xeEJnZzEwdm1YUVdhQ3FiTEpqV0lFSngKNHdrTXduQ25TWDA9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K%0A%20%20%20%20client-key-data%3A%20LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBMDJnZjJ3YU00OWRQeWEvY3doNVVlOWRtSEl3Ykk1SUNuVHdRS3JSR0NCUkZzSWp0CjA5V2RFcjg0cG1Oakh3SkwzWFlkditzL09WRkdHdXF6SnlHYXY3TDZmYnV4eGZjVm9YVm5tOElDOWxyZzJ1NnkKcHp4VlNDREhFb0IxQ2pOVkZjT3hSU2YyTlMxZHVpOTNqZTZQZUpacHFEVllZSzNkMnJrdENjaEdBMzBhNGMzSApYeUpZV3FaOWNtNjV4K3RvaEJvaDZkSWVZVUxwRXNua2pJblBJSHZ4VzUvSFllYTFEeEp1dnkycUJTL1phaHMwCjh3bkw5eDVHVDl6ZVViTGxPR3BmUGo5YmZYdTNremFLaWJINmtHYnJqei91ZVJkUWFWbXU2RTFaenFScjhYOWEKTVhkdVVNdTJEdjdkQkZ2NTQzb1FJdDJiZkdHdjROMFZyUG1XYVFJREFRQUJBb0lCQUVqaTFsREtRbHJ2U2RmcwpaUDBjUGQ1d2xnanptUXU3ZEdGSGF2OStKY0wxVWsyWjkvMFg0YzZyMU5rdzNPUzlBdkQ0bnlzaTdTcFN4Z3ZUCnJTNnBuRlBKWGlscFE5SlA3TW84MHhyVldmWWJ3UGhhWVlmYytqNGk1dCtQSUVzREJhdTZTMnpmYVRoT1NzazkKUWtmUjN1OGhWSTRrempLTzN6VmdzSkYxMWdXdlJNeXc4b01UVER0aHhOTXU4NTU3MnQ2bHBlaEtPclFhcWgyKwpqcDlwYmEwNW9yazNCZVhvNk5MazliZVRDUjdibzZSZ2UyeC9Od2dadFlFc0xGb0NndG94VDl5eEYzNDdZTFk1CkpKRlowWFZ3aTJseTJ2b3I3bTJsSXNmVGxlQ3E4bldYUzRER005RHduTktWVVZZbVZmV2R1Ym4zcjlFNlh0VGIKR3Zuckc0RUNnWUVBNFMva1JsNC8xYlJYaGtUcGlKeWNhVHJCZk42cGZlcjZTajI2OUJRVVZ0Zk4rR0dicC94MwowNmJaUFUxK3BYTFBhYjI1aXpyMVAxbTBmaGxRSm55Qk9GdFFuU04zbm94eElCVlE5ZVhjV1FGQ1pqL3VXRzVMCkRtOFl6Mk1BWHE5K0djWXA3MDNLY2ZVNExPODJYWE55SUQrYURncHVMaTFDVFVERUFLS0FGamtDZ1lFQThGV0UKOHNvdm5sSXNITCtaaUJPQ3BYck9rbDlmWW0xRnJ2a0kzTUc5aHZ1V1RnNEorcHZScTFUZjA3U1crcXV0M3VZeQpYOEhvVE1SaE80SUZwZDl0TVdlWWovSnBoQ1FTcU1rcHlnc3hjM28wYzhzNFN2ZzlqOGo1SGFTb1o5UmZ3eWdiCmYzRHJDSDJQcjZleUVPRHErMTZ5cno0NkxiL0RRc1k4T3NUMEFiRUNnWUFndW1vdUJBSzVGNVhrOE4wVU90Yk0Kd0hwZ29LZjNvaEF3ZkJwUTRSNDNwUFBObHJvZHh5YlBQeCt4dGpLaTd6WFFBNEFWQ1VPZHFuYitJTVd5WWtRUgpvY3ZzbXJ3RzhoaDY5ajRuRHZwZ2dUdGFTdzVrRWR1Y3hHN1JyV3pmVmhnNHZNRlpnMi9aOGk3dzhPOXcwNWVSCnNreThuNjExenFRbFFEVjhkaUd4bVFLQmdRRGJ3dWR4OXkyNTBKdmpvZFByV1NQUzIwdi9EbFN6TlFaT0xBeE4KaUo4Y3lmc3ozcVNEVTI1VEE2WXorT05CempDTUxPU05LVXVZdnMzR1UydUV0Snd0Vy9SbVZCem1KdklsQXVWQwppaCtxMzJrTkpSdVJlaE1ZNG9YZzlFckZ2cTNlVDFOdG9qeFlwQy82U0JhTVZvNm9VbnlEd0J3RTcxL0dOR3lvCnRLWUcwUUtCZ0ZwYTY5anBmaW1wSTJJVTFTSmlqZkwxNFVWL09vRXBrUjFNMHU5cVFLZnNsM00xdzBmZ2huRVcKaEFYUUQzSTMxWnBkWkdYT0R6VzVJdUV5aExHaGJIWUgzejZldENJanFpNzdxWTlXbkJ3QjJwQ2Q3YzFuNVRCdwpNcElxV1BDbVQvSEpRUDAyYkhSZGlHSm55aURqZ3FpWnNYVmlrYk1yWjJCNjJqRHkvVVpmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg%3D%3D%0A", + "verification": {} + }, + "mode": 420 + }] + }, + "systemd": { + "units": [{ + "contents": "[Unit]\nDescription=etcd (System Application Container)\nDocumentation=https://github.com/coreos/etcd\nAfter=network-online.target\nWants=network-online.target\nRequires=setup-etcd-environment.service\n\n[Service]\nRestart=on-failure\nRestartSec=10s\nTimeoutStartSec=0\nLimitNOFILE=40000\n\n## FIXME(abhinav): these images should be replacable by release image.\nEnvironment=\"SIGNER_IMAGE=quay.io/coreos/kube-client-agent:678cc8e6841e2121ebfdb6e2db568fce290b67d6\"\nEnvironment=\"ETCD_IMAGE=quay.io/coreos/etcd:v3.2.14\"\nEnvironmentFile=/etc/etcd.env/etcd-environment\n\nExecStartPre=/bin/sh -c \" \\\n [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \\\n -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \\\n /bin/podman \\\n run \\\n --rm \\\n --volume /etc/ssl/etcd:/etc/ssl/etcd:rw,z \\\n --network host \\\n '${SIGNER_IMAGE}' \\\n request \\\n --orgname=system:etcd-servers \\\n --cacrt=/etc/ssl/etcd/root-ca.crt \\\n --assetsdir=/etc/ssl/etcd \\\n --address=https://adahiya-0-api.tt.testing:6443 \\\n --dnsnames=localhost,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,${ETCD_DNS_NAME} \\\n --commonname=system:etcd-server:${ETCD_DNS_NAME} \\\n --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \\\n\"\nExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt\nExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key\n\nExecStartPre=/bin/sh -c \" \\\n [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \\\n -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \\\n /bin/podman \\\n run \\\n --rm \\\n --volume /etc/ssl/etcd:/etc/ssl/etcd:rw,z \\\n --network host \\\n '${SIGNER_IMAGE}' \\\n request \\\n --orgname=system:etcd-peers \\\n --cacrt=/etc/ssl/etcd/root-ca.crt \\\n --assetsdir=/etc/ssl/etcd \\\n --address=https://adahiya-0-api.tt.testing:6443 \\\n --dnsnames=${ETCD_DNS_NAME},tt.testing \\\n --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\\n --ipaddrs=${ETCD_IPV4_ADDRESS} \\\n\"\nExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt\nExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key\n\nExecStartPre=-/bin/podman rm etcd-member\nExecStartPre=/usr/bin/mkdir --parents /var/lib/etcd\nExecStartPre=/usr/bin/mkdir --parents /run/etcd\nExecStartPre=/usr/bin/chown etcd /var/lib/etcd\nExecStartPre=/usr/bin/chown etcd /run/etcd\n\nExecStart= /usr/bin/bash -c \" \\\n /bin/podman \\\n run \\\n --rm \\\n --name etcd-member \\\n --volume /run/systemd/system:/run/systemd/system:ro,z \\\n --volume /etc/ssl/certs:/etc/ssl/certs:ro,z \\\n --volume /etc/ssl/etcd:/etc/ssl/etcd:ro,z \\\n --volume /var/lib/etcd:/var/lib/etcd:rw,z \\\n --volume /etc/ssl/certs:/etc/ssl/certs:ro,z \\\n --env 'ETCD_NAME=%m' \\\n --env ETCD_DATA_DIR=/var/lib/etcd \\\n --network host \\\n --user=$(id --user etcd) \\\n '${ETCD_IMAGE}' \\\n /usr/local/bin/etcd \\\n --name ${ETCD_DNS_NAME} \\\n --discovery-srv tt.testing \\\n --initial-advertise-peer-urls=https://${ETCD_IPV4_ADDRESS}:2380 \\\n --cert-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt \\\n --key-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key \\\n --trusted-ca-file=/etc/ssl/etcd/ca.crt \\\n --client-cert-auth=true \\\n --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt \\\n --peer-key-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key \\\n --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \\\n --peer-client-cert-auth=true \\\n --advertise-client-urls=https://${ETCD_IPV4_ADDRESS}:2379 \\\n --listen-client-urls=https://0.0.0.0:2379 \\\n --listen-peer-urls=https://0.0.0.0:2380 \\\n \"\n\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "etcd-member.service" + }, { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nWants=rpc-statd.service\n\n[Service]\nExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests\nEnvironmentFile=-/etc/kubernetes/kubelet-workaround\nEnvironmentFile=-/etc/kubernetes/kubelet-env\n\nExecStart=/usr/bin/hyperkube \\\n kubelet \\\n --config=/etc/kubernetes/kubelet.conf \\\n --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \\\n --rotate-certificates \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --container-runtime=remote \\\n --container-runtime-endpoint=/var/run/crio/crio.sock \\\n --allow-privileged \\\n --node-labels=node-role.kubernetes.io/master \\\n --minimum-container-ttl-duration=6m0s \\\n --client-ca-file=/etc/kubernetes/ca.crt \\\n --cloud-provider= \\\n \\\n --anonymous-auth=false \\\n --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \\\n\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "kubelet.service" + }, { + "contents": "[Unit]\nDescription=Setup Etcd Environment \nRequires=network-online.target \nAfter=network-online.target\n\n[Service]\nRemainAfterExit=yes \nType=oneshot\n\n## FIXME(abhinav): switch this to official image.\nEnvironment=\"IMAGE=docker.io/abhinavdahiya/origin-setup-etcd-environment\"\n\nExecStartPre=/usr/bin/mkdir --parents /etc/etcd.env\nExecStart=/bin/podman \\\n run \\\n --net host \\\n --rm \\\n --volume /etc/etcd.env:/etc/etcd.env:z \\\n ${IMAGE} \\\n --discovery-srv=tt.testing \\\n --output-file=/etc/etcd.env/etcd-environment \\\n --v=4 \\\n\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "setup-etcd-environment.service" + }] + } +} ``` diff --git a/hack/build-image.sh b/hack/build-image.sh index c9607b5cd6..01fa179089 100755 --- a/hack/build-image.sh +++ b/hack/build-image.sh @@ -63,5 +63,5 @@ fi for IMAGE_TO_BUILD in $TOBUILD; do NAME="${IMAGE_TO_BUILD#Dockerfile.}" set -x - podman build -t "${NAME}:${VERSION}" -f "${IMAGE_TO_BUILD}" + podman build -t "${NAME}:${VERSION}" -f "${IMAGE_TO_BUILD}" --no-cache done \ No newline at end of file diff --git a/manifests/machineconfigcontroller/controllerconfig.yaml b/manifests/machineconfigcontroller/controllerconfig.yaml index e432d39f97..4624535bcb 100644 --- a/manifests/machineconfigcontroller/controllerconfig.yaml +++ b/manifests/machineconfigcontroller/controllerconfig.yaml @@ -9,6 +9,5 @@ spec: clusterName: {{.ControllerConfig.ClusterName}} platform: {{.ControllerConfig.Platform}} baseDomain: {{.ControllerConfig.BaseDomain}} - etcdInitialCount: {{.ControllerConfig.EtcdInitialCount}} etcdCAData: {{.ControllerConfig.EtcdCAData | toString | b64enc}} rootCAData: {{.ControllerConfig.RootCAData | toString | b64enc}} diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/types.go b/pkg/apis/machineconfiguration.openshift.io/v1/types.go index 7e29d5364f..201de3fefb 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/types.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/types.go @@ -58,9 +58,6 @@ type MCOConfigSpec struct { Platform string `json:"platform"` BaseDomain string `json:"baseDomain"` - - // Size of the initial etcd cluster. - EtcdInitialCount int `json:"etcdInitialCount"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -125,9 +122,6 @@ type ControllerConfigSpec struct { BaseDomain string `json:"baseDomain"` - // Size of the initial etcd cluster. - EtcdInitialCount int `json:"etcdInitialCount"` - // CAs EtcdCAData []byte `json:"etcdCAData"` RootCAData []byte `json:"rootCAData"` diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index efa82b5996..0d1fd3b1ac 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -265,7 +265,6 @@ func renderTemplate(config renderConfig, path string, b []byte) ([]byte, error) funcs["skip"] = skipMissing funcs["etcdServerCertDNSNames"] = etcdServerCertDNSNames funcs["etcdPeerCertDNSNames"] = etcdPeerCertDNSNames - funcs["etcdInitialCluster"] = etcdInitialCluster funcs["apiServerURL"] = apiServerURL funcs["cloudProvider"] = cloudProvider tmpl, err := template.New(path).Funcs(funcs).Parse(string(b)) @@ -294,53 +293,31 @@ func skipMissing(key string) (interface{}, error) { // Process the {{etcdPeerCertDNSNames}} and {{etcdServerCertDNSNames}} func etcdServerCertDNSNames(cfg renderConfig) (interface{}, error) { - if cfg.ClusterName == "" || cfg.BaseDomain == "" || cfg.EtcdInitialCount <= 0 { + if cfg.BaseDomain == "" { return nil, fmt.Errorf("invalid configuration") } var dnsNames = []string{ "localhost", - "*.kube-etcd.kube-system.svc.cluster.local", - "kube-etcd-client.kube-system.svc.cluster.local", "etcd.kube-system.svc", // sign for the local etcd service name that cluster-network apiservers use to communicate "etcd.kube-system.svc.cluster.local", // sign for the local etcd service name that cluster-network apiservers use to communicate - } - - for i := 0; i < cfg.EtcdInitialCount; i++ { - dnsNames = append(dnsNames, fmt.Sprintf("%s-etcd-%d.%s", cfg.ClusterName, i, cfg.BaseDomain)) + "${ETCD_DNS_NAME}", } return strings.Join(dnsNames, ","), nil } func etcdPeerCertDNSNames(cfg renderConfig) (interface{}, error) { - if cfg.ClusterName == "" || cfg.BaseDomain == "" || cfg.EtcdInitialCount <= 0 { + if cfg.BaseDomain == "" { return nil, fmt.Errorf("invalid configuration") } var dnsNames = []string{ - "*.kube-etcd.kube-system.svc.cluster.local", - "kube-etcd-client.kube-system.svc.cluster.local", - } - - for i := 0; i < cfg.EtcdInitialCount; i++ { - dnsNames = append(dnsNames, fmt.Sprintf("%s-etcd-%d.%s", cfg.ClusterName, i, cfg.BaseDomain)) + "${ETCD_DNS_NAME}", + cfg.BaseDomain, // https://github.com/etcd-io/etcd/blob/583763261f1c843e07c1bf7fea5fb4cfb684fe87/Documentation/op-guide/clustering.md#dns-discovery } return strings.Join(dnsNames, ","), nil } -func etcdInitialCluster(cfg renderConfig) (interface{}, error) { - if cfg.ClusterName == "" || cfg.BaseDomain == "" || cfg.EtcdInitialCount <= 0 { - return nil, fmt.Errorf("invalid configuration") - } - - var addresses []string - for i := 0; i < cfg.EtcdInitialCount; i++ { - endpoint := fmt.Sprintf("%s-etcd-%d.%s", cfg.ClusterName, i, cfg.BaseDomain) - addresses = append(addresses, fmt.Sprintf("%s=https://%s:2380", endpoint, endpoint)) - } - return strings.Join(addresses, ","), nil -} - // generate apiserver url using cluster-name, basename func apiServerURL(cfg renderConfig) (interface{}, error) { if cfg.ClusterName == "" || cfg.BaseDomain == "" { diff --git a/pkg/controller/template/render_test.go b/pkg/controller/template/render_test.go index 25720b5645..e3538d02c4 100644 --- a/pkg/controller/template/render_test.go +++ b/pkg/controller/template/render_test.go @@ -108,106 +108,29 @@ func TestAPIServerURL(t *testing.T) { } } -func TestEtcdInitialCluster(t *testing.T) { - dummyTemplate := []byte(`{{etcdInitialCluster .}}`) - - cases := []struct { - clusterName string - baseDomain string - etcdCount int - - url string - err bool - }{{ - clusterName: "", - baseDomain: "", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "tt.testing", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "tt.testing", - etcdCount: 3, - url: "test-cluster-etcd-0.tt.testing=https://test-cluster-etcd-0.tt.testing:2380,test-cluster-etcd-1.tt.testing=https://test-cluster-etcd-1.tt.testing:2380,test-cluster-etcd-2.tt.testing=https://test-cluster-etcd-2.tt.testing:2380", - err: false, - }} - for idx, c := range cases { - name := fmt.Sprintf("case #%d", idx) - t.Run(name, func(t *testing.T) { - config := &mcfgv1.ControllerConfig{ - Spec: mcfgv1.ControllerConfigSpec{ - BaseDomain: c.baseDomain, - ClusterName: c.clusterName, - EtcdInitialCount: c.etcdCount, - }, - } - got, err := renderTemplate(renderConfig{&config.Spec}, name, dummyTemplate) - if err != nil && !c.err { - t.Fatalf("expected nil error %v", err) - } - - if string(got) != c.url { - t.Fatalf("mismatch got: %s want: %s", got, c.url) - } - }) - } -} - func TestEtcdPeerCertDNSNames(t *testing.T) { dummyTemplate := []byte(`{{etcdPeerCertDNSNames .}}`) cases := []struct { - clusterName string - baseDomain string - etcdCount int + baseDomain string url string err bool }{{ - clusterName: "", - baseDomain: "", - etcdCount: 0, - url: "", - err: true, + baseDomain: "", + url: "", + err: true, }, { - clusterName: "test-cluster", - baseDomain: "", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "tt.testing", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "tt.testing", - etcdCount: 3, - url: "*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,test-cluster-etcd-0.tt.testing,test-cluster-etcd-1.tt.testing,test-cluster-etcd-2.tt.testing", - err: false, + baseDomain: "tt.testing", + url: "${ETCD_DNS_NAME},tt.testing", + err: false, }} for idx, c := range cases { name := fmt.Sprintf("case #%d", idx) t.Run(name, func(t *testing.T) { config := &mcfgv1.ControllerConfig{ Spec: mcfgv1.ControllerConfigSpec{ - BaseDomain: c.baseDomain, - ClusterName: c.clusterName, - EtcdInitialCount: c.etcdCount, + BaseDomain: c.baseDomain, }, } got, err := renderTemplate(renderConfig{&config.Spec}, name, dummyTemplate) @@ -226,45 +149,25 @@ func TestEtcdServerCertDNSNames(t *testing.T) { dummyTemplate := []byte(`{{etcdServerCertDNSNames .}}`) cases := []struct { - clusterName string - baseDomain string - etcdCount int + baseDomain string url string err bool }{{ - clusterName: "", - baseDomain: "", - etcdCount: 0, - url: "", - err: true, + baseDomain: "", + url: "", + err: true, }, { - clusterName: "test-cluster", - baseDomain: "", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "tt.testing", - etcdCount: 0, - url: "", - err: true, - }, { - clusterName: "test-cluster", - baseDomain: "tt.testing", - etcdCount: 3, - url: "localhost,*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,test-cluster-etcd-0.tt.testing,test-cluster-etcd-1.tt.testing,test-cluster-etcd-2.tt.testing", - err: false, + baseDomain: "tt.testing", + url: "localhost,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,${ETCD_DNS_NAME}", + err: false, }} for idx, c := range cases { name := fmt.Sprintf("case #%d", idx) t.Run(name, func(t *testing.T) { config := &mcfgv1.ControllerConfig{ Spec: mcfgv1.ControllerConfigSpec{ - BaseDomain: c.baseDomain, - ClusterName: c.clusterName, - EtcdInitialCount: c.etcdCount, + BaseDomain: c.baseDomain, }, } got, err := renderTemplate(renderConfig{&config.Spec}, name, dummyTemplate) @@ -331,9 +234,9 @@ const ( var ( configs = map[string]string{ - "aws": "./test_data/controller_config_aws.yaml", - "openstack": "./test_data/controller_config_openstack.yaml", - "libvirt": "./test_data/controller_config_libvirt.yaml", + "aws": "./test_data/controller_config_aws.yaml", + "openstack": "./test_data/controller_config_openstack.yaml", + "libvirt": "./test_data/controller_config_libvirt.yaml", } ) diff --git a/pkg/controller/template/template_controller_test.go b/pkg/controller/template/template_controller_test.go index 2bb68c1ea0..92697a9473 100644 --- a/pkg/controller/template/template_controller_test.go +++ b/pkg/controller/template/template_controller_test.go @@ -51,11 +51,10 @@ func newControllerConfig(name string) *mcfgv1.ControllerConfig { TypeMeta: metav1.TypeMeta{APIVersion: mcfgv1.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: metav1.NamespaceDefault}, Spec: mcfgv1.ControllerConfigSpec{ - ClusterDNSIP: "10.3.0.1/16", - ClusterName: name, - BaseDomain: "openshift.testing", - EtcdInitialCount: 1, - Platform: "libvirt", + ClusterDNSIP: "10.3.0.1/16", + ClusterName: name, + BaseDomain: "openshift.testing", + Platform: "libvirt", }, } } diff --git a/pkg/controller/template/test_data/templates/aws/master/units/etcd-member.service b/pkg/controller/template/test_data/templates/aws/master/units/etcd-member.service index 152775a4f6..4d4ef27ddc 100644 --- a/pkg/controller/template/test_data/templates/aws/master/units/etcd-member.service +++ b/pkg/controller/template/test_data/templates/aws/master/units/etcd-member.service @@ -4,6 +4,7 @@ contents: | Documentation=https://github.com/coreos/etcd After=network-online.target Wants=network-online.target + Requires=setup-etcd-environment.service [Service] Restart=on-failure @@ -11,12 +12,14 @@ contents: | TimeoutStartSec=0 LimitNOFILE=40000 + ## FIXME(abhinav): these images should be replacable by release image. Environment="SIGNER_IMAGE=quay.io/coreos/kube-client-agent:678cc8e6841e2121ebfdb6e2db568fce290b67d6" Environment="ETCD_IMAGE=quay.io/coreos/etcd:v3.2.14" + EnvironmentFile=/run/etcd/environment ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt -a \ - -e /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -28,16 +31,16 @@ contents: | --cacrt=/etc/ssl/etcd/root-ca.crt \ --assetsdir=/etc/ssl/etcd \ --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \ - --dnsnames=localhost,*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \ - --commonname=system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ - --ipaddrs=127.0.0.1 \ + --dnsnames=localhost,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,${ETCD_DNS_NAME} \ + --commonname=system:etcd-server:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt -a \ - -e /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -49,11 +52,12 @@ contents: | --cacrt=/etc/ssl/etcd/root-ca.crt \ --assetsdir=/etc/ssl/etcd \ --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \ - --dnsnames=*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \ - --commonname=system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ + --dnsnames=${ETCD_DNS_NAME},installer.team.coreos.systems \ + --commonname=system:etcd-peer:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS} \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ExecStartPre=-/bin/podman rm etcd-member ExecStartPre=/usr/bin/mkdir --parents /var/lib/etcd @@ -77,18 +81,18 @@ contents: | --user=$(id --user etcd) \ '${ETCD_IMAGE}' \ /usr/local/bin/etcd \ - --name=my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ - --advertise-client-urls=https://my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems:2379 \ - --cert-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt \ - --key-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key \ + --name ${ETCD_DNS_NAME} \ + --discovery-srv installer.team.coreos.systems \ + --initial-advertise-peer-urls=https://${ETCD_IPV4_ADDRESS}:2380 \ + --cert-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt \ + --key-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key \ --trusted-ca-file=/etc/ssl/etcd/ca.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt \ - --peer-key-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key \ + --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt \ + --peer-key-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key \ --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \ --peer-client-cert-auth=true \ - --initial-cluster='my-test-cluster-etcd-0.installer.team.coreos.systems=https://my-test-cluster-etcd-0.installer.team.coreos.systems:2380,my-test-cluster-etcd-1.installer.team.coreos.systems=https://my-test-cluster-etcd-1.installer.team.coreos.systems:2380,my-test-cluster-etcd-2.installer.team.coreos.systems=https://my-test-cluster-etcd-2.installer.team.coreos.systems:2380' \ - --initial-advertise-peer-urls=https://my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems:2380 \ + --advertise-client-urls=https://${ETCD_IPV4_ADDRESS}:2379 \ --listen-client-urls=https://0.0.0.0:2379 \ --listen-peer-urls=https://0.0.0.0:2380 \ " diff --git a/pkg/controller/template/test_data/templates/aws/master/units/setup-etcd-environment.service b/pkg/controller/template/test_data/templates/aws/master/units/setup-etcd-environment.service new file mode 100644 index 0000000000..428e07adcf --- /dev/null +++ b/pkg/controller/template/test_data/templates/aws/master/units/setup-etcd-environment.service @@ -0,0 +1,8 @@ +contents: "[Unit]\nDescription=Setup Etcd Environment \nRequires=network-online.target + \ \nAfter=network-online.target\n\n[Service]\nRestart=on-failure\nRestartSec=5s\n\n## + FIXME(abhinav): switch this to official image.\nEnvironment=\"IMAGE=docker.io/abhinavdahiya/origin-setup-etcd-environment\"\n\nExecStartPre=/usr/bin/mkdir + --parents /run/etcd\nExecStart=/bin/podman \\\n run \\\n --net host \\\n --rm + \\\n --volume /run/etcd:/run/etcd:z \\\n ${IMAGE} \\\n --discovery-srv=installer.team.coreos.systems + \\\n --output-file=/run/etcd/environment \\\n --v=4 \\\n\n[Install]\nWantedBy=multi-user.target\n" +enabled: true +name: setup-etcd-environment.service diff --git a/pkg/controller/template/test_data/templates/libvirt/master/units/etcd-member.service b/pkg/controller/template/test_data/templates/libvirt/master/units/etcd-member.service index 152775a4f6..4d4ef27ddc 100644 --- a/pkg/controller/template/test_data/templates/libvirt/master/units/etcd-member.service +++ b/pkg/controller/template/test_data/templates/libvirt/master/units/etcd-member.service @@ -4,6 +4,7 @@ contents: | Documentation=https://github.com/coreos/etcd After=network-online.target Wants=network-online.target + Requires=setup-etcd-environment.service [Service] Restart=on-failure @@ -11,12 +12,14 @@ contents: | TimeoutStartSec=0 LimitNOFILE=40000 + ## FIXME(abhinav): these images should be replacable by release image. Environment="SIGNER_IMAGE=quay.io/coreos/kube-client-agent:678cc8e6841e2121ebfdb6e2db568fce290b67d6" Environment="ETCD_IMAGE=quay.io/coreos/etcd:v3.2.14" + EnvironmentFile=/run/etcd/environment ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt -a \ - -e /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -28,16 +31,16 @@ contents: | --cacrt=/etc/ssl/etcd/root-ca.crt \ --assetsdir=/etc/ssl/etcd \ --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \ - --dnsnames=localhost,*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \ - --commonname=system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ - --ipaddrs=127.0.0.1 \ + --dnsnames=localhost,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,${ETCD_DNS_NAME} \ + --commonname=system:etcd-server:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt -a \ - -e /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -49,11 +52,12 @@ contents: | --cacrt=/etc/ssl/etcd/root-ca.crt \ --assetsdir=/etc/ssl/etcd \ --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \ - --dnsnames=*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \ - --commonname=system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ + --dnsnames=${ETCD_DNS_NAME},installer.team.coreos.systems \ + --commonname=system:etcd-peer:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS} \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ExecStartPre=-/bin/podman rm etcd-member ExecStartPre=/usr/bin/mkdir --parents /var/lib/etcd @@ -77,18 +81,18 @@ contents: | --user=$(id --user etcd) \ '${ETCD_IMAGE}' \ /usr/local/bin/etcd \ - --name=my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ - --advertise-client-urls=https://my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems:2379 \ - --cert-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt \ - --key-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key \ + --name ${ETCD_DNS_NAME} \ + --discovery-srv installer.team.coreos.systems \ + --initial-advertise-peer-urls=https://${ETCD_IPV4_ADDRESS}:2380 \ + --cert-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt \ + --key-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key \ --trusted-ca-file=/etc/ssl/etcd/ca.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt \ - --peer-key-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key \ + --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt \ + --peer-key-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key \ --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \ --peer-client-cert-auth=true \ - --initial-cluster='my-test-cluster-etcd-0.installer.team.coreos.systems=https://my-test-cluster-etcd-0.installer.team.coreos.systems:2380,my-test-cluster-etcd-1.installer.team.coreos.systems=https://my-test-cluster-etcd-1.installer.team.coreos.systems:2380,my-test-cluster-etcd-2.installer.team.coreos.systems=https://my-test-cluster-etcd-2.installer.team.coreos.systems:2380' \ - --initial-advertise-peer-urls=https://my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems:2380 \ + --advertise-client-urls=https://${ETCD_IPV4_ADDRESS}:2379 \ --listen-client-urls=https://0.0.0.0:2379 \ --listen-peer-urls=https://0.0.0.0:2380 \ " diff --git a/pkg/controller/template/test_data/templates/libvirt/master/units/setup-etcd-environment.service b/pkg/controller/template/test_data/templates/libvirt/master/units/setup-etcd-environment.service new file mode 100644 index 0000000000..428e07adcf --- /dev/null +++ b/pkg/controller/template/test_data/templates/libvirt/master/units/setup-etcd-environment.service @@ -0,0 +1,8 @@ +contents: "[Unit]\nDescription=Setup Etcd Environment \nRequires=network-online.target + \ \nAfter=network-online.target\n\n[Service]\nRestart=on-failure\nRestartSec=5s\n\n## + FIXME(abhinav): switch this to official image.\nEnvironment=\"IMAGE=docker.io/abhinavdahiya/origin-setup-etcd-environment\"\n\nExecStartPre=/usr/bin/mkdir + --parents /run/etcd\nExecStart=/bin/podman \\\n run \\\n --net host \\\n --rm + \\\n --volume /run/etcd:/run/etcd:z \\\n ${IMAGE} \\\n --discovery-srv=installer.team.coreos.systems + \\\n --output-file=/run/etcd/environment \\\n --v=4 \\\n\n[Install]\nWantedBy=multi-user.target\n" +enabled: true +name: setup-etcd-environment.service diff --git a/pkg/controller/template/test_data/templates/openstack/master/units/etcd-member.service b/pkg/controller/template/test_data/templates/openstack/master/units/etcd-member.service index 152775a4f6..4d4ef27ddc 100644 --- a/pkg/controller/template/test_data/templates/openstack/master/units/etcd-member.service +++ b/pkg/controller/template/test_data/templates/openstack/master/units/etcd-member.service @@ -4,6 +4,7 @@ contents: | Documentation=https://github.com/coreos/etcd After=network-online.target Wants=network-online.target + Requires=setup-etcd-environment.service [Service] Restart=on-failure @@ -11,12 +12,14 @@ contents: | TimeoutStartSec=0 LimitNOFILE=40000 + ## FIXME(abhinav): these images should be replacable by release image. Environment="SIGNER_IMAGE=quay.io/coreos/kube-client-agent:678cc8e6841e2121ebfdb6e2db568fce290b67d6" Environment="ETCD_IMAGE=quay.io/coreos/etcd:v3.2.14" + EnvironmentFile=/run/etcd/environment ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt -a \ - -e /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -28,16 +31,16 @@ contents: | --cacrt=/etc/ssl/etcd/root-ca.crt \ --assetsdir=/etc/ssl/etcd \ --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \ - --dnsnames=localhost,*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \ - --commonname=system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ - --ipaddrs=127.0.0.1 \ + --dnsnames=localhost,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,${ETCD_DNS_NAME} \ + --commonname=system:etcd-server:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt -a \ - -e /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -49,11 +52,12 @@ contents: | --cacrt=/etc/ssl/etcd/root-ca.crt \ --assetsdir=/etc/ssl/etcd \ --address=https://my-test-cluster-api.installer.team.coreos.systems:6443 \ - --dnsnames=*.kube-etcd.kube-system.svc.cluster.local,kube-etcd-client.kube-system.svc.cluster.local,my-test-cluster-etcd-0.installer.team.coreos.systems,my-test-cluster-etcd-1.installer.team.coreos.systems,my-test-cluster-etcd-2.installer.team.coreos.systems \ - --commonname=system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ + --dnsnames=${ETCD_DNS_NAME},installer.team.coreos.systems \ + --commonname=system:etcd-peer:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS} \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ExecStartPre=-/bin/podman rm etcd-member ExecStartPre=/usr/bin/mkdir --parents /var/lib/etcd @@ -77,18 +81,18 @@ contents: | --user=$(id --user etcd) \ '${ETCD_IMAGE}' \ /usr/local/bin/etcd \ - --name=my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems \ - --advertise-client-urls=https://my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems:2379 \ - --cert-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt \ - --key-file=/etc/ssl/etcd/system:etcd-server:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key \ + --name ${ETCD_DNS_NAME} \ + --discovery-srv installer.team.coreos.systems \ + --initial-advertise-peer-urls=https://${ETCD_IPV4_ADDRESS}:2380 \ + --cert-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt \ + --key-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key \ --trusted-ca-file=/etc/ssl/etcd/ca.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.crt \ - --peer-key-file=/etc/ssl/etcd/system:etcd-peer:my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems.key \ + --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt \ + --peer-key-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key \ --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \ --peer-client-cert-auth=true \ - --initial-cluster='my-test-cluster-etcd-0.installer.team.coreos.systems=https://my-test-cluster-etcd-0.installer.team.coreos.systems:2380,my-test-cluster-etcd-1.installer.team.coreos.systems=https://my-test-cluster-etcd-1.installer.team.coreos.systems:2380,my-test-cluster-etcd-2.installer.team.coreos.systems=https://my-test-cluster-etcd-2.installer.team.coreos.systems:2380' \ - --initial-advertise-peer-urls=https://my-test-cluster-etcd-{{.etcd_index}}.installer.team.coreos.systems:2380 \ + --advertise-client-urls=https://${ETCD_IPV4_ADDRESS}:2379 \ --listen-client-urls=https://0.0.0.0:2379 \ --listen-peer-urls=https://0.0.0.0:2380 \ " diff --git a/pkg/controller/template/test_data/templates/openstack/master/units/setup-etcd-environment.service b/pkg/controller/template/test_data/templates/openstack/master/units/setup-etcd-environment.service new file mode 100644 index 0000000000..428e07adcf --- /dev/null +++ b/pkg/controller/template/test_data/templates/openstack/master/units/setup-etcd-environment.service @@ -0,0 +1,8 @@ +contents: "[Unit]\nDescription=Setup Etcd Environment \nRequires=network-online.target + \ \nAfter=network-online.target\n\n[Service]\nRestart=on-failure\nRestartSec=5s\n\n## + FIXME(abhinav): switch this to official image.\nEnvironment=\"IMAGE=docker.io/abhinavdahiya/origin-setup-etcd-environment\"\n\nExecStartPre=/usr/bin/mkdir + --parents /run/etcd\nExecStart=/bin/podman \\\n run \\\n --net host \\\n --rm + \\\n --volume /run/etcd:/run/etcd:z \\\n ${IMAGE} \\\n --discovery-srv=installer.team.coreos.systems + \\\n --output-file=/run/etcd/environment \\\n --v=4 \\\n\n[Install]\nWantedBy=multi-user.target\n" +enabled: true +name: setup-etcd-environment.service diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 585ca98423..b217c70ad2 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -304,7 +304,6 @@ spec: clusterName: {{.ControllerConfig.ClusterName}} platform: {{.ControllerConfig.Platform}} baseDomain: {{.ControllerConfig.BaseDomain}} - etcdInitialCount: {{.ControllerConfig.EtcdInitialCount}} etcdCAData: {{.ControllerConfig.EtcdCAData | toString | b64enc}} rootCAData: {{.ControllerConfig.RootCAData | toString | b64enc}} `) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 786f697904..f0f4a0dd49 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -300,7 +300,6 @@ func getRenderConfig(mc *mcfgv1.MCOConfig, etcdCAData, rootCAData []byte, imgs I ClusterName: mc.Spec.ClusterName, Platform: mc.Spec.Platform, BaseDomain: mc.Spec.BaseDomain, - EtcdInitialCount: mc.Spec.EtcdInitialCount, EtcdCAData: etcdCAData, RootCAData: rootCAData, } diff --git a/pkg/operator/render.go b/pkg/operator/render.go index 797dbf0ddb..4f92e56d7f 100644 --- a/pkg/operator/render.go +++ b/pkg/operator/render.go @@ -7,7 +7,7 @@ import ( "text/template" "github.com/Masterminds/sprig" - cidr "github.com/apparentlymart/go-cidr/cidr" + "github.com/apparentlymart/go-cidr/cidr" installertypes "github.com/openshift/installer/pkg/types" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" @@ -54,16 +54,6 @@ func discoverMCOConfig(f installConfigGetter) (*mcfgv1.MCOConfig, error) { return nil, err } - var eic int - for _, m := range ic.Machines { - if m.Name == "master" && m.Replicas != nil { - eic = int(*(m.Replicas)) - } - } - if eic == 0 { - return nil, fmt.Errorf("EtcdInitialCount cannot be empty") - } - return &mcfgv1.MCOConfig{ Spec: mcfgv1.MCOConfigSpec{ ClusterDNSIP: dnsIP, @@ -71,7 +61,6 @@ func discoverMCOConfig(f installConfigGetter) (*mcfgv1.MCOConfig, error) { ClusterName: ic.ObjectMeta.Name, Platform: platformFromInstallConfig(ic), BaseDomain: ic.BaseDomain, - EtcdInitialCount: eic, }, }, nil } diff --git a/pkg/operator/render_test.go b/pkg/operator/render_test.go index 9f70ed225c..73f28ee521 100644 --- a/pkg/operator/render_test.go +++ b/pkg/operator/render_test.go @@ -104,7 +104,4 @@ func TestDiscoverMCOConfig(t *testing.T) { if got, want := mco.Spec.BaseDomain, "tt.testing"; got != want { t.Fatalf("mismatch got = %v want = %v", got, want) } - if got, want := mco.Spec.EtcdInitialCount, 1; got != want { - t.Fatalf("mismatch got = %v want = %v", got, want) - } } diff --git a/pkg/server/api.go b/pkg/server/api.go index bca0cb1c85..fd77fcb5dc 100644 --- a/pkg/server/api.go +++ b/pkg/server/api.go @@ -16,7 +16,6 @@ const ( type poolRequest struct { machinePool string - etcdIndex string } // APIServer provides the HTTP(s) endpoint @@ -91,7 +90,6 @@ func (sh *APIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { cr := poolRequest{ machinePool: path.Base(r.URL.Path), - etcdIndex: r.URL.Query().Get(apiParamEtcd), } conf, err := sh.server.GetConfig(cr) diff --git a/pkg/server/server.go b/pkg/server/server.go index 43be703c7e..b36864a5f6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "net/url" - "strings" ignv2_2types "github.com/coreos/ignition/config/v2_2/types" "github.com/openshift/machine-config-operator/pkg/daemon" @@ -43,8 +42,6 @@ type Server interface { func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc) []appenderFunc { appenders := []appenderFunc{ - // execute etcd templating. - func(config *ignv2_2types.Config) error { return execEtcdTemplates(config, cr.etcdIndex) }, // append machine annotations file. func(config *ignv2_2types.Config) error { return appendNodeAnnotations(config, currMachineConfig) }, // append kubeconfig. @@ -112,23 +109,6 @@ func appendFileToIgnition(conf *ignv2_2types.Config, outPath, contents string) { conf.Storage.Files = append(conf.Storage.Files, file) } -func execEtcdTemplates(conf *ignv2_2types.Config, etcdIndex string) error { - if etcdIndex == "" { - return nil - } - if len(conf.Systemd.Units) > 0 { - for i := range conf.Systemd.Units { - conf.Systemd.Units[i].Contents = strings.Replace(conf.Systemd.Units[i].Contents, etcdTemplateParam, etcdIndex, -1) - - for j := range conf.Systemd.Units[i].Dropins { - conf.Systemd.Units[i].Dropins[j].Contents = - strings.Replace(conf.Systemd.Units[i].Dropins[j].Contents, etcdTemplateParam, etcdIndex, -1) - } - } - } - return nil -} - func getDecodedContent(inp string) (string, error) { d, err := dataurl.DecodeString(inp) if err != nil { diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index fc36c23d4e..5cd031d673 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "path" "reflect" - "strings" "testing" ignv2_2types "github.com/coreos/ignition/config/v2_2/types" @@ -46,42 +45,6 @@ func TestStringEncode(t *testing.T) { } } -func TestEtcdTemplate(t *testing.T) { - etcdInd := "1" - inpContent := "etcd-{{.etcd_index}}" - outContent := fmt.Sprintf("etcd-%s", etcdInd) - - inpIgn := ignv2_2types.Config{ - Systemd: ignv2_2types.Systemd{ - Units: []ignv2_2types.Unit{ - { - Contents: inpContent, - }, - }, - }, - } - expIgn := ignv2_2types.Config{ - Systemd: ignv2_2types.Systemd{ - Units: []ignv2_2types.Unit{ - { - Contents: outContent, - }, - }, - }, - } - execEtcdTemplates(&inpIgn, "") - if inpIgn.Systemd.Units[0].Contents != inpContent { - t.Errorf("expected no transformations when etcd_index is \"\" ") - } - - err := execEtcdTemplates(&inpIgn, etcdInd) - if err != nil { - t.Errorf("expected err to not be nil, received: %v", err) - } - - validateIgnitionSystemd(t, expIgn.Systemd.Units, inpIgn.Systemd.Units) -} - // TestBootstrapServer tests the behavior of the machine config server // when it's running in bootstrap mode. // The test does the following: @@ -90,15 +53,12 @@ func TestEtcdTemplate(t *testing.T) { // 2. Fetch the machine-config from the testdata. // 3. Manually update the ignition config from Step 2 by adding // the node-annotations file, the kubeconfig file(which is read -// from the testdata), update the etcd_index in the systemd unit to -// desired value, by a string replace. This ignition config is then +// from the testdata). This ignition config is then // labeled as expected Ignition config. // 4. Call the Bootstrap GetConfig method by passing the reference to the // machine pool present in the testdata folder. // 5. Compare the Ignition configs from Step 3 and Step 4. func TestBootstrapServer(t *testing.T) { - etcdIndex := "1" - mp, err := getTestMachinePool() if err != nil { t.Fatal(err) @@ -110,10 +70,8 @@ func TestBootstrapServer(t *testing.T) { t.Fatalf("unexpected error while reading machine-config: %s, err: %v", mcPath, err) } - // replace etcd_index param - finalMCData := strings.Replace(string(mcData), etcdTemplateParam, etcdIndex, -1) mc := new(v1.MachineConfig) - err = yaml.Unmarshal([]byte(finalMCData), mc) + err = yaml.Unmarshal([]byte(mcData), mc) if err != nil { t.Fatalf("unexpected error while unmarshaling machine-config: %s, err: %v", mcPath, err) } @@ -140,7 +98,6 @@ func TestBootstrapServer(t *testing.T) { } res, err := bs.GetConfig(poolRequest{ machinePool: testPool, - etcdIndex: etcdIndex, }) if err != nil { t.Fatalf("expected err to be nil, received: %v", err) @@ -159,8 +116,7 @@ func TestBootstrapServer(t *testing.T) { // 2. Fetch the machine-config from the testdata, call this origMC. // 3. Manually update the ignition config from Step 2 by adding // the node-annotations file, the kubeconfig file(which is read -// from the testdata), update the etcd_index in the systemd unit to -// desired value, by a string replace. This ignition config is then +// from the testdata). This ignition config is then // labeled as expected Ignition config (mc). // 4. Use the Kubernetes fake client to Create the machine pool and the config // objects from Step 1, 2 inside the cluster. @@ -193,16 +149,13 @@ func TestClusterServer(t *testing.T) { t.Logf("err: %v", err) } - etcdIndex := "1" csc := &clusterServer{ machineClient: cs.MachineconfigurationV1(), kubeconfigFunc: func() ([]byte, []byte, error) { return getKubeConfigContent(t) }, } - // replace etcd_index param - finalMCData := strings.Replace(string(mcData), etcdTemplateParam, etcdIndex, -1) mc := new(v1.MachineConfig) - err = yaml.Unmarshal([]byte(finalMCData), mc) + err = yaml.Unmarshal([]byte(mcData), mc) if err != nil { t.Fatalf("unexpected error while unmarshaling machine-config: %s, err: %v", mcPath, err) } @@ -220,7 +173,6 @@ func TestClusterServer(t *testing.T) { res, err := csc.GetConfig(poolRequest{ machinePool: testPool, - etcdIndex: etcdIndex, }) if err != nil { t.Fatalf("expected err to be nil, received: %v", err) diff --git a/templates/_base/master/units/etcd-member.yaml b/templates/_base/master/units/etcd-member.yaml index dabd69ba1f..4d6c1dba22 100644 --- a/templates/_base/master/units/etcd-member.yaml +++ b/templates/_base/master/units/etcd-member.yaml @@ -6,6 +6,7 @@ contents: | Documentation=https://github.com/coreos/etcd After=network-online.target Wants=network-online.target + Requires=setup-etcd-environment.service [Service] Restart=on-failure @@ -13,12 +14,14 @@ contents: | TimeoutStartSec=0 LimitNOFILE=40000 + ## FIXME(abhinav): these images should be replacable by release image. Environment="SIGNER_IMAGE=quay.io/coreos/kube-client-agent:678cc8e6841e2121ebfdb6e2db568fce290b67d6" Environment="ETCD_IMAGE=quay.io/coreos/etcd:v3.2.14" + EnvironmentFile=/run/etcd/environment ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.crt -a \ - -e /etc/ssl/etcd/system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -31,15 +34,15 @@ contents: | --assetsdir=/etc/ssl/etcd \ --address={{apiServerURL .}} \ --dnsnames={{etcdServerCertDNSNames .}} \ - --commonname=system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}} \ - --ipaddrs=127.0.0.1 \ + --commonname=system:etcd-server:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS},127.0.0.1 \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key ExecStartPre=/bin/sh -c " \ - [ -e /etc/ssl/etcd/system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.crt -a \ - -e /etc/ssl/etcd/system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.key ] || \ + [ -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt -a \ + -e /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ] || \ /bin/podman \ run \ --rm \ @@ -52,10 +55,11 @@ contents: | --assetsdir=/etc/ssl/etcd \ --address={{apiServerURL .}} \ --dnsnames={{etcdPeerCertDNSNames .}} \ - --commonname=system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}} \ + --commonname=system:etcd-peer:${ETCD_DNS_NAME} \ + --ipaddrs=${ETCD_IPV4_ADDRESS} \ " - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.crt - ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.key + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt + ExecStartPre=/bin/chown etcd:etcd /etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key ExecStartPre=-/bin/podman rm etcd-member ExecStartPre=/usr/bin/mkdir --parents /var/lib/etcd @@ -79,18 +83,18 @@ contents: | --user=$(id --user etcd) \ '${ETCD_IMAGE}' \ /usr/local/bin/etcd \ - --name={{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}} \ - --advertise-client-urls=https://{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}:2379 \ - --cert-file=/etc/ssl/etcd/system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.crt \ - --key-file=/etc/ssl/etcd/system:etcd-server:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.key \ + --name ${ETCD_DNS_NAME} \ + --discovery-srv {{.BaseDomain}} \ + --initial-advertise-peer-urls=https://${ETCD_IPV4_ADDRESS}:2380 \ + --cert-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.crt \ + --key-file=/etc/ssl/etcd/system:etcd-server:${ETCD_DNS_NAME}.key \ --trusted-ca-file=/etc/ssl/etcd/ca.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.crt \ - --peer-key-file=/etc/ssl/etcd/system:etcd-peer:{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}.key \ + --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.crt \ + --peer-key-file=/etc/ssl/etcd/system:etcd-peer:${ETCD_DNS_NAME}.key \ --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \ --peer-client-cert-auth=true \ - --initial-cluster='{{etcdInitialCluster .}}' \ - --initial-advertise-peer-urls=https://{{.ClusterName}}-etcd-{{skip "etcd_index"}}.{{.BaseDomain}}:2380 \ + --advertise-client-urls=https://${ETCD_IPV4_ADDRESS}:2379 \ --listen-client-urls=https://0.0.0.0:2379 \ --listen-peer-urls=https://0.0.0.0:2380 \ " diff --git a/templates/_base/master/units/setup-etcd-environment.yaml b/templates/_base/master/units/setup-etcd-environment.yaml new file mode 100644 index 0000000000..7b841a24ad --- /dev/null +++ b/templates/_base/master/units/setup-etcd-environment.yaml @@ -0,0 +1,28 @@ +name: "setup-etcd-environment.service" +enabled: true +contents: | + [Unit] + Description=Setup Etcd Environment + Requires=network-online.target + After=network-online.target + + [Service] + Restart=on-failure + RestartSec=5s + + ## FIXME(abhinav): switch this to official image. + Environment="IMAGE=docker.io/abhinavdahiya/origin-setup-etcd-environment" + + ExecStartPre=/usr/bin/mkdir --parents /run/etcd + ExecStart=/bin/podman \ + run \ + --net host \ + --rm \ + --volume /run/etcd:/run/etcd:z \ + ${IMAGE} \ + --discovery-srv={{.BaseDomain}} \ + --output-file=/run/etcd/environment \ + --v=4 \ + + [Install] + WantedBy=multi-user.target