From 0268df075604b204ac9beb35f79c7c3b7f8bb780 Mon Sep 17 00:00:00 2001 From: Luis Sanchez Date: Tue, 11 Feb 2020 13:25:26 -0500 Subject: [PATCH 1/2] do not clobber bootstrap --- .../hostendpointscontroller/host_endpoints_controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/operator/hostendpointscontroller/host_endpoints_controller.go b/pkg/operator/hostendpointscontroller/host_endpoints_controller.go index 295e55fdb5..a02c4e0fb4 100644 --- a/pkg/operator/hostendpointscontroller/host_endpoints_controller.go +++ b/pkg/operator/hostendpointscontroller/host_endpoints_controller.go @@ -139,10 +139,6 @@ func (c *HostEndpointsController) syncHostEndpoints() error { } } - if len(addresses) == 0 { - return fmt.Errorf("no etcd member pods are ready") - } - required := hostEndpointsAsset() if required.Annotations == nil { @@ -172,6 +168,10 @@ func (c *HostEndpointsController) syncHostEndpoints() error { } } + if len(required.Subsets[0].Addresses) == 0 { + return fmt.Errorf("no etcd member pods are ready") + } + return c.applyEndpoints(required) } From c9b66c525935a60681580baeb1b1aefb8c0a811d Mon Sep 17 00:00:00 2001 From: Luis Sanchez Date: Tue, 11 Feb 2020 13:23:59 -0500 Subject: [PATCH 2/2] add all etcd nodes to host-etcd resource --- .../host_endpoints_controller.go | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/pkg/operator/hostendpointscontroller/host_endpoints_controller.go b/pkg/operator/hostendpointscontroller/host_endpoints_controller.go index a02c4e0fb4..5e1429342f 100644 --- a/pkg/operator/hostendpointscontroller/host_endpoints_controller.go +++ b/pkg/operator/hostendpointscontroller/host_endpoints_controller.go @@ -41,7 +41,7 @@ const ( type HostEndpointsController struct { operatorClient v1helpers.OperatorClient infrastructureLister configv1listers.InfrastructureLister - podLister corev1listers.PodLister + nodeLister corev1listers.NodeLister endpointsLister corev1listers.EndpointsLister endpointsClient corev1client.EndpointsGetter @@ -54,12 +54,13 @@ func NewHostEndpointsController( operatorClient v1helpers.OperatorClient, eventRecorder events.Recorder, kubeClient kubernetes.Interface, - kubeInformersForNamespaces operatorv1helpers.KubeInformersForNamespaces, + kubeInformers operatorv1helpers.KubeInformersForNamespaces, infrastructureInformer configv1informers.InfrastructureInformer, ) *HostEndpointsController { - kubeInformersForTargetNamespace := kubeInformersForNamespaces.InformersFor("openshift-etcd") + kubeInformersForTargetNamespace := kubeInformers.InformersFor("openshift-etcd") endpointsInformer := kubeInformersForTargetNamespace.Core().V1().Endpoints() - podInformer := kubeInformersForTargetNamespace.Core().V1().Pods() + kubeInformersForCluster := kubeInformers.InformersFor("") + nodeInformer := kubeInformersForCluster.Core().V1().Nodes() c := &HostEndpointsController{ eventRecorder: eventRecorder.WithComponentSuffix("host-etcd-endpoints-controller"), @@ -67,19 +68,19 @@ func NewHostEndpointsController( cachesToSync: []cache.InformerSynced{ operatorClient.Informer().HasSynced, endpointsInformer.Informer().HasSynced, - podInformer.Informer().HasSynced, + nodeInformer.Informer().HasSynced, infrastructureInformer.Informer().HasSynced, }, operatorClient: operatorClient, infrastructureLister: infrastructureInformer.Lister(), - podLister: podInformer.Lister(), + nodeLister: nodeInformer.Lister(), endpointsLister: endpointsInformer.Lister(), endpointsClient: kubeClient.CoreV1(), } operatorClient.Informer().AddEventHandler(c.eventHandler()) endpointsInformer.Informer().AddEventHandler(c.eventHandler()) infrastructureInformer.Informer().AddEventHandler(c.eventHandler()) - podInformer.Informer().AddEventHandler(c.eventHandler()) + nodeInformer.Informer().AddEventHandler(c.eventHandler()) return c } @@ -112,48 +113,51 @@ func (c *HostEndpointsController) sync() error { } func (c *HostEndpointsController) syncHostEndpoints() error { + required := hostEndpointsAsset() + discoveryDomain, err := c.getEtcdDiscoveryDomain() if err != nil { return fmt.Errorf("unable to determine etcd discovery domain: %v", err) } - // list etcd member pods - pods, err := c.podLister.List(labels.Set{"k8s-app": "etcd"}.AsSelector()) + if required.Annotations == nil { + required.Annotations = map[string]string{} + } + required.Annotations["alpha.installer.openshift.io/dns-suffix"] = discoveryDomain - // get dns names of ready etcd member pods - var addresses []string - for _, pod := range pods { - var ready bool - for _, condition := range pod.Status.Conditions { - if condition.Type == corev1.PodReady { - ready = condition.Status == corev1.ConditionTrue + // create endpoint addresses for each node + nodes, err := c.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector()) + if err != nil { + fmt.Errorf("unable to list expected etcd member nodes: %v", err) + } + endpointAddresses := &required.Subsets[0].Addresses + for _, node := range nodes { + var nodeInternalIP string + for _, nodeAddress := range node.Status.Addresses { + if nodeAddress.Type == corev1.NodeInternalIP { + nodeInternalIP = nodeAddress.Address break } } - if ready { - dnsName, err := c.getEtcdDNSName(discoveryDomain, pod.Status.HostIP) - if err != nil { - return fmt.Errorf("unable to determine dns name for etcd member on node %s: %v", pod.Spec.NodeName, err) - } - addresses = append(addresses, dnsName) + if len(nodeInternalIP) == 0 { + return fmt.Errorf("unable to determine internal ip address for node %s", node.Name) + } + dnsName, err := c.getEtcdDNSName(discoveryDomain, nodeInternalIP) + if err != nil { + return fmt.Errorf("unable to determine etcd member dns name for node %s: %v", node.Name, err) } - } - - required := hostEndpointsAsset() - - if required.Annotations == nil { - required.Annotations = map[string]string{} - } - required.Annotations["alpha.installer.openshift.io/dns-suffix"] = discoveryDomain - sort.Strings(addresses) - for i, address := range addresses { - required.Subsets[0].Addresses = append(required.Subsets[0].Addresses, corev1.EndpointAddress{ - Hostname: strings.TrimSuffix(address, "."+discoveryDomain), - IP: net.IPv4(byte(192), byte(0), byte(2), byte(i+1)).String(), + *endpointAddresses = append(*endpointAddresses, corev1.EndpointAddress{ + IP: nodeInternalIP, + Hostname: strings.TrimSuffix(dnsName, "."+discoveryDomain), + NodeName: &node.Name, }) } + sort.Slice(*endpointAddresses, func(i, j int) bool { + return (*endpointAddresses)[i].Hostname < (*endpointAddresses)[j].Hostname + }) + // if etcd-bootstrap exists, keep it (at the end of the list) existing, err := c.endpointsLister.Endpoints("openshift-etcd").Get("host-etcd") if err != nil && !errors.IsNotFound(err) { @@ -162,14 +166,14 @@ func (c *HostEndpointsController) syncHostEndpoints() error { if !errors.IsNotFound(err) { for _, endpointAddress := range existing.Subsets[0].Addresses { if endpointAddress.Hostname == "etcd-bootstrap" { - required.Subsets[0].Addresses = append(required.Subsets[0].Addresses, *endpointAddress.DeepCopy()) + *endpointAddresses = append(*endpointAddresses, *endpointAddress.DeepCopy()) break } } } if len(required.Subsets[0].Addresses) == 0 { - return fmt.Errorf("no etcd member pods are ready") + return fmt.Errorf("no etcd member nodes are ready") } return c.applyEndpoints(required)