add all etcd nodes to host-etcd resource

pkg/operator/hostendpointscontroller/host_endpoints_controller.go

@@ -41,7 +41,7 @@ const (
 type HostEndpointsController struct {
 	operatorClient       v1helpers.OperatorClient
 	infrastructureLister configv1listers.InfrastructureLister
-	podLister            corev1listers.PodLister
+	nodeLister           corev1listers.NodeLister
 	endpointsLister      corev1listers.EndpointsLister
 	endpointsClient      corev1client.EndpointsGetter
 
@@ -54,32 +54,33 @@ func NewHostEndpointsController(
 	operatorClient v1helpers.OperatorClient,
 	eventRecorder events.Recorder,
 	kubeClient kubernetes.Interface,
-	kubeInformersForNamespaces operatorv1helpers.KubeInformersForNamespaces,
+	kubeInformers operatorv1helpers.KubeInformersForNamespaces,
 	infrastructureInformer configv1informers.InfrastructureInformer,
 ) *HostEndpointsController {
-	kubeInformersForTargetNamespace := kubeInformersForNamespaces.InformersFor("openshift-etcd")
+	kubeInformersForTargetNamespace := kubeInformers.InformersFor("openshift-etcd")
 	endpointsInformer := kubeInformersForTargetNamespace.Core().V1().Endpoints()
-	podInformer := kubeInformersForTargetNamespace.Core().V1().Pods()
+	kubeInformersForCluster := kubeInformers.InformersFor("")
+	nodeInformer := kubeInformersForCluster.Core().V1().Nodes()
 
 	c := &HostEndpointsController{
 		eventRecorder: eventRecorder.WithComponentSuffix("host-etcd-endpoints-controller"),
 		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "HostEndpointsController"),
 		cachesToSync: []cache.InformerSynced{
 			operatorClient.Informer().HasSynced,
 			endpointsInformer.Informer().HasSynced,
-			podInformer.Informer().HasSynced,
+			nodeInformer.Informer().HasSynced,
 			infrastructureInformer.Informer().HasSynced,
 		},
 		operatorClient:       operatorClient,
 		infrastructureLister: infrastructureInformer.Lister(),
-		podLister:            podInformer.Lister(),
+		nodeLister:           nodeInformer.Lister(),
 		endpointsLister:      endpointsInformer.Lister(),
 		endpointsClient:      kubeClient.CoreV1(),
 	}
 	operatorClient.Informer().AddEventHandler(c.eventHandler())
 	endpointsInformer.Informer().AddEventHandler(c.eventHandler())
 	infrastructureInformer.Informer().AddEventHandler(c.eventHandler())
-	podInformer.Informer().AddEventHandler(c.eventHandler())
+	nodeInformer.Informer().AddEventHandler(c.eventHandler())
 	return c
 }
 
@@ -112,52 +113,51 @@ func (c *HostEndpointsController) sync() error {
 }
 
 func (c *HostEndpointsController) syncHostEndpoints() error {
+	required := hostEndpointsAsset()
+
 	discoveryDomain, err := c.getEtcdDiscoveryDomain()
 	if err != nil {
 		return fmt.Errorf("unable to determine etcd discovery domain: %v", err)
 	}
 
-	// list etcd member pods
-	pods, err := c.podLister.List(labels.Set{"k8s-app": "etcd"}.AsSelector())
+	if required.Annotations == nil {
+		required.Annotations = map[string]string{}
+	}
+	required.Annotations["alpha.installer.openshift.io/dns-suffix"] = discoveryDomain
 
-	// get dns names of ready etcd member pods
-	var addresses []string
-	for _, pod := range pods {
-		var ready bool
-		for _, condition := range pod.Status.Conditions {
-			if condition.Type == corev1.PodReady {
-				ready = condition.Status == corev1.ConditionTrue
+	// create endpoint addresses for each node
+	nodes, err := c.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector())
+	if err != nil {
+		fmt.Errorf("unable to list expected etcd member nodes: %v", err)
+	}
+	endpointAddresses := &required.Subsets[0].Addresses
+	for _, node := range nodes {
+		var nodeInternalIP string
+		for _, nodeAddress := range node.Status.Addresses {
+			if nodeAddress.Type == corev1.NodeInternalIP {
+				nodeInternalIP = nodeAddress.Address
 				break
 			}
 		}
-		if ready {
-			dnsName, err := c.getEtcdDNSName(discoveryDomain, pod.Status.HostIP)
-			if err != nil {
-				return fmt.Errorf("unable to determine dns name for etcd member on node %s: %v", pod.Spec.NodeName, err)
-			}
-			addresses = append(addresses, dnsName)
+		if len(nodeInternalIP) == 0 {
+			return fmt.Errorf("unable to determine internal ip address for node %s", node.Name)
+		}
+		dnsName, err := c.getEtcdDNSName(discoveryDomain, nodeInternalIP)
+		if err != nil {
+			return fmt.Errorf("unable to determine etcd member dns name for node %s: %v", node.Name, err)
 		}
-	}
-
-	if len(addresses) == 0 {
-		return fmt.Errorf("no etcd member pods are ready")
-	}
-
-	required := hostEndpointsAsset()
-
-	if required.Annotations == nil {
-		required.Annotations = map[string]string{}
-	}
-	required.Annotations["alpha.installer.openshift.io/dns-suffix"] = discoveryDomain
 
-	sort.Strings(addresses)
-	for i, address := range addresses {
-		required.Subsets[0].Addresses = append(required.Subsets[0].Addresses, corev1.EndpointAddress{
-			Hostname: strings.TrimSuffix(address, "."+discoveryDomain),
-			IP:       net.IPv4(byte(192), byte(0), byte(2), byte(i+1)).String(),
+		*endpointAddresses = append(*endpointAddresses, corev1.EndpointAddress{
+			IP:       nodeInternalIP,
+			Hostname: strings.TrimSuffix(dnsName, "."+discoveryDomain),
+			NodeName: &node.Name,
 		})
 	}
 
+	sort.Slice(*endpointAddresses, func(i, j int) bool {
+		return (*endpointAddresses)[i].Hostname < (*endpointAddresses)[j].Hostname
+	})
+
 	// if etcd-bootstrap exists, keep it (at the end of the list)
 	existing, err := c.endpointsLister.Endpoints("openshift-etcd").Get("host-etcd")
 	if err != nil && !errors.IsNotFound(err) {
@@ -166,12 +166,16 @@ func (c *HostEndpointsController) syncHostEndpoints() error {
 	if !errors.IsNotFound(err) {
 		for _, endpointAddress := range existing.Subsets[0].Addresses {
 			if endpointAddress.Hostname == "etcd-bootstrap" {
-				required.Subsets[0].Addresses = append(required.Subsets[0].Addresses, *endpointAddress.DeepCopy())
+				*endpointAddresses = append(*endpointAddresses, *endpointAddress.DeepCopy())
 				break
 			}
 		}
 	}
 
+	if len(required.Subsets[0].Addresses) == 0 {
+		return fmt.Errorf("no etcd member nodes are ready")
+	}
+
 	return c.applyEndpoints(required)
 }