diff --git a/.changelog/2571.txt b/.changelog/2571.txt new file mode 100644 index 0000000000..91b3f2943b --- /dev/null +++ b/.changelog/2571.txt @@ -0,0 +1,3 @@ +```release-note:bug +control-plane: fix bug in endpoints controller when deregistering services from consul when a node is deleted. +``` diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go index 800df9cc24..b971406f68 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go @@ -152,7 +152,6 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } err = r.Client.Get(ctx, req.NamespacedName, &serviceEndpoints) - // endpointPods holds a set of all pods this endpoints object is currently pointing to. // We use this later when we reconcile ACL tokens to decide whether an ACL token in Consul // is for a pod that no longer exists. @@ -176,7 +175,7 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // It is possible that the endpoints object has never been registered, in which case deregistration is a no-op. if isLabeledIgnore(serviceEndpoints.Labels) { // We always deregister the service to handle the case where a user has registered the service, then added the label later. - r.Log.Info("Ignoring endpoint labeled with `consul.hashicorp.com/service-ignore: \"true\"`", "name", req.Name, "namespace", req.Namespace) + r.Log.Info("ignoring endpoint labeled with `consul.hashicorp.com/service-ignore: \"true\"`", "name", req.Name, "namespace", req.Namespace) err = r.deregisterService(apiClient, req.Name, req.Namespace, nil) return ctrl.Result{}, err } @@ -869,14 +868,14 @@ func getHealthCheckStatusReason(healthCheckStatus, podName, podNamespace string) // them only if they are not in endpointsAddressesMap. If the map is nil, it will deregister all instances. If the map // has addresses, it will only deregister instances not in the map. func (r *Controller) deregisterService(apiClient *api.Client, k8sSvcName, k8sSvcNamespace string, endpointsAddressesMap map[string]bool) error { - // Get services matching metadata. - nodesWithSvcs, err := r.serviceInstancesForK8sNodes(apiClient, k8sSvcName, k8sSvcNamespace) + // Get services matching metadata from Consul + nodesWithSvcs, err := r.serviceInstancesForNodes(apiClient, k8sSvcName, k8sSvcNamespace) if err != nil { r.Log.Error(err, "failed to get service instances", "name", k8sSvcName) return err } - // Deregister each service instance that matches the metadata. + var errs error for _, nodeSvcs := range nodesWithSvcs { for _, svc := range nodeSvcs.Services { // We need to get services matching "k8s-service-name" and "k8s-namespace" metadata. @@ -887,42 +886,48 @@ func (r *Controller) deregisterService(apiClient *api.Client, k8sSvcName, k8sSvc if _, ok := endpointsAddressesMap[svc.Address]; !ok { // If the service address is not in the Endpoints addresses, deregister it. r.Log.Info("deregistering service from consul", "svc", svc.ID) - _, err = apiClient.Catalog().Deregister(&api.CatalogDeregistration{ + _, err := apiClient.Catalog().Deregister(&api.CatalogDeregistration{ Node: nodeSvcs.Node.Node, ServiceID: svc.ID, Namespace: svc.Namespace, }, nil) if err != nil { + // Do not exit right away as there might be other services that need to be deregistered. r.Log.Error(err, "failed to deregister service instance", "id", svc.ID) - return err + errs = multierror.Append(errs, err) + } else { + serviceDeregistered = true } - serviceDeregistered = true } } else { r.Log.Info("deregistering service from consul", "svc", svc.ID) - if _, err = apiClient.Catalog().Deregister(&api.CatalogDeregistration{ + _, err := apiClient.Catalog().Deregister(&api.CatalogDeregistration{ Node: nodeSvcs.Node.Node, ServiceID: svc.ID, Namespace: svc.Namespace, - }, nil); err != nil { + }, nil) + if err != nil { + // Do not exit right away as there might be other services that need to be deregistered. r.Log.Error(err, "failed to deregister service instance", "id", svc.ID) - return err + errs = multierror.Append(errs, err) + } else { + serviceDeregistered = true } - serviceDeregistered = true } if r.AuthMethod != "" && serviceDeregistered { r.Log.Info("reconciling ACL tokens for service", "svc", svc.Service) - err = r.deleteACLTokensForServiceInstance(apiClient, svc, k8sSvcNamespace, svc.Meta[constants.MetaKeyPodName]) + err := r.deleteACLTokensForServiceInstance(apiClient, svc, k8sSvcNamespace, svc.Meta[constants.MetaKeyPodName]) if err != nil { r.Log.Error(err, "failed to reconcile ACL tokens for service", "svc", svc.Service) - return err + errs = multierror.Append(errs, err) } } } } - return nil + return errs + } // deleteACLTokensForServiceInstance finds the ACL tokens that belongs to the service instance and deletes it from Consul. @@ -1042,21 +1047,32 @@ func getTokenMetaFromDescription(description string) (map[string]string, error) return tokenMeta, nil } -func (r *Controller) serviceInstancesForK8sNodes(apiClient *api.Client, k8sServiceName, k8sServiceNamespace string) ([]*api.CatalogNodeServiceList, error) { +func (r *Controller) serviceInstancesForNodes(apiClient *api.Client, k8sServiceName, k8sServiceNamespace string) ([]*api.CatalogNodeServiceList, error) { var serviceList []*api.CatalogNodeServiceList - // Get a list of k8s nodes. - var nodeList corev1.NodeList - err := r.Client.List(r.Context, &nodeList) + + // The nodelist may have changed between this point and when the event was raised + // For example, if a pod is evicted because a node has been deleted, there is no guarantee that that node will show up here + // query consul catalog for a list of nodes supporting this service + // quite a lot of results as synthetic nodes are never deregistered. + var nodes []*api.Node + filter := fmt.Sprintf(`Meta[%q] == %q `, "synthetic-node", "true") + nodes, _, err := apiClient.Catalog().Nodes(&api.QueryOptions{Filter: filter, Namespace: namespaces.WildcardNamespace}) if err != nil { return nil, err } - for _, node := range nodeList.Items { + + var errs error + for _, node := range nodes { var nodeServices *api.CatalogNodeServiceList - nodeServices, err = r.serviceInstancesForK8SServiceNameAndNamespace(apiClient, k8sServiceName, k8sServiceNamespace, common.ConsulNodeNameFromK8sNode(node.Name)) - serviceList = append(serviceList, nodeServices) + nodeServices, err := r.serviceInstancesForK8SServiceNameAndNamespace(apiClient, k8sServiceName, k8sServiceNamespace, node.Node) + if err != nil { + errs = multierror.Append(errs, err) + } else { + serviceList = append(serviceList, nodeServices) + } } - return serviceList, err + return serviceList, errs } // serviceInstancesForK8SServiceNameAndNamespace calls Consul's ServicesWithFilter to get the list diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go index 7b5cf92c2f..e6694d6718 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go @@ -890,6 +890,9 @@ func TestReconcileCreateEndpoint_MultiportService(t *testing.T) { catalogRegistration := &api.CatalogRegistration{ Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: svc, } _, err := consulClient.Catalog().Register(catalogRegistration, nil) @@ -2198,6 +2201,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -2217,6 +2223,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -2303,6 +2312,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -2322,6 +2334,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: "127.0.0.1", + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -2408,6 +2423,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -2425,6 +2443,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -2490,6 +2511,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-different-consul-svc-name", Service: "different-consul-svc-name", @@ -2507,6 +2531,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-different-consul-svc-name-sidecar-proxy", @@ -2580,6 +2607,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -2591,6 +2621,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -2694,6 +2727,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -2705,6 +2741,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -2721,6 +2760,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod2-service-updated", Service: "service-updated", @@ -2732,6 +2774,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod2-service-updated-sidecar-proxy", @@ -2791,6 +2836,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-different-consul-svc-name", Service: "different-consul-svc-name", @@ -2802,6 +2850,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-different-consul-svc-name-sidecar-proxy", @@ -2818,6 +2869,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod2-different-consul-svc-name", Service: "different-consul-svc-name", @@ -2829,6 +2883,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod2-different-consul-svc-name-sidecar-proxy", @@ -2874,6 +2931,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -2885,6 +2945,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -2901,6 +2964,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod2-service-updated", Service: "service-updated", @@ -2912,6 +2978,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod2-service-updated-sidecar-proxy", @@ -2947,6 +3016,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-different-consul-svc-name", Service: "different-consul-svc-name", @@ -2958,6 +3030,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-different-consul-svc-name-sidecar-proxy", @@ -2974,6 +3049,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod2-different-consul-svc-name", Service: "different-consul-svc-name", @@ -2985,6 +3063,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod2-different-consul-svc-name-sidecar-proxy", @@ -3033,6 +3114,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -3050,6 +3134,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -3129,6 +3216,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -3146,6 +3236,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -3168,6 +3261,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod2-service-updated", Service: "service-updated", @@ -3185,6 +3281,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod2-service-updated-sidecar-proxy", @@ -3268,6 +3367,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-service-updated", Service: "service-updated", @@ -3285,6 +3387,9 @@ func TestReconcileUpdateEndpoint(t *testing.T) { { Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ Kind: api.ServiceKindConnectProxy, ID: "pod1-service-updated-sidecar-proxy", @@ -3975,6 +4080,9 @@ func TestReconcileDeleteEndpoint(t *testing.T) { serviceRegistration := &api.CatalogRegistration{ Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: svc, } _, err := consulClient.Catalog().Register(serviceRegistration, nil) @@ -4121,6 +4229,9 @@ func TestReconcileIgnoresServiceIgnoreLabel(t *testing.T) { serviceRegistration := &api.CatalogRegistration{ Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-" + svcName, Service: svcName, @@ -4244,6 +4355,9 @@ func TestReconcile_podSpecifiesExplicitService(t *testing.T) { _, err := consulClient.Catalog().Register(&api.CatalogRegistration{ Node: consulNodeName, Address: consulNodeAddress, + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: &api.AgentService{ ID: "pod1-" + svcName, Service: svcName, @@ -4401,6 +4515,9 @@ func TestServiceInstancesForK8SServiceNameAndNamespace(t *testing.T) { catalogRegistration := &api.CatalogRegistration{ Node: consulNodeName, Address: "127.0.0.1", + NodeMeta: map[string]string{ + metaKeySyntheticNode: "true", + }, Service: svc, } _, err = consulClient.Catalog().Register(catalogRegistration, nil)