diff --git a/control-plane/connect-inject/common/common.go b/control-plane/connect-inject/common/common.go index b7946d78e0..1e7cf5415c 100644 --- a/control-plane/connect-inject/common/common.go +++ b/control-plane/connect-inject/common/common.go @@ -13,6 +13,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1" @@ -160,3 +161,19 @@ func GetPortProtocol(appProtocol *string) pbcatalog.Protocol { // If unrecognized or empty string, return default return pbcatalog.Protocol_PROTOCOL_UNSPECIFIED } + +// PortValueFromIntOrString returns the integer port value from the port that can be +// a named port, an integer string (e.g. "80"), or an integer. If the port is a named port, +// this function will attempt to find the value from the containers of the pod. +func PortValueFromIntOrString(pod corev1.Pod, port intstr.IntOrString) (uint32, error) { + if port.Type == intstr.Int { + return uint32(port.IntValue()), nil + } + + // Otherwise, find named port or try to parse the string as an int. + portVal, err := PortValue(pod, port.StrVal) + if err != nil { + return 0, err + } + return uint32(portVal), nil +} diff --git a/control-plane/connect-inject/constants/constants.go b/control-plane/connect-inject/constants/constants.go index 506654dfe1..f9dccd4f14 100644 --- a/control-plane/connect-inject/constants/constants.go +++ b/control-plane/connect-inject/constants/constants.go @@ -13,6 +13,9 @@ const ( // DefaultConsulPartition is the default Consul partition name. DefaultConsulPartition = "default" + // DefaultConsulPeer is the name used to refer to resources that are in the same cluster. + DefaultConsulPeer = "local" + // ProxyDefaultInboundPort is the default inbound port for the proxy. ProxyDefaultInboundPort = 20000 @@ -39,4 +42,12 @@ const ( // DefaultGracefulShutdownPath is the default path that consul-dataplane uses for graceful shutdown. DefaultGracefulShutdownPath = "/graceful_shutdown" + + // ConsulKubernetesCheckType is the type of health check in Consul for Kubernetes readiness status. + ConsulKubernetesCheckType = "kubernetes-readiness" + + // ConsulKubernetesCheckName is the name of health check in Consul for Kubernetes readiness status. + ConsulKubernetesCheckName = "Kubernetes Readiness Check" + + KubernetesSuccessReasonMsg = "Kubernetes health checks passing" ) diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go index 2a48f30bc9..136934a451 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go @@ -44,7 +44,6 @@ const ( terminatingGateway = "terminating-gateway" ingressGateway = "ingress-gateway" - kubernetesSuccessReasonMsg = "Kubernetes health checks passing" envoyPrometheusBindAddr = "envoy_prometheus_bind_addr" envoyTelemetryCollectorBindSocketDir = "envoy_telemetry_collector_bind_socket_dir" defaultNS = "default" @@ -57,12 +56,6 @@ const ( // This address does not need to be routable as this node is ephemeral, and we're only providing it because // Consul's API currently requires node address to be provided when registering a node. consulNodeAddress = "127.0.0.1" - - // consulKubernetesCheckType is the type of health check in Consul for Kubernetes readiness status. - consulKubernetesCheckType = "kubernetes-readiness" - - // consulKubernetesCheckName is the name of health check in Consul for Kubernetes readiness status. - consulKubernetesCheckName = "Kubernetes Readiness Check" ) type Controller struct { @@ -469,8 +462,8 @@ func (r *Controller) createServiceRegistrations(pod corev1.Pod, serviceEndpoints Service: service, Check: &api.AgentCheck{ CheckID: consulHealthCheckID(pod.Namespace, svcID), - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: healthStatus, ServiceID: svcID, Output: getHealthCheckStatusReason(healthStatus, pod.Name, pod.Namespace), @@ -664,8 +657,8 @@ func (r *Controller) createServiceRegistrations(pod corev1.Pod, serviceEndpoints Service: proxyService, Check: &api.AgentCheck{ CheckID: consulHealthCheckID(pod.Namespace, proxySvcID), - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: healthStatus, ServiceID: proxySvcID, Output: getHealthCheckStatusReason(healthStatus, pod.Name, pod.Namespace), @@ -807,8 +800,8 @@ func (r *Controller) createGatewayRegistrations(pod corev1.Pod, serviceEndpoints Service: service, Check: &api.AgentCheck{ CheckID: consulHealthCheckID(pod.Namespace, pod.Name), - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: healthStatus, ServiceID: pod.Name, Namespace: consulNS, @@ -900,7 +893,7 @@ func consulHealthCheckID(k8sNS string, serviceID string) string { // as well as pod name and namespace and returns the reason message. func getHealthCheckStatusReason(healthCheckStatus, podName, podNamespace string) string { if healthCheckStatus == api.HealthPassing { - return kubernetesSuccessReasonMsg + return constants.KubernetesSuccessReasonMsg } return fmt.Sprintf("Pod \"%s/%s\" is not ready", podNamespace, podName) diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go index 49150dec6a..9f9f54ba45 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go @@ -172,40 +172,40 @@ func TestReconcileCreateEndpointWithNamespaces(t *testing.T) { CheckID: fmt.Sprintf("%s/pod1-service-created", testCase.SourceKubeNS), ServiceName: "service-created", ServiceID: "pod1-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: testCase.ExpConsulNS, }, { CheckID: fmt.Sprintf("%s/pod1-service-created-sidecar-proxy", testCase.SourceKubeNS), ServiceName: "service-created-sidecar-proxy", ServiceID: "pod1-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: testCase.ExpConsulNS, }, { CheckID: fmt.Sprintf("%s/pod2-service-created", testCase.SourceKubeNS), ServiceName: "service-created", ServiceID: "pod2-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: testCase.ExpConsulNS, }, { CheckID: fmt.Sprintf("%s/pod2-service-created-sidecar-proxy", testCase.SourceKubeNS), ServiceName: "service-created-sidecar-proxy", ServiceID: "pod2-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: testCase.ExpConsulNS, }, }, @@ -446,30 +446,30 @@ func TestReconcileCreateGatewayWithNamespaces(t *testing.T) { CheckID: "default/mesh-gateway", ServiceName: "mesh-gateway", ServiceID: "mesh-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: "default", }, { CheckID: "default/terminating-gateway", ServiceName: "terminating-gateway", ServiceID: "terminating-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: testCase.ConsulNS, }, { CheckID: "default/ingress-gateway", ServiceName: "ingress-gateway", ServiceID: "ingress-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, Namespace: testCase.ConsulNS, }, }, diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go index 380db5a0b6..6a888d61f6 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go @@ -787,37 +787,37 @@ func TestReconcileCreateEndpoint_MultiportService(t *testing.T) { CheckID: "default/pod1-web", ServiceName: "web", ServiceID: "pod1-web", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-web-sidecar-proxy", ServiceName: "web-sidecar-proxy", ServiceID: "pod1-web-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-web-admin", ServiceName: "web-admin", ServiceID: "pod1-web-admin", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-web-admin-sidecar-proxy", ServiceName: "web-admin-sidecar-proxy", ServiceID: "pod1-web-admin-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1045,19 +1045,19 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/pod1-service-created", ServiceName: "service-created", ServiceID: "pod1-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-created-sidecar-proxy", ServiceName: "service-created-sidecar-proxy", ServiceID: "pod1-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1130,10 +1130,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/mesh-gateway", ServiceName: "mesh-gateway", ServiceID: "mesh-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1202,10 +1202,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/mesh-gateway", ServiceName: "mesh-gateway", ServiceID: "mesh-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, metricsEnabled: true, @@ -1275,10 +1275,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/mesh-gateway", ServiceName: "mesh-gateway", ServiceID: "mesh-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, metricsEnabled: true, @@ -1338,10 +1338,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/terminating-gateway", ServiceName: "terminating-gateway", ServiceID: "terminating-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1404,10 +1404,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/terminating-gateway", ServiceName: "terminating-gateway", ServiceID: "terminating-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1504,10 +1504,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/ingress-gateway", ServiceName: "ingress-gateway", ServiceID: "ingress-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1606,10 +1606,10 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/ingress-gateway", ServiceName: "ingress-gateway", ServiceID: "ingress-gateway", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1707,37 +1707,37 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/pod1-service-created", ServiceName: "service-created", ServiceID: "pod1-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-created-sidecar-proxy", ServiceName: "service-created-sidecar-proxy", ServiceID: "pod1-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod2-service-created", ServiceName: "service-created", ServiceID: "pod2-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod2-service-created-sidecar-proxy", ServiceName: "service-created-sidecar-proxy", ServiceID: "pod2-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -1848,28 +1848,28 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/pod1-service-created", ServiceName: "service-created", ServiceID: "pod1-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-created-sidecar-proxy", ServiceName: "service-created-sidecar-proxy", ServiceID: "pod1-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod2-service-created-sidecar-proxy", ServiceName: "service-created-sidecar-proxy", ServiceID: "pod2-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, expErr: "1 error occurred:\n\t* pods \"pod3\" not found\n\n", @@ -1991,19 +1991,19 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/pod1-different-consul-svc-name", ServiceName: "different-consul-svc-name", ServiceID: "pod1-different-consul-svc-name", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-different-consul-svc-name-sidecar-proxy", ServiceName: "different-consul-svc-name-sidecar-proxy", ServiceID: "pod1-different-consul-svc-name-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -2082,19 +2082,19 @@ func TestReconcileCreateEndpoint(t *testing.T) { CheckID: "default/pod1-service-created", ServiceName: "service-created", ServiceID: "pod1-service-created", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-created-sidecar-proxy", ServiceName: "service-created-sidecar-proxy", ServiceID: "pod1-service-created-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -2307,8 +2307,8 @@ func TestReconcileUpdateEndpoint(t *testing.T) { }, Check: &api.AgentCheck{ CheckID: "default/pod1-service-updated", - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: api.HealthCritical, ServiceID: "pod1-service-updated", ServiceName: "service-updated", @@ -2334,8 +2334,8 @@ func TestReconcileUpdateEndpoint(t *testing.T) { }, Check: &api.AgentCheck{ CheckID: "default/pod1-service-updated-sidecar-proxy", - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: api.HealthCritical, ServiceID: "pod1-service-updated-sidecar-proxy", ServiceName: "service-updated-sidecar-proxy", @@ -2359,19 +2359,19 @@ func TestReconcileUpdateEndpoint(t *testing.T) { CheckID: "default/pod1-service-updated", ServiceName: "service-updated", ServiceID: "pod1-service-updated", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-updated-sidecar-proxy", ServiceName: "service-updated-sidecar-proxy", ServiceID: "pod1-service-updated-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -2418,8 +2418,8 @@ func TestReconcileUpdateEndpoint(t *testing.T) { }, Check: &api.AgentCheck{ CheckID: "default/pod1-service-updated", - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: api.HealthPassing, ServiceName: "service-updated", ServiceID: "pod1-service-updated", @@ -2445,8 +2445,8 @@ func TestReconcileUpdateEndpoint(t *testing.T) { }, Check: &api.AgentCheck{ CheckID: "default/pod1-service-updated-sidecar-proxy", - Name: consulKubernetesCheckName, - Type: consulKubernetesCheckType, + Name: constants.ConsulKubernetesCheckName, + Type: constants.ConsulKubernetesCheckType, Status: api.HealthPassing, ServiceName: "service-updated-sidecar-proxy", ServiceID: "pod1-service-updated-sidecar-proxy", @@ -2470,19 +2470,19 @@ func TestReconcileUpdateEndpoint(t *testing.T) { CheckID: "default/pod1-service-updated", ServiceName: "service-updated", ServiceID: "pod1-service-updated", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthCritical, Output: "Pod \"default/pod1\" is not ready", - Type: consulKubernetesCheckType, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-updated-sidecar-proxy", ServiceName: "service-updated-sidecar-proxy", ServiceID: "pod1-service-updated-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthCritical, Output: "Pod \"default/pod1\" is not ready", - Type: consulKubernetesCheckType, + Type: constants.ConsulKubernetesCheckType, }, }, }, @@ -2756,37 +2756,37 @@ func TestReconcileUpdateEndpoint(t *testing.T) { CheckID: "default/pod1-service-updated", ServiceName: "service-updated", ServiceID: "pod1-service-updated", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod1-service-updated-sidecar-proxy", ServiceName: "service-updated-sidecar-proxy", ServiceID: "pod1-service-updated-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod2-service-updated", ServiceName: "service-updated", ServiceID: "pod2-service-updated", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, { CheckID: "default/pod2-service-updated-sidecar-proxy", ServiceName: "service-updated-sidecar-proxy", ServiceID: "pod2-service-updated-sidecar-proxy", - Name: consulKubernetesCheckName, + Name: constants.ConsulKubernetesCheckName, Status: api.HealthPassing, - Output: kubernetesSuccessReasonMsg, - Type: consulKubernetesCheckType, + Output: constants.KubernetesSuccessReasonMsg, + Type: constants.ConsulKubernetesCheckType, }, }, }, diff --git a/control-plane/connect-inject/controllers/pod/pod_controller.go b/control-plane/connect-inject/controllers/pod/pod_controller.go index f4e3f56053..d21e3d663a 100644 --- a/control-plane/connect-inject/controllers/pod/pod_controller.go +++ b/control-plane/connect-inject/controllers/pod/pod_controller.go @@ -5,15 +5,17 @@ package pod import ( "context" + "encoding/json" "fmt" "strconv" mapset "github.com/deckarep/golang-set" "github.com/go-logr/logr" pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1" "github.com/hashicorp/consul/proto-public/pbresource" "github.com/hashicorp/go-multierror" - "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/proto" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -28,8 +30,9 @@ import ( ) const ( - metaKeyManagedBy = "managed-by" - tokenMetaPodNameKey = "pod" + metaKeyManagedBy = "managed-by" + + DefaultTelemetryBindSocketDir = "/consul/mesh-inject" ) type Controller struct { @@ -63,6 +66,13 @@ type Controller struct { // TODO: EnableWANFederation + // EnableTransparentProxy controls whether transparent proxy should be enabled + // for all proxy service registrations. + EnableTransparentProxy bool + // TProxyOverwriteProbes controls whether the pods controller should expose pod's HTTP probes + // via Envoy proxy. + TProxyOverwriteProbes bool + // AuthMethod is the name of the Kubernetes Auth Method that // was used to login with Consul. The Endpoints controller // will delete any tokens associated with this auth method @@ -74,14 +84,13 @@ type Controller struct { EnableTelemetryCollector bool MetricsConfig metrics.Config - - Log logr.Logger + Log logr.Logger // ResourceClient is a gRPC client for the resource service. It is public for testing purposes ResourceClient pbresource.ResourceServiceClient } -// TODO(dans): logs, logs, logs +// TODO: logs, logs, logs // Reconcile reads the state of an Endpoints object for a Kubernetes Service and reconciles Consul services which // correspond to the Kubernetes Service. These events are driven by changes to the Pods backing the Kube service. @@ -110,26 +119,21 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // we need to remove the Workload from Consul. if k8serrors.IsNotFound(err) { + // Consul should also clean up the orphaned HealthStatus if err := r.deleteWorkload(ctx, req.NamespacedName); err != nil { errs = multierror.Append(errs, err) } + // TODO: clean up ACL Tokens + // TODO: delete explicit upstreams //if err := r.deleteUpstreams(ctx, pod); err != nil { // errs = multierror.Append(errs, err) //} - // TODO(dans): delete proxyConfiguration - //if err := r.deleteProxyConfiguration(ctx, pod); err != nil { - // errs = multierror.Append(errs, err) - //} - - // TODO: clean up ACL Tokens - - // TODO(dans): delete health status, since we don't have finalizers - //if err := r.deleteHealthStatus(ctx, pod); err != nil { - // errs = multierror.Append(errs, err) - //} + if err := r.deleteProxyConfiguration(ctx, req.NamespacedName); err != nil { + errs = multierror.Append(errs, err) + } return ctrl.Result{}, errs } else if err != nil { @@ -140,21 +144,22 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.Log.Info("retrieved", "name", pod.Name, "ns", pod.Namespace) if hasBeenInjected(pod) { - if err := r.writeWorkload(ctx, pod); err != nil { + if err := r.writeProxyConfiguration(ctx, pod); err != nil { errs = multierror.Append(errs, err) } - // TODO(dans): create proxyConfiguration + if err := r.writeWorkload(ctx, pod); err != nil { + errs = multierror.Append(errs, err) + } // TODO: create explicit upstreams //if err := r.writeUpstreams(ctx, pod); err != nil { // errs = multierror.Append(errs, err) //} - // TODO(dans): write health status - //if err := r.writeHealthStatus(ctx, pod); err != nil { - // errs = multierror.Append(errs, err) - //} + if err := r.writeHealthStatus(ctx, pod); err != nil { + errs = multierror.Append(errs, err) + } } return ctrl.Result{}, errs @@ -183,9 +188,14 @@ func (r *Controller) deleteWorkload(ctx context.Context, pod types.NamespacedNam return err } -//func (r *Controller) deleteHealthStatus(ctx context.Context, pod corev1.Pod) error { -// return nil -//} +func (r *Controller) deleteProxyConfiguration(ctx context.Context, pod types.NamespacedName) error { + req := &pbresource.DeleteRequest{ + Id: getProxyConfigurationID(pod.Name, r.getConsulNamespace(pod.Namespace), r.getPartition()), + } + + _, err := r.ResourceClient.Delete(ctx, req) + return err +} func (r *Controller) writeWorkload(ctx context.Context, pod corev1.Pod) error { @@ -208,35 +218,221 @@ func (r *Controller) writeWorkload(ctx context.Context, pod corev1.Pod) error { NodeName: common.ConsulNodeNameFromK8sNode(pod.Spec.NodeName), Ports: workloadPorts, } + data := common.ToProtoAny(workload) - // TODO(dans): replace with common.ToProtoAny when available - proto, err := anypb.New(workload) + req := &pbresource.WriteRequest{ + Resource: &pbresource.Resource{ + Id: getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()), + Metadata: metaFromPod(pod), + Data: data, + }, + } + _, err := r.ResourceClient.Write(ctx, req) + return err +} + +func (r *Controller) writeProxyConfiguration(ctx context.Context, pod corev1.Pod) error { + mode, err := r.getTproxyMode(ctx, pod) if err != nil { - return fmt.Errorf("could not serialize workload: %w", err) + return fmt.Errorf("failed to get transparent proxy mode: %w", err) } - // TODO: allow custom workload metadata - meta := map[string]string{ - constants.MetaKeyKubeNS: pod.Namespace, - metaKeyManagedBy: constants.ManagedByPodValue, + exposeConfig, err := r.getExposeConfig(pod) + if err != nil { + return fmt.Errorf("failed to get expose config: %w", err) } + bootstrapConfig, err := r.getBootstrapConfig(pod) + if err != nil { + return fmt.Errorf("failed to get bootstrap config: %w", err) + } + + if exposeConfig == nil && + bootstrapConfig == nil && + mode == pbmesh.ProxyMode_PROXY_MODE_DEFAULT { + // It's possible to remove interesting annotations and need to clear any existing config, + // but for now we treat pods as immutable configs owned by other managers. + return nil + } + + pc := &pbmesh.ProxyConfiguration{ + Workloads: &pbcatalog.WorkloadSelector{ + Names: []string{pod.GetName()}, + }, + DynamicConfig: &pbmesh.DynamicConfig{ + Mode: mode, + ExposeConfig: exposeConfig, + }, + BootstrapConfig: bootstrapConfig, + } + data := common.ToProtoAny(pc) + req := &pbresource.WriteRequest{ Resource: &pbresource.Resource{ - Id: getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()), - Metadata: meta, - Data: proto, + Id: getProxyConfigurationID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()), + Metadata: metaFromPod(pod), + Data: data, }, } _, err = r.ResourceClient.Write(ctx, req) return err } -//func (r *Controller) writeHealthStatus(pod corev1.Pod) error { -// return nil -//} +func (r *Controller) getTproxyMode(ctx context.Context, pod corev1.Pod) (pbmesh.ProxyMode, error) { + // A user can enable/disable tproxy for an entire namespace. + var ns corev1.Namespace + err := r.Client.Get(ctx, types.NamespacedName{Name: pod.GetNamespace()}, &ns) + if err != nil { + return pbmesh.ProxyMode_PROXY_MODE_DEFAULT, fmt.Errorf("could not get namespace info for %s: %w", pod.GetNamespace(), err) + } + + tproxyEnabled, err := common.TransparentProxyEnabled(ns, pod, r.EnableTransparentProxy) + if err != nil { + return pbmesh.ProxyMode_PROXY_MODE_DEFAULT, fmt.Errorf("could not determine if transparent proxy is enabled: %w", err) + } + + if tproxyEnabled { + return pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT, nil + } + return pbmesh.ProxyMode_PROXY_MODE_DEFAULT, nil +} + +func (r *Controller) getExposeConfig(pod corev1.Pod) (*pbmesh.ExposeConfig, error) { + // Expose k8s probes as Envoy listeners if needed. + overwriteProbes, err := common.ShouldOverwriteProbes(pod, r.TProxyOverwriteProbes) + if err != nil { + return nil, fmt.Errorf("could not determine if probes should be overwritten: %w", err) + } + + if !overwriteProbes { + return nil, nil + } + + var originalPod corev1.Pod + err = json.Unmarshal([]byte(pod.Annotations[constants.AnnotationOriginalPod]), &originalPod) + if err != nil { + return nil, fmt.Errorf("failed to get original pod spec: %w", err) + } -// TODO(dans): delete ACL token for workload + exposeConfig := &pbmesh.ExposeConfig{} + for _, mutatedContainer := range pod.Spec.Containers { + for _, originalContainer := range originalPod.Spec.Containers { + if originalContainer.Name == mutatedContainer.Name { + paths, err := getContainerExposePaths(originalPod, originalContainer, mutatedContainer) + if err != nil { + return nil, fmt.Errorf("error getting container expose path for %s: %w", originalContainer.Name, err) + } + + exposeConfig.ExposePaths = append(exposeConfig.ExposePaths, paths...) + } + } + } + + if len(exposeConfig.ExposePaths) == 0 { + return nil, nil + } + return exposeConfig, nil +} + +func getContainerExposePaths(originalPod corev1.Pod, originalContainer, mutatedContainer corev1.Container) ([]*pbmesh.ExposePath, error) { + var paths []*pbmesh.ExposePath + if mutatedContainer.LivenessProbe != nil && mutatedContainer.LivenessProbe.HTTPGet != nil { + originalLivenessPort, err := common.PortValueFromIntOrString(originalPod, originalContainer.LivenessProbe.HTTPGet.Port) + if err != nil { + return nil, err + } + + newPath := &pbmesh.ExposePath{ + ListenerPort: uint32(mutatedContainer.LivenessProbe.HTTPGet.Port.IntValue()), + LocalPathPort: originalLivenessPort, + Path: mutatedContainer.LivenessProbe.HTTPGet.Path, + } + paths = append(paths, newPath) + } + if mutatedContainer.ReadinessProbe != nil && mutatedContainer.ReadinessProbe.HTTPGet != nil { + originalReadinessPort, err := common.PortValueFromIntOrString(originalPod, originalContainer.ReadinessProbe.HTTPGet.Port) + if err != nil { + return nil, err + } + + newPath := &pbmesh.ExposePath{ + ListenerPort: uint32(mutatedContainer.ReadinessProbe.HTTPGet.Port.IntValue()), + LocalPathPort: originalReadinessPort, + Path: mutatedContainer.ReadinessProbe.HTTPGet.Path, + } + paths = append(paths, newPath) + } + if mutatedContainer.StartupProbe != nil && mutatedContainer.StartupProbe.HTTPGet != nil { + originalStartupPort, err := common.PortValueFromIntOrString(originalPod, originalContainer.StartupProbe.HTTPGet.Port) + if err != nil { + return nil, err + } + + newPath := &pbmesh.ExposePath{ + ListenerPort: uint32(mutatedContainer.StartupProbe.HTTPGet.Port.IntValue()), + LocalPathPort: originalStartupPort, + Path: mutatedContainer.StartupProbe.HTTPGet.Path, + } + paths = append(paths, newPath) + } + return paths, nil +} + +func (r *Controller) getBootstrapConfig(pod corev1.Pod) (*pbmesh.BootstrapConfig, error) { + bootstrap := &pbmesh.BootstrapConfig{} + + // If metrics are enabled, the BootstrapConfig should set envoy_prometheus_bind_addr to a listener on 0.0.0.0 on + // the PrometheusScrapePort that points to a metrics backend. The backend for this listener will be determined by + // the envoy bootstrapping command (consul connect envoy) or the consul-dataplane GetBoostrapParams rpc. + // If there is a merged metrics server, the backend would be that server. + // If we are not running the merged metrics server, the backend should just be the Envoy metrics endpoint. + enableMetrics, err := r.MetricsConfig.EnableMetrics(pod) + if err != nil { + return nil, fmt.Errorf("error determining if metrics are enabled: %w", err) + } + if enableMetrics { + prometheusScrapePort, err := r.MetricsConfig.PrometheusScrapePort(pod) + if err != nil { + return nil, err + } + prometheusScrapeListener := fmt.Sprintf("0.0.0.0:%s", prometheusScrapePort) + bootstrap.PrometheusBindAddr = prometheusScrapeListener + } + + if r.EnableTelemetryCollector { + bootstrap.TelemetryCollectorBindSocketDir = DefaultTelemetryBindSocketDir + } + + if proto.Equal(bootstrap, &pbmesh.BootstrapConfig{}) { + return nil, nil + } + return bootstrap, nil +} + +func (r *Controller) writeHealthStatus(ctx context.Context, pod corev1.Pod) error { + status := getHealthStatusFromPod(pod) + + hs := &pbcatalog.HealthStatus{ + Type: constants.ConsulKubernetesCheckType, + Status: status, + Description: constants.ConsulKubernetesCheckName, + Output: getHealthStatusReason(status, pod), + } + data := common.ToProtoAny(hs) + + req := &pbresource.WriteRequest{ + Resource: &pbresource.Resource{ + Id: getHealthStatusID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()), + Owner: getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()), + Metadata: metaFromPod(pod), + Data: data, + }, + } + _, err := r.ResourceClient.Write(ctx, req) + return err +} + +// TODO: delete ACL token for workload // deleteACLTokensForServiceInstance finds the ACL tokens that belongs to the service instance and deletes it from Consul. // It will only check for ACL tokens that have been created with the auth method this controller // has been configured with and will only delete tokens for the provided podName. @@ -245,6 +441,8 @@ func (r *Controller) writeWorkload(ctx context.Context, pod corev1.Pod) error { // TODO: add support for explicit upstreams //func (r *Controller) writeUpstreams(pod corev1.Pod) error +//func (r *Controller) deleteUpstreams(pod corev1.Pod) error + // consulNamespace returns the Consul destination namespace for a provided Kubernetes namespace // depending on Consul Namespaces being enabled and the value of namespace mirroring. func (r *Controller) getConsulNamespace(kubeNamespace string) string { @@ -321,6 +519,40 @@ func parseLocality(node corev1.Node) *pbcatalog.Locality { } } +func metaFromPod(pod corev1.Pod) map[string]string { + // TODO: allow custom workload metadata + return map[string]string{ + constants.MetaKeyKubeNS: pod.GetNamespace(), + metaKeyManagedBy: constants.ManagedByPodValue, + } +} + +// getHealthStatusFromPod checks the Pod for a "Ready" condition that is true. +// This is true when all the containers are ready, vs. "Running" on the PodPhase, +// which is true if any container is running. +func getHealthStatusFromPod(pod corev1.Pod) pbcatalog.Health { + if pod.Status.Conditions == nil { + return pbcatalog.Health_HEALTH_CRITICAL + } + + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return pbcatalog.Health_HEALTH_PASSING + } + } + return pbcatalog.Health_HEALTH_CRITICAL +} + +// getHealthStatusReason takes Consul's health check status (either passing or critical) +// and the pod to return a descriptive output for the HealthStatus Output. +func getHealthStatusReason(state pbcatalog.Health, pod corev1.Pod) string { + if state == pbcatalog.Health_HEALTH_PASSING { + return constants.KubernetesSuccessReasonMsg + } + + return fmt.Sprintf("Pod \"%s/%s\" is not ready", pod.GetNamespace(), pod.GetName()) +} + func getWorkloadID(name, namespace, partition string) *pbresource.ID { return &pbresource.ID{ Name: name, @@ -332,6 +564,48 @@ func getWorkloadID(name, namespace, partition string) *pbresource.ID { Tenancy: &pbresource.Tenancy{ Partition: partition, Namespace: namespace, + + // Because we are explicitly defining NS/partition, this will not default and must be explicit. + // At a future point, this will move out of the Tenancy block. + PeerName: constants.DefaultConsulPeer, + }, + } +} + +func getProxyConfigurationID(name, namespace, partition string) *pbresource.ID { + return &pbresource.ID{ + Name: name, + Type: &pbresource.Type{ + Group: "mesh", + GroupVersion: "v1alpha1", + Kind: "ProxyConfiguration", + }, + Tenancy: &pbresource.Tenancy{ + Partition: partition, + Namespace: namespace, + + // Because we are explicitly defining NS/partition, this will not default and must be explicit. + // At a future point, this will move out of the Tenancy block. + PeerName: constants.DefaultConsulPeer, + }, + } +} + +func getHealthStatusID(name, namespace, partition string) *pbresource.ID { + return &pbresource.ID{ + Name: name, + Type: &pbresource.Type{ + Group: "catalog", + GroupVersion: "v1alpha1", + Kind: "HealthStatus", + }, + Tenancy: &pbresource.Tenancy{ + Partition: partition, + Namespace: namespace, + + // Because we are explicitly defining NS/partition, this will not default and must be explicit. + // At a future point, this will move out of the Tenancy block. + PeerName: constants.DefaultConsulPeer, }, } } diff --git a/control-plane/connect-inject/controllers/pod/pod_controller_test.go b/control-plane/connect-inject/controllers/pod/pod_controller_test.go index 52a054581b..c4fb6cb5ab 100644 --- a/control-plane/connect-inject/controllers/pod/pod_controller_test.go +++ b/control-plane/connect-inject/controllers/pod/pod_controller_test.go @@ -5,22 +5,27 @@ package pod import ( "context" + "encoding/json" "testing" mapset "github.com/deckarep/golang-set" logrtest "github.com/go-logr/logr/testr" + "github.com/google/go-cmp/cmp" pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1" + pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1" "github.com/hashicorp/consul/proto-public/pbresource" "github.com/hashicorp/consul/sdk/testutil" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/anypb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -160,26 +165,14 @@ func TestWorkloadWrite(t *testing.T) { err = pc.writeWorkload(context.Background(), *tc.pod) require.NoError(t, err) - req := &pbresource.ReadRequest{Id: &pbresource.ID{ - Name: tc.pod.GetName(), - Type: &pbresource.Type{ - Group: "catalog", - GroupVersion: "v1alpha1", - Kind: "Workload", - }, - Tenancy: &pbresource.Tenancy{ - Partition: constants.DefaultConsulPartition, - Namespace: metav1.NamespaceDefault, - }, - }} + req := &pbresource.ReadRequest{ + Id: getWorkloadID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition), + } actualRes, err := resourceClient.Read(context.Background(), req) require.NoError(t, err) require.NotNil(t, actualRes) - require.Equal(t, tc.pod.GetName(), actualRes.GetResource().GetId().GetName()) - require.Equal(t, constants.DefaultConsulNS, actualRes.GetResource().GetId().GetTenancy().GetNamespace()) - require.Equal(t, constants.DefaultConsulPartition, actualRes.GetResource().GetId().GetTenancy().GetPartition()) - + requireEqualID(t, actualRes, tc.pod.GetName(), constants.DefaultConsulNS, constants.DefaultConsulPartition) require.NotNil(t, actualRes.GetResource().GetData()) actualWorkload := &pbcatalog.Workload{} @@ -191,29 +184,9 @@ func TestWorkloadWrite(t *testing.T) { testCases := []testCase{ { - name: "multi-port single-container", - pod: createPod("foo", "10.0.0.1", "foo", true, true), - expectedWorkload: &pbcatalog.Workload{ - Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "10.0.0.1", Ports: []string{"public", "admin", "mesh"}}, - }, - Ports: map[string]*pbcatalog.WorkloadPort{ - "public": { - Port: 80, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "admin": { - Port: 8080, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "mesh": { - Port: constants.ProxyDefaultInboundPort, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, - }, - }, - NodeName: consulNodeName, - Identity: "foo", - }, + name: "multi-port single-container", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + expectedWorkload: createWorkload(), }, { name: "multi-port multi-container", @@ -399,18 +372,9 @@ func TestWorkloadDelete(t *testing.T) { err = pc.deleteWorkload(context.Background(), reconcileReq) require.NoError(t, err) - readReq := &pbresource.ReadRequest{Id: &pbresource.ID{ - Name: tc.pod.GetName(), - Type: &pbresource.Type{ - Group: "catalog", - GroupVersion: "v1alpha1", - Kind: "Workload", - }, - Tenancy: &pbresource.Tenancy{ - Partition: constants.DefaultConsulPartition, - Namespace: metav1.NamespaceDefault, - }, - }} + readReq := &pbresource.ReadRequest{ + Id: getWorkloadID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition), + } _, err = resourceClient.Read(context.Background(), readReq) require.Error(t, err) s, ok := status.FromError(err) @@ -420,28 +384,312 @@ func TestWorkloadDelete(t *testing.T) { testCases := []testCase{ { - name: "basic pod delete", + name: "basic pod delete", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + existingWorkload: createWorkload(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + run(t, tc) + }) + } +} + +func TestHealthStatusWrite(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + pod *corev1.Pod + podModifier func(pod *corev1.Pod) + expectedHealthStatus *pbcatalog.HealthStatus + } + + run := func(t *testing.T, tc testCase) { + if tc.podModifier != nil { + tc.podModifier(tc.pod) + } + + fakeClient := fake.NewClientBuilder().WithRuntimeObjects().Build() + + // Create test consulServer server. + testClient := test.TestServerWithMockConnMgrWatcher(t, func(c *testutil.TestServerConfig) { + c.Experiments = []string{"resource-apis"} + }) + resourceClient, err := consul.NewResourceServiceClient(testClient.Watcher) + require.NoError(t, err) + + // Create the pod controller. + pc := &Controller{ + Client: fakeClient, + Log: logrtest.New(t), + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + ResourceClient: resourceClient, + } + + // The owner of a resource is validated, so create a dummy workload for the HealthStatus + workloadData, err := anypb.New(createWorkload()) + require.NoError(t, err) + + workloadID := getWorkloadID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition) + writeReq := &pbresource.WriteRequest{ + Resource: &pbresource.Resource{ + Id: workloadID, + Data: workloadData, + }, + } + _, err = resourceClient.Write(context.Background(), writeReq) + require.NoError(t, err) + + // Test writing the pod to a HealthStatus + err = pc.writeHealthStatus(context.Background(), *tc.pod) + require.NoError(t, err) + + req := &pbresource.ReadRequest{ + Id: getHealthStatusID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition), + } + actualRes, err := resourceClient.Read(context.Background(), req) + require.NoError(t, err) + require.NotNil(t, actualRes) + + requireEqualID(t, actualRes, tc.pod.GetName(), constants.DefaultConsulNS, constants.DefaultConsulPartition) + require.NotNil(t, actualRes.GetResource().GetData()) + + actualHealthStatus := &pbcatalog.HealthStatus{} + err = actualRes.GetResource().GetData().UnmarshalTo(actualHealthStatus) + require.NoError(t, err) + + require.True(t, proto.Equal(actualHealthStatus, tc.expectedHealthStatus)) + } + + testCases := []testCase{ + { + name: "ready pod", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + expectedHealthStatus: createPassingHealthStatus(), + }, + { + name: "not ready pod", + pod: createPod("foo", "10.0.0.1", "foo", true, false), + expectedHealthStatus: createCriticalHealthStatus(), + }, + { + name: "pod with no condition", pod: createPod("foo", "10.0.0.1", "foo", true, true), - existingWorkload: &pbcatalog.Workload{ - Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "10.0.0.1", Ports: []string{"public", "admin", "mesh"}}, + podModifier: func(pod *corev1.Pod) { + pod.Status.Conditions = []corev1.PodCondition{} + }, + expectedHealthStatus: createCriticalHealthStatus(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + run(t, tc) + }) + } +} + +func TestProxyConfigurationWrite(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + pod *corev1.Pod + podModifier func(pod *corev1.Pod) + expectedProxyConfiguration *pbmesh.ProxyConfiguration + + tproxy bool + overwriteProbes bool + metrics bool + telemetry bool + } + + run := func(t *testing.T, tc testCase) { + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: metav1.NamespaceDefault, + }} + + nsTproxy := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: "tproxy-party", + Labels: map[string]string{ + constants.KeyTransparentProxy: "true", + }, + }} + + if tc.podModifier != nil { + tc.podModifier(tc.pod) + } + + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(&ns, &nsTproxy).Build() + + // Create test consulServer server. + testClient := test.TestServerWithMockConnMgrWatcher(t, func(c *testutil.TestServerConfig) { + c.Experiments = []string{"resource-apis"} + }) + resourceClient, err := consul.NewResourceServiceClient(testClient.Watcher) + require.NoError(t, err) + + // Create the pod controller. + pc := &Controller{ + Client: fakeClient, + Log: logrtest.New(t), + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + EnableTransparentProxy: tc.tproxy, + TProxyOverwriteProbes: tc.overwriteProbes, + EnableTelemetryCollector: tc.telemetry, + ResourceClient: resourceClient, + } + + if tc.metrics { + pc.MetricsConfig = metrics.Config{ + DefaultEnableMetrics: true, + DefaultPrometheusScrapePort: "5678", + } + } + + // Test writing the pod to a HealthStatus + err = pc.writeProxyConfiguration(context.Background(), *tc.pod) + require.NoError(t, err) + + req := &pbresource.ReadRequest{ + Id: getProxyConfigurationID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition), + } + actualRes, err := resourceClient.Read(context.Background(), req) + + if tc.expectedProxyConfiguration == nil { + require.Error(t, err) + s, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.NotFound, s.Code()) + return + } + + require.NoError(t, err) + require.NotNil(t, actualRes) + + requireEqualID(t, actualRes, tc.pod.GetName(), constants.DefaultConsulNS, constants.DefaultConsulPartition) + require.NotNil(t, actualRes.GetResource().GetData()) + + actualProxyConfiguration := &pbmesh.ProxyConfiguration{} + err = actualRes.GetResource().GetData().UnmarshalTo(actualProxyConfiguration) + require.NoError(t, err) + + diff := cmp.Diff(actualProxyConfiguration, tc.expectedProxyConfiguration, protocmp.Transform()) + require.Equal(t, "", diff) + } + + testCases := []testCase{ + { + name: "no tproxy, no telemetry, no metrics, no probe overwrite", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + expectedProxyConfiguration: nil, + }, + { + name: "kitchen sink - globally enabled", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + podModifier: func(pod *corev1.Pod) { + addProbesAndOriginalPodAnnotation(pod) + }, + tproxy: true, + overwriteProbes: true, + metrics: true, + telemetry: true, + expectedProxyConfiguration: &pbmesh.ProxyConfiguration{ + Workloads: &pbcatalog.WorkloadSelector{ + Names: []string{"foo"}, }, - Ports: map[string]*pbcatalog.WorkloadPort{ - "public": { - Port: 80, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "admin": { - Port: 8080, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, + DynamicConfig: &pbmesh.DynamicConfig{ + Mode: pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT, + ExposeConfig: &pbmesh.ExposeConfig{ + ExposePaths: []*pbmesh.ExposePath{ + { + ListenerPort: 20400, + LocalPathPort: 2001, + Path: "/livez", + }, + { + ListenerPort: 20300, + LocalPathPort: 2000, + Path: "/readyz", + }, + { + ListenerPort: 20500, + LocalPathPort: 2002, + Path: "/startupz", + }, + }, }, - "mesh": { - Port: constants.ProxyDefaultInboundPort, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, + BootstrapConfig: &pbmesh.BootstrapConfig{ + PrometheusBindAddr: "0.0.0.0:5678", + TelemetryCollectorBindSocketDir: DefaultTelemetryBindSocketDir, + }, + }, + }, + { + name: "tproxy, metrics, and probe overwrite enabled on pod", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + podModifier: func(pod *corev1.Pod) { + pod.Annotations[constants.KeyTransparentProxy] = "true" + pod.Annotations[constants.AnnotationTransparentProxyOverwriteProbes] = "true" + pod.Annotations[constants.AnnotationEnableMetrics] = "true" + pod.Annotations[constants.AnnotationPrometheusScrapePort] = "21234" + + addProbesAndOriginalPodAnnotation(pod) + }, + expectedProxyConfiguration: &pbmesh.ProxyConfiguration{ + Workloads: &pbcatalog.WorkloadSelector{ + Names: []string{"foo"}, + }, + DynamicConfig: &pbmesh.DynamicConfig{ + Mode: pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT, + ExposeConfig: &pbmesh.ExposeConfig{ + ExposePaths: []*pbmesh.ExposePath{ + { + ListenerPort: 20400, + LocalPathPort: 2001, + Path: "/livez", + }, + { + ListenerPort: 20300, + LocalPathPort: 2000, + Path: "/readyz", + }, + { + ListenerPort: 20500, + LocalPathPort: 2002, + Path: "/startupz", + }, + }, }, }, - NodeName: consulNodeName, - Identity: "foo", + BootstrapConfig: &pbmesh.BootstrapConfig{ + PrometheusBindAddr: "0.0.0.0:21234", + }, + }, + }, + { + name: "tproxy enabled on namespace", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + podModifier: func(pod *corev1.Pod) { + pod.Namespace = "tproxy-party" + }, + expectedProxyConfiguration: &pbmesh.ProxyConfiguration{ + Workloads: &pbcatalog.WorkloadSelector{ + Names: []string{"foo"}, + }, + DynamicConfig: &pbmesh.DynamicConfig{ + Mode: pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT, + }, }, }, } @@ -453,11 +701,89 @@ func TestWorkloadDelete(t *testing.T) { } } -// TODO -// func TestHealthStatusWrite(t *testing.T) +func requireEqualID(t *testing.T, res *pbresource.ReadResponse, name string, ns string, partition string) { + require.Equal(t, name, res.GetResource().GetId().GetName()) + require.Equal(t, ns, res.GetResource().GetId().GetTenancy().GetNamespace()) + require.Equal(t, partition, res.GetResource().GetId().GetTenancy().GetPartition()) +} -// TODO -// func TestHealthStatusDelete(t *testing.T) +func TestProxyConfigurationDelete(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + pod *corev1.Pod + existingProxyConfiguration *pbmesh.ProxyConfiguration + } + + run := func(t *testing.T, tc testCase) { + fakeClient := fake.NewClientBuilder().WithRuntimeObjects().Build() + + // Create test consulServer server. + testClient := test.TestServerWithMockConnMgrWatcher(t, func(c *testutil.TestServerConfig) { + c.Experiments = []string{"resource-apis"} + }) + resourceClient, err := consul.NewResourceServiceClient(testClient.Watcher) + require.NoError(t, err) + + // Create the pod controller. + pc := &Controller{ + Client: fakeClient, + Log: logrtest.New(t), + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + ResourceClient: resourceClient, + } + + // Create the existing ProxyConfiguration + pcData, err := anypb.New(tc.existingProxyConfiguration) + require.NoError(t, err) + + pcID := getProxyConfigurationID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition) + writeReq := &pbresource.WriteRequest{ + Resource: &pbresource.Resource{ + Id: pcID, + Data: pcData, + }, + } + + _, err = resourceClient.Write(context.Background(), writeReq) + require.NoError(t, err) + test.ResourceHasPersisted(t, resourceClient, pcID) + + reconcileReq := types.NamespacedName{ + Namespace: metav1.NamespaceDefault, + Name: tc.pod.GetName(), + } + err = pc.deleteProxyConfiguration(context.Background(), reconcileReq) + require.NoError(t, err) + + readReq := &pbresource.ReadRequest{ + Id: getProxyConfigurationID(tc.pod.GetName(), metav1.NamespaceDefault, constants.DefaultConsulPartition), + } + _, err = resourceClient.Read(context.Background(), readReq) + require.Error(t, err) + s, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.NotFound, s.Code()) + } + + testCases := []testCase{ + { + name: "proxy configuration delete", + pod: createPod("foo", "10.0.0.1", "foo", true, true), + existingProxyConfiguration: createProxyConfiguration(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + run(t, tc) + }) + } +} // TODO // func TestUpstreamsWrite(t *testing.T) @@ -475,8 +801,7 @@ func TestReconcileCreatePod(t *testing.T) { t.Parallel() ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ - Name: metav1.NamespaceDefault, - Namespace: metav1.NamespaceDefault, + Name: metav1.NamespaceDefault, }} node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}} @@ -485,14 +810,16 @@ func TestReconcileCreatePod(t *testing.T) { podName string // This needs to be aligned with the pod created in `k8sObjects` namespace string // Defaults to metav1.NamespaceDefault if empty. Should be aligned with the ns in the pod - k8sObjects func() []runtime.Object // testing node is injected separately - expectedWorkload *pbcatalog.Workload - //expectedHealthStatus *pbcatalog.HealthStatus - //expectedProxyConfiguration *pbmesh.ProxyConfiguration + k8sObjects func() []runtime.Object // testing node is injected separately + expectedWorkload *pbcatalog.Workload + expectedHealthStatus *pbcatalog.HealthStatus + expectedProxyConfiguration *pbmesh.ProxyConfiguration //expectedUpstreams *pbmesh.Upstreams - metricsEnabled bool - telemetryEnabled bool + tproxy bool + overwriteProbes bool + metrics bool + telemetry bool expErr string } @@ -517,20 +844,22 @@ func TestReconcileCreatePod(t *testing.T) { // Create the pod controller. pc := &Controller{ - Client: fakeClient, - Log: logrtest.New(t), - ConsulClientConfig: testClient.Cfg, - ConsulServerConnMgr: testClient.Watcher, - AllowK8sNamespacesSet: mapset.NewSetWith("*"), - DenyK8sNamespacesSet: mapset.NewSetWith(), + Client: fakeClient, + Log: logrtest.New(t), + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + TProxyOverwriteProbes: tc.overwriteProbes, + EnableTransparentProxy: tc.tproxy, + EnableTelemetryCollector: tc.telemetry, } - if tc.metricsEnabled { + if tc.metrics { pc.MetricsConfig = metrics.Config{ - DefaultEnableMetrics: true, - EnableGatewayMetrics: true, + DefaultEnableMetrics: true, + DefaultPrometheusScrapePort: "1234", } } - pc.EnableTelemetryCollector = tc.telemetryEnabled namespace := tc.namespace if namespace == "" { @@ -553,9 +882,8 @@ func TestReconcileCreatePod(t *testing.T) { require.False(t, resp.Requeue) expectedWorkloadMatches(t, resourceClient, tc.podName, tc.expectedWorkload) - // TODO(dans): compare the following to expected values - // expectedHealthStatus - // expectedProxyConfiguration + expectedHealthStatusMatches(t, resourceClient, tc.podName, tc.expectedHealthStatus) + expectedProxyConfigurationMatches(t, resourceClient, tc.podName, tc.expectedProxyConfiguration) // expectedUpstreams } @@ -565,29 +893,17 @@ func TestReconcileCreatePod(t *testing.T) { podName: "foo", k8sObjects: func() []runtime.Object { pod := createPod("foo", "10.0.0.1", "foo", true, true) + addProbesAndOriginalPodAnnotation(pod) + return []runtime.Object{pod} }, - expectedWorkload: &pbcatalog.Workload{ - Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "10.0.0.1", Ports: []string{"public", "admin", "mesh"}}, - }, - Ports: map[string]*pbcatalog.WorkloadPort{ - "public": { - Port: 80, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "admin": { - Port: 8080, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "mesh": { - Port: constants.ProxyDefaultInboundPort, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, - }, - }, - NodeName: consulNodeName, - Identity: "foo", - }, + tproxy: true, + telemetry: true, + metrics: true, + overwriteProbes: true, + expectedWorkload: createWorkload(), + expectedHealthStatus: createPassingHealthStatus(), + expectedProxyConfiguration: createProxyConfiguration(), }, { name: "pod in ignored namespace", @@ -599,12 +915,39 @@ func TestReconcileCreatePod(t *testing.T) { return []runtime.Object{pod} }, }, - // TODO(dans): NotHealthyPod - // TODO(dans): tproxy + Metrics + Telemetry - // TODO: explicit upstreams - // TODO: at least one error cases + { + name: "unhealthy new pod", + podName: "foo", + k8sObjects: func() []runtime.Object { + pod := createPod("foo", "10.0.0.1", "foo", true, false) + return []runtime.Object{pod} + }, + expectedWorkload: createWorkload(), + expectedHealthStatus: createCriticalHealthStatus(), + }, + { + name: "return error - pod has no original pod annotation", + podName: "foo", + k8sObjects: func() []runtime.Object { + pod := createPod("foo", "10.0.0.1", "foo", true, false) + return []runtime.Object{pod} + }, + tproxy: true, + overwriteProbes: true, + expectedWorkload: createWorkload(), + expectedHealthStatus: createCriticalHealthStatus(), + expErr: "1 error occurred:\n\t* failed to get expose config: failed to get original pod spec: unexpected end of JSON input\n\n", + }, + { + name: "pod has not been injected", + podName: "foo", + k8sObjects: func() []runtime.Object { + pod := createPod("foo", "10.0.0.1", "foo", false, true) + return []runtime.Object{pod} + }, + }, // TODO: make sure multi-error accumulates errors - // TODO: injection annotation added + // TODO: explicit upstreams } for _, tc := range testCases { @@ -615,12 +958,14 @@ func TestReconcileCreatePod(t *testing.T) { } // TestReconcileUpdatePod test updating a Pod object when there is already matching resources in Consul. +// Updates are unlikely because of the immutable behaviors of pods as members of deployment/statefulset, +// but theoretically it is possible to update annotations and labels in-place. Most likely this will be +// from a change in health status. func TestReconcileUpdatePod(t *testing.T) { t.Parallel() ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ - Name: metav1.NamespaceDefault, - Namespace: metav1.NamespaceDefault, + Name: metav1.NamespaceDefault, }} node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}} @@ -631,18 +976,20 @@ func TestReconcileUpdatePod(t *testing.T) { k8sObjects func() []runtime.Object // testing node is injected separately - existingWorkload *pbcatalog.Workload - //existingHealthStatus *pbcatalog.HealthStatus - //existingProxyConfiguration *pbmesh.ProxyConfiguration + existingWorkload *pbcatalog.Workload + existingHealthStatus *pbcatalog.HealthStatus + existingProxyConfiguration *pbmesh.ProxyConfiguration //existingUpstreams *pbmesh.Upstreams - expectedWorkload *pbcatalog.Workload - //expectedHealthStatus *pbcatalog.HealthStatus - //expectedProxyConfiguration *pbmesh.ProxyConfiguration + expectedWorkload *pbcatalog.Workload + expectedHealthStatus *pbcatalog.HealthStatus + expectedProxyConfiguration *pbmesh.ProxyConfiguration //expectedUpstreams *pbmesh.Upstreams - metricsEnabled bool - telemetryEnabled bool + tproxy bool + overwriteProbes bool + metrics bool + telemetry bool expErr string } @@ -667,36 +1014,43 @@ func TestReconcileUpdatePod(t *testing.T) { // Create the pod controller. pc := &Controller{ - Client: fakeClient, - Log: logrtest.New(t), - ConsulClientConfig: testClient.Cfg, - ConsulServerConnMgr: testClient.Watcher, - AllowK8sNamespacesSet: mapset.NewSetWith("*"), - DenyK8sNamespacesSet: mapset.NewSetWith(), + Client: fakeClient, + Log: logrtest.New(t), + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + TProxyOverwriteProbes: tc.overwriteProbes, + EnableTransparentProxy: tc.tproxy, + EnableTelemetryCollector: tc.telemetry, } - if tc.metricsEnabled { + if tc.metrics { pc.MetricsConfig = metrics.Config{ - DefaultEnableMetrics: true, - EnableGatewayMetrics: true, + DefaultEnableMetrics: true, + DefaultPrometheusScrapePort: "1234", } } - pc.EnableTelemetryCollector = tc.telemetryEnabled namespace := tc.namespace if namespace == "" { namespace = metav1.NamespaceDefault } + workloadID := getWorkloadID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition) + loadResource(t, resourceClient, workloadID, tc.existingWorkload, nil) loadResource( t, resourceClient, - getWorkloadID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition), - tc.existingWorkload, - ) + getHealthStatusID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition), + tc.existingHealthStatus, + workloadID) + loadResource(t, + resourceClient, + getProxyConfigurationID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition), + tc.existingProxyConfiguration, + nil) - // TODO(dans): load the existing resources - // loadHealthStatus - // loadProxyConfiguration + // TODO: load the existing resources // loadUpstreams namespacedName := types.NamespacedName{ @@ -715,9 +1069,9 @@ func TestReconcileUpdatePod(t *testing.T) { require.False(t, resp.Requeue) expectedWorkloadMatches(t, resourceClient, tc.podName, tc.expectedWorkload) - // TODO(dans): compare the following to expected values - // expectedHealthStatus - // expectedProxyConfiguration + expectedHealthStatusMatches(t, resourceClient, tc.podName, tc.expectedHealthStatus) + expectedProxyConfigurationMatches(t, resourceClient, tc.podName, tc.expectedProxyConfiguration) + // TODO: compare the following to expected values // expectedUpstreams } @@ -729,6 +1083,7 @@ func TestReconcileUpdatePod(t *testing.T) { pod := createPod("foo", "10.0.0.1", "foo", true, true) return []runtime.Object{pod} }, + existingHealthStatus: createPassingHealthStatus(), existingWorkload: &pbcatalog.Workload{ Addresses: []*pbcatalog.WorkloadAddress{ {Host: "10.0.0.1", Ports: []string{"public", "mesh"}}, @@ -746,30 +1101,69 @@ func TestReconcileUpdatePod(t *testing.T) { NodeName: consulNodeName, Identity: "foo", }, - expectedWorkload: &pbcatalog.Workload{ - Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "10.0.0.1", Ports: []string{"public", "admin", "mesh"}}, + expectedWorkload: createWorkload(), + expectedHealthStatus: createPassingHealthStatus(), + }, + { + name: "pod healthy to unhealthy", + podName: "foo", + k8sObjects: func() []runtime.Object { + pod := createPod("foo", "10.0.0.1", "foo", true, false) + return []runtime.Object{pod} + }, + existingWorkload: createWorkload(), + existingHealthStatus: createPassingHealthStatus(), + expectedWorkload: createWorkload(), + expectedHealthStatus: createCriticalHealthStatus(), + }, + { + name: "add metrics, tproxy and probe overwrite to pod", + podName: "foo", + k8sObjects: func() []runtime.Object { + pod := createPod("foo", "10.0.0.1", "foo", true, true) + pod.Annotations[constants.KeyTransparentProxy] = "true" + pod.Annotations[constants.AnnotationTransparentProxyOverwriteProbes] = "true" + pod.Annotations[constants.AnnotationEnableMetrics] = "true" + pod.Annotations[constants.AnnotationPrometheusScrapePort] = "21234" + addProbesAndOriginalPodAnnotation(pod) + + return []runtime.Object{pod} + }, + existingWorkload: createWorkload(), + existingHealthStatus: createPassingHealthStatus(), + expectedWorkload: createWorkload(), + expectedHealthStatus: createPassingHealthStatus(), + expectedProxyConfiguration: &pbmesh.ProxyConfiguration{ + Workloads: &pbcatalog.WorkloadSelector{ + Names: []string{"foo"}, }, - Ports: map[string]*pbcatalog.WorkloadPort{ - "public": { - Port: 80, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "admin": { - Port: 8080, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "mesh": { - Port: constants.ProxyDefaultInboundPort, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + DynamicConfig: &pbmesh.DynamicConfig{ + Mode: pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT, + ExposeConfig: &pbmesh.ExposeConfig{ + ExposePaths: []*pbmesh.ExposePath{ + { + ListenerPort: 20400, + LocalPathPort: 2001, + Path: "/livez", + }, + { + ListenerPort: 20300, + LocalPathPort: 2000, + Path: "/readyz", + }, + { + ListenerPort: 20500, + LocalPathPort: 2002, + Path: "/startupz", + }, + }, }, }, - NodeName: consulNodeName, - Identity: "foo", + BootstrapConfig: &pbmesh.BootstrapConfig{ + PrometheusBindAddr: "0.0.0.0:21234", + }, }, }, - // TODO(dans): Pod Health to Unhealthy - // TODO(dans): update tproxy + Metrics + Telemetry // TODO: update explicit upstreams } @@ -797,19 +1191,17 @@ func TestReconcileDeletePod(t *testing.T) { k8sObjects func() []runtime.Object // testing node is injected separately - existingWorkload *pbcatalog.Workload - //existingHealthStatus *pbcatalog.HealthStatus - //existingProxyConfiguration *pbmesh.ProxyConfiguration + existingWorkload *pbcatalog.Workload + existingHealthStatus *pbcatalog.HealthStatus + existingProxyConfiguration *pbmesh.ProxyConfiguration //existingUpstreams *pbmesh.Upstreams - expectedWorkload *pbcatalog.Workload - //expectedHealthStatus *pbcatalog.HealthStatus - //expectedProxyConfiguration *pbmesh.ProxyConfiguration + expectedWorkload *pbcatalog.Workload + expectedHealthStatus *pbcatalog.HealthStatus + expectedProxyConfiguration *pbmesh.ProxyConfiguration //expectedUpstreams *pbmesh.Upstreams - aclsEnabled bool - metricsEnabled bool - telemetryEnabled bool + aclsEnabled bool expErr string } @@ -841,32 +1233,30 @@ func TestReconcileDeletePod(t *testing.T) { AllowK8sNamespacesSet: mapset.NewSetWith("*"), DenyK8sNamespacesSet: mapset.NewSetWith(), } - if tc.metricsEnabled { - pc.MetricsConfig = metrics.Config{ - DefaultEnableMetrics: true, - EnableGatewayMetrics: true, - } - } if tc.aclsEnabled { pc.AuthMethod = test.AuthMethod } - pc.EnableTelemetryCollector = tc.telemetryEnabled namespace := tc.namespace if namespace == "" { namespace = metav1.NamespaceDefault } + workloadID := getWorkloadID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition) + loadResource(t, resourceClient, workloadID, tc.existingWorkload, nil) loadResource( t, resourceClient, - getWorkloadID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition), - tc.existingWorkload, - ) - - // TODO(dans): load the existing resources - // loadHealthStatus - // loadProxyConfiguration + getHealthStatusID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition), + tc.existingHealthStatus, + workloadID) + loadResource( + t, + resourceClient, + getProxyConfigurationID(tc.podName, constants.DefaultConsulNS, constants.DefaultConsulPartition), + tc.existingProxyConfiguration, + nil) + // TODO: load the existing resources // loadUpstreams namespacedName := types.NamespacedName{ @@ -885,37 +1275,19 @@ func TestReconcileDeletePod(t *testing.T) { require.False(t, resp.Requeue) expectedWorkloadMatches(t, resourceClient, tc.podName, tc.expectedWorkload) - // TODO(dans): compare the following to expected values - // expectedHealthStatus - // expectedProxyConfiguration + expectedHealthStatusMatches(t, resourceClient, tc.podName, tc.expectedHealthStatus) + expectedProxyConfigurationMatches(t, resourceClient, tc.podName, tc.expectedProxyConfiguration) + // TODO: compare the following to expected values // expectedUpstreams } testCases := []testCase{ { - name: "vanilla delete pod", - podName: "foo", - existingWorkload: &pbcatalog.Workload{ - Addresses: []*pbcatalog.WorkloadAddress{ - {Host: "10.0.0.1", Ports: []string{"public", "admin", "mesh"}}, - }, - Ports: map[string]*pbcatalog.WorkloadPort{ - "public": { - Port: 80, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "admin": { - Port: 8080, - Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, - }, - "mesh": { - Port: constants.ProxyDefaultInboundPort, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, - }, - }, - NodeName: consulNodeName, - Identity: "foo", - }, + name: "vanilla delete pod", + podName: "foo", + existingWorkload: createWorkload(), + existingHealthStatus: createPassingHealthStatus(), + existingProxyConfiguration: createProxyConfiguration(), }, // TODO: enable ACLs and make sure they are deleted } @@ -927,6 +1299,7 @@ func TestReconcileDeletePod(t *testing.T) { } } +// createPod creates a multi-port pod as a base for tests. func createPod(name, ip string, identity string, inject bool, ready bool) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -957,6 +1330,30 @@ func createPod(name, ip string, identity string, inject bool, ready bool) *corev ContainerPort: 8080, }, }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(2000), + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/livez", + Port: intstr.FromInt(2001), + }, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/startupz", + Port: intstr.FromInt(2002), + }, + }, + }, }, }, NodeName: nodeName, @@ -986,6 +1383,87 @@ func createPod(name, ip string, identity string, inject bool, ready bool) *corev return pod } +// createWorkload creates a workload that matches the pod from createPod. +func createWorkload() *pbcatalog.Workload { + return &pbcatalog.Workload{ + Addresses: []*pbcatalog.WorkloadAddress{ + {Host: "10.0.0.1", Ports: []string{"public", "admin", "mesh"}}, + }, + Ports: map[string]*pbcatalog.WorkloadPort{ + "public": { + Port: 80, + Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, + }, + "admin": { + Port: 8080, + Protocol: pbcatalog.Protocol_PROTOCOL_UNSPECIFIED, + }, + "mesh": { + Port: constants.ProxyDefaultInboundPort, + Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, + }, + NodeName: consulNodeName, + Identity: "foo", + } +} + +// createPassingHealthStatus creates a passing HealthStatus that matches the pod from createPod. +func createPassingHealthStatus() *pbcatalog.HealthStatus { + return &pbcatalog.HealthStatus{ + Type: constants.ConsulKubernetesCheckType, + Status: pbcatalog.Health_HEALTH_PASSING, + Output: constants.KubernetesSuccessReasonMsg, + Description: constants.ConsulKubernetesCheckName, + } +} + +// createCriticalHealthStatus creates a failing HealthStatus that matches the pod from createPod. +func createCriticalHealthStatus() *pbcatalog.HealthStatus { + return &pbcatalog.HealthStatus{ + Type: constants.ConsulKubernetesCheckType, + Status: pbcatalog.Health_HEALTH_CRITICAL, + Output: "Pod \"default/foo\" is not ready", + Description: constants.ConsulKubernetesCheckName, + } +} + +// createProxyConfiguration creates a proxyConfiguration that matches the pod from createPod, +// assuming that metrics, telemetry, and overwrite probes are enabled separately. +func createProxyConfiguration() *pbmesh.ProxyConfiguration { + return &pbmesh.ProxyConfiguration{ + Workloads: &pbcatalog.WorkloadSelector{ + Names: []string{"foo"}, + }, + DynamicConfig: &pbmesh.DynamicConfig{ + Mode: pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT, + ExposeConfig: &pbmesh.ExposeConfig{ + ExposePaths: []*pbmesh.ExposePath{ + { + ListenerPort: 20400, + LocalPathPort: 2001, + Path: "/livez", + }, + { + ListenerPort: 20300, + LocalPathPort: 2000, + Path: "/readyz", + }, + { + ListenerPort: 20500, + LocalPathPort: 2002, + Path: "/startupz", + }, + }, + }, + }, + BootstrapConfig: &pbmesh.BootstrapConfig{ + PrometheusBindAddr: "0.0.0.0:1234", + TelemetryCollectorBindSocketDir: DefaultTelemetryBindSocketDir, + }, + } +} + func expectedWorkloadMatches(t *testing.T, client pbresource.ResourceServiceClient, name string, expectedWorkload *pbcatalog.Workload) { req := &pbresource.ReadRequest{Id: getWorkloadID(name, metav1.NamespaceDefault, constants.DefaultConsulPartition)} @@ -1015,8 +1493,66 @@ func expectedWorkloadMatches(t *testing.T, client pbresource.ResourceServiceClie require.True(t, proto.Equal(actualWorkload, expectedWorkload)) } -func loadResource(t *testing.T, client pbresource.ResourceServiceClient, id *pbresource.ID, proto proto.Message) { - if id == nil || proto == nil { +func expectedHealthStatusMatches(t *testing.T, client pbresource.ResourceServiceClient, name string, expectedHealthStatus *pbcatalog.HealthStatus) { + req := &pbresource.ReadRequest{Id: getHealthStatusID(name, metav1.NamespaceDefault, constants.DefaultConsulPartition)} + + res, err := client.Read(context.Background(), req) + + if expectedHealthStatus == nil { + require.Error(t, err) + s, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.NotFound, s.Code()) + return + } + + require.NoError(t, err) + require.NotNil(t, res) + + require.Equal(t, name, res.GetResource().GetId().GetName()) + require.Equal(t, constants.DefaultConsulNS, res.GetResource().GetId().GetTenancy().GetNamespace()) + require.Equal(t, constants.DefaultConsulPartition, res.GetResource().GetId().GetTenancy().GetPartition()) + + require.NotNil(t, res.GetResource().GetData()) + + actualHealthStatus := &pbcatalog.HealthStatus{} + err = res.GetResource().GetData().UnmarshalTo(actualHealthStatus) + require.NoError(t, err) + + require.True(t, proto.Equal(actualHealthStatus, expectedHealthStatus)) +} + +func expectedProxyConfigurationMatches(t *testing.T, client pbresource.ResourceServiceClient, name string, expectedProxyConfiguration *pbmesh.ProxyConfiguration) { + req := &pbresource.ReadRequest{Id: getProxyConfigurationID(name, metav1.NamespaceDefault, constants.DefaultConsulPartition)} + + res, err := client.Read(context.Background(), req) + + if expectedProxyConfiguration == nil { + require.Error(t, err) + s, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.NotFound, s.Code()) + return + } + + require.NoError(t, err) + require.NotNil(t, res) + + require.Equal(t, name, res.GetResource().GetId().GetName()) + require.Equal(t, constants.DefaultConsulNS, res.GetResource().GetId().GetTenancy().GetNamespace()) + require.Equal(t, constants.DefaultConsulPartition, res.GetResource().GetId().GetTenancy().GetPartition()) + + require.NotNil(t, res.GetResource().GetData()) + + actualProxyConfiguration := &pbmesh.ProxyConfiguration{} + err = res.GetResource().GetData().UnmarshalTo(actualProxyConfiguration) + require.NoError(t, err) + + require.True(t, proto.Equal(actualProxyConfiguration, expectedProxyConfiguration)) +} + +func loadResource(t *testing.T, client pbresource.ResourceServiceClient, id *pbresource.ID, proto proto.Message, owner *pbresource.ID) { + if id == nil || !proto.ProtoReflect().IsValid() { return } @@ -1024,8 +1560,9 @@ func loadResource(t *testing.T, client pbresource.ResourceServiceClient, id *pbr require.NoError(t, err) resource := &pbresource.Resource{ - Id: id, - Data: data, + Id: id, + Data: data, + Owner: owner, } req := &pbresource.WriteRequest{Resource: resource} @@ -1033,3 +1570,13 @@ func loadResource(t *testing.T, client pbresource.ResourceServiceClient, id *pbr require.NoError(t, err) test.ResourceHasPersisted(t, client, id) } + +func addProbesAndOriginalPodAnnotation(pod *corev1.Pod) { + podBytes, _ := json.Marshal(pod) + pod.Annotations[constants.AnnotationOriginalPod] = string(podBytes) + + // Fake the probe changes that would be added by the mesh webhook + pod.Spec.Containers[0].ReadinessProbe.HTTPGet.Port = intstr.FromInt(20300) + pod.Spec.Containers[0].LivenessProbe.HTTPGet.Port = intstr.FromInt(20400) + pod.Spec.Containers[0].StartupProbe.HTTPGet.Port = intstr.FromInt(20500) +} diff --git a/control-plane/consul/resource_client_test.go b/control-plane/consul/resource_client_test.go index 86d8cee029..3992e766fa 100644 --- a/control-plane/consul/resource_client_test.go +++ b/control-plane/consul/resource_client_test.go @@ -14,6 +14,8 @@ import ( "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" ) func Test_NewResourceServiceClient(t *testing.T) { @@ -100,8 +102,9 @@ func createWriteRequest(t *testing.T, name string) *pbresource.WriteRequest { Kind: "Workload", }, Tenancy: &pbresource.Tenancy{ - Partition: "default", - Namespace: "default", + Namespace: constants.DefaultConsulNS, + Partition: constants.DefaultConsulPartition, + PeerName: constants.DefaultConsulPeer, }, }, Data: proto, diff --git a/control-plane/go.mod b/control-plane/go.mod index 8581b62e35..2fafd2ed1c 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -15,7 +15,7 @@ require ( github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20230825213844-4ea04860c5ed github.com/hashicorp/consul-server-connection-manager v0.1.4 github.com/hashicorp/consul/api v1.10.1-0.20230906155245-56917eb4c968 - github.com/hashicorp/consul/proto-public v0.1.2-0.20230829221456-f8812eddf1ef + github.com/hashicorp/consul/proto-public v0.1.2-0.20230906155245-56917eb4c968 github.com/hashicorp/consul/sdk v0.14.1 github.com/hashicorp/go-bexpr v0.1.11 github.com/hashicorp/go-discover v0.0.0-20230519164032-214571b6a530 diff --git a/control-plane/go.sum b/control-plane/go.sum index 8e391110ee..435a090003 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -81,7 +81,6 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I= github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -128,7 +127,6 @@ github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TR github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -218,7 +216,6 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -260,22 +257,16 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20230825213844-4ea04860c5ed h1:eM9tGgSqAZbm4Ndkp35Dg8YROT0dNH3ghTYu5pcUIAc= github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20230825213844-4ea04860c5ed/go.mod h1:mwODEC+VTCA1LY/m2RUG4S2c5lNRvBcsvqaMJtMLLos= github.com/hashicorp/consul-server-connection-manager v0.1.4 h1:wrcSRV6WGXFBNpNbN6XsdoGgBOyso7ZbN5VaWPEX1jY= github.com/hashicorp/consul-server-connection-manager v0.1.4/go.mod h1:LMqHkALoLP0HUQKOG21xXYr0YPUayIQIHNTlmxG100E= -github.com/hashicorp/consul/api v1.10.1-0.20230825164720-ecdcde430924 h1:NPhzdwDho2r8pQv31oeGLlco7fnJ1i0WLYjtSXqWEck= -github.com/hashicorp/consul/api v1.10.1-0.20230825164720-ecdcde430924/go.mod h1:NZJGRFYruc/80wYowkPFCp1LbGmJC9L8izrwfyVx/Wg= github.com/hashicorp/consul/api v1.10.1-0.20230906155245-56917eb4c968 h1:lQ7QmlL0N4/ftLBex8n73Raji29o7EVssqCoeeczKac= github.com/hashicorp/consul/api v1.10.1-0.20230906155245-56917eb4c968/go.mod h1:NZJGRFYruc/80wYowkPFCp1LbGmJC9L8izrwfyVx/Wg= -github.com/hashicorp/consul/proto-public v0.1.2-0.20230829221456-f8812eddf1ef h1:Vt5NSnXc+RslTxXH2pz7dCb3hnE33CD2TrBP5AIQtMg= -github.com/hashicorp/consul/proto-public v0.1.2-0.20230829221456-f8812eddf1ef/go.mod h1:ENwzmloQTUPAYPu7nC1mli3VY0Ny9QNi/FSzJ+KlZD0= +github.com/hashicorp/consul/proto-public v0.1.2-0.20230906155245-56917eb4c968 h1:J6FLkHXcGd80fUbouFn3kklR3GGHVV0OCyjItyZS8h0= +github.com/hashicorp/consul/proto-public v0.1.2-0.20230906155245-56917eb4c968/go.mod h1:ENwzmloQTUPAYPu7nC1mli3VY0Ny9QNi/FSzJ+KlZD0= github.com/hashicorp/consul/sdk v0.4.1-0.20230825164720-ecdcde430924 h1:gkb6/ix0Tg1Th5FTjyq4QklLgrtIVQ/TUB0kbhIcPsY= github.com/hashicorp/consul/sdk v0.4.1-0.20230825164720-ecdcde430924/go.mod h1:vFt03juSzocLRFo59NkeQHHmQa6+g7oU0pfzdI1mUhg= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -432,7 +423,6 @@ github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQh github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -444,10 +434,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2 h1:BQ1HW7hr4IVovMwWg0E0PYcyW8CzqDcVmaew9cujU4s= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= @@ -467,7 +455,6 @@ github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otz github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/control-plane/subcommand/inject-connect/v2controllers.go b/control-plane/subcommand/inject-connect/v2controllers.go index df3830a799..75c01ea943 100644 --- a/control-plane/subcommand/inject-connect/v2controllers.go +++ b/control-plane/subcommand/inject-connect/v2controllers.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/endpointsv2" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/pod" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" ) @@ -31,14 +32,14 @@ func (c *Command) configureV2Controllers(ctx context.Context, mgr manager.Manage // DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, //} - //metricsConfig := metrics.Config{ - // DefaultEnableMetrics: c.flagDefaultEnableMetrics, - // EnableGatewayMetrics: c.flagEnableGatewayMetrics, - // DefaultEnableMetricsMerging: c.flagDefaultEnableMetricsMerging, - // DefaultMergedMetricsPort: c.flagDefaultMergedMetricsPort, - // DefaultPrometheusScrapePort: c.flagDefaultPrometheusScrapePort, - // DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath, - //} + metricsConfig := metrics.Config{ + DefaultEnableMetrics: c.flagDefaultEnableMetrics, + EnableGatewayMetrics: c.flagEnableGatewayMetrics, + DefaultEnableMetricsMerging: c.flagDefaultEnableMetricsMerging, + DefaultMergedMetricsPort: c.flagDefaultMergedMetricsPort, + DefaultPrometheusScrapePort: c.flagDefaultPrometheusScrapePort, + DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath, + } if err := (&pod.Controller{ Client: mgr.GetClient(), @@ -52,7 +53,11 @@ func (c *Command) configureV2Controllers(ctx context.Context, mgr manager.Manage EnableNSMirroring: c.flagEnableK8SNSMirroring, NSMirroringPrefix: c.flagK8SNSMirroringPrefix, ConsulPartition: c.consul.Partition, + EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, + TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, AuthMethod: c.flagACLAuthMethod, + MetricsConfig: metricsConfig, + EnableTelemetryCollector: c.flagEnableTelemetryCollector, Log: ctrl.Log.WithName("controller").WithName("pods"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", pod.Controller{})