Datadog Integration

.changelog/3383.txt

@@ -0,0 +1,15 @@
+```release-note:feature
+breaking-change: users who previously had agent `enable_debug` set in `global.server.extraConfig`, should remove the extraConfig entry to avoid configuration conflicts.
+breaking-change: users who previously had agent `telemetry` stanza settings for `disable_hostname`, `enable_host_metrics`, `prefix_filter`, `dogstatsd_addr`, and/or `dogstatsd_tags`, should remove the extraConfig entry/entries to avoid configuration conflicts.
+helm: introduces `global.metrics.datadogIntegration` overrides to streamline consul-k8s datadog integration.
+helm: introduces `global.metrics.enableConsulAgentDebug` to expose agent [`enable_debug`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#enable_debug) configuration.
+helm: introduces `global.metrics.disableAgentHostName` to expose agent [`telemetry.disable_hostname`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#telemetry-disable_hostname) configuration.
+helm: introduces `global.metrics.enableHostMetrics` to expose agent [`telemetry.enable_host_metrics`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#telemetry-enable_host_metrics) configuration.
+helm: introduces `global.metrics.metricsPrefixFiltering` to expose agent [`telemetry.prefix_filter`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#telemetry-prefix_filter) configuration.
+helm: introduces `global.metrics.datadogIntegration.dogstatsd.dogstatsdAddr` to expose agent [`telemetry.dogstatsd_addr`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#telemetry-dogstatsd_addr) configuration.
+helm: introduces `global.metrics.datadogIntegration.dogstatsd.dogstatsdTags` to expose agent [`telemetry.dogstatsd_tags`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#telemetry-dogstatsd_tags) configuration.
+helm: introduces required `ad.datadoghq.com/` annotations and `tags.datadoghq.com/` labels for integration with [Datadog Autodiscovery](https://docs.datadoghq.com/integrations/consul/?tab=containerized) and [Datadog Unified Service Tagging](https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/?tab=kubernetes#serverless-environment) for Consul.
+helm: introduces automated unix domain socket hostPath mounting for containerized integration with datadog within consul-server statefulset.
+helm: introduces `global.metrics.datadogIntegration.datadogOpenTelemetryCollector` override options to allow OTLP metrics forwarding to Datadog Agent.
+control-plane: adds `server-acl-init` datadog agent token creation for datadog integration.
+```

charts/consul/templates/_helpers.tpl

@@ -425,10 +425,10 @@ Usage: {{ template "consul.validateTelemetryCollectorCloud" . }}
 */}}
 {{- define "consul.validateTelemetryCollectorCloud" -}}
 {{- if (and .Values.telemetryCollector.cloud.clientId.secretName (and (not .Values.global.cloud.clientSecret.secretName) (not .Values.telemetryCollector.cloud.clientSecret.secretName))) }}
-{{fail "When telemetryCollector.cloud.clientId.secretName is set, telemetryCollector.cloud.clientSecret.secretName must also be set."}}
+{{fail "When telemetryCollector.cloud.clientId.secretName is set, telemetryCollector.cloud.clientSecret.secretName must also be set." }}
 {{- end }}
 {{- if (and .Values.telemetryCollector.cloud.clientSecret.secretName (and (not .Values.global.cloud.clientId.secretName) (not .Values.telemetryCollector.cloud.clientId.secretName))) }}
-{{fail "When telemetryCollector.cloud.clientSecret.secretName is set, telemetryCollector.cloud.clientId.secretName must also be set."}}
+{{fail "When telemetryCollector.cloud.clientSecret.secretName is set, telemetryCollector.cloud.clientId.secretName must also be set." }}
 {{- end }}
 {{- end }}
 
@@ -515,4 +515,106 @@ Usage: {{ template "consul.validateResourceAPIs" . }}
 {{- if (and (mustHas "resource-apis" .Values.global.experiments) .Values.apiGateway.enabled ) }}
 {{fail "When the value global.experiments.resourceAPIs is set, apiGateway.enabled is currently unsupported."}}
 {{- end }}
-{{- end }}
+{{- end }}
+
+{{/*
+Validation for Consul Datadog Integration deployment:
+
+Fail if Datadog integration enabled and Consul server agent telemetry is not enabled.
+    - global.metrics.datadogIntegration.enabled=true
+    - global.metrics.enableAgentMetrics=false || global.metrics.enabled=false
+
+Fail if Consul OpenMetrics (Prometheus) and DogStatsD metrics are both enabled and configured.
+    - global.metrics.datadogIntegration.dogstatsd.enabled (scrapes `/v1/agent/metrics?format=prometheus` via the `use_prometheus_endpoint` option)
+    - global.metrics.datadogIntegration.openMetricsPrometheus.enabled (scrapes `/v1/agent/metrics?format=prometheus`)
+    - see https://docs.datadoghq.com/integrations/consul/?tab=host#host for recommendation to not have both
+
+Fail if Datadog OTLP forwarding is enabled and Consul Telemetry Collection is not enabled.
+    - global.metrics.datadogIntegration.datadogOpenTelemetryCollector.enabled=true
+    - telemetryCollector.enabled=false
+
+Fail if Consul Open Telemetry collector forwarding protocol is not one of either "http" or "grpc"
+    - global.metrics.datadogIntegration.datadogOpenTelemetryCollector.protocol!="http" || global.metrics.datadogIntegration.datadogOpenTelemetryCollector.protocol!="grpc"
+
+Usage: {{ template "consul.validateDatadogConfiguration" . }}
+
+*/}}
+
+{{- define "consul.validateDatadogConfiguration" -}}
+{{- if and .Values.global.metrics.datadogIntegration.enabled (or (not .Values.global.metrics.enableAgentMetrics) (not .Values.global.metrics.enabled) )}}
+{{fail "When enabling datadog metrics collection, the /v1/agent/metrics is required to be accessible, therefore global.metrics.enableAgentMetrics and global.metrics.enabled must be also be enabled."}}
+{{- end }}
+{{- if and .Values.global.metrics.datadogIntegration.dogstatsd.enabled .Values.global.metrics.datadogIntegration.openMetricsPrometheus.enabled }}
+{{fail "You must have one of DogStatsD (global.metrics.datadogIntegration.dogstatsd.enabled) or OpenMetrics (global.metrics.datadogIntegration.openMetricsPrometheus.enabled) enabled, not both as this is an unsupported configuration." }}
+{{- end }}
+{{- if and .Values.global.metrics.datadogIntegration.datadogOpenTelemetryCollector.enabled (not .Values.telemetryCollector.enabled) }}
+{{fail "Cannot enable Datadog OTLP metrics collection (global.metrics.datadogIntegration.datadogOpenTelemetryCollector.enabled) without consul-telemetry-collector. Ensure Consul OTLP collection is enabled (telemetryCollector.enabled) and configured." }}
+{{- end }}
+{{- if and .Values.global.metrics.datadogIntegration.datadogOpenTelemetryCollector.enabled (or (eq (.Values.global.metrics.datadogIntegration.datadogOpenTelemetryCollector.protocol | trimAll "\"" | quote) "http") (eq (.Values.global.metrics.datadogIntegration.datadogOpenTelemetryCollector.protocol | trimAll "\"" | quote) "grpc")) }}
+{{fail "Valid values for global.metrics.datadogIntegration.datadogOpenTelemetryCollector.protocol must be one of either \"http\" or \"grpc\"." }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Sets the dogstatsd_addr field of the agent configuration dependent on the
+socket transport type being used:
+  - "UDS" (Unix Domain Socket): prefixes "unix://" to URL and appends path to socket (i.e., unix:///var/run/datadog/dsd.socket)
+  - "UDP" (User Datagram Protocol): adds no prefix and appends dogstatsd port number to hostname/IP (i.e., 172.20.180.10:8125)
+- global.metrics.enableDatadogIntegration.dogstatsd configuration
+
+Usage: {{ template "consul.dogstatsdAaddressInfo" . }}
+*/}}
+
+{{- define "consul.dogstatsdAaddressInfo" -}}
+{{- if (and .Values.global.metrics.datadogIntegration.enabled .Values.global.metrics.datadogIntegration.dogstatsd.enabled) }}
+        "dogstatsd_addr": "{{- if eq .Values.global.metrics.datadogIntegration.dogstatsd.socketTransportType "UDS" }}unix://{{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdAddr }}{{- else }}{{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdAddr | trimAll "\"" }}:{{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdPort | toString }}{{- end }}",{{- end }}
+{{- end -}}
+
+{{/*
+Configures the metrics prefixing that's required to either allow or dissallow certaing RPC or gRPC server calls:
+
+Usage: {{ template "consul.metricsPrefixFiltering" . }}
+*/}}
+{{- define "consul.metricsPrefixFiltering" -}}
+{{- $allowList := .Values.global.metrics.metricsPrefixFiltering.allowList }}
+{{- $blockList := .Values.global.metrics.metricsPrefixFiltering.blockList }}
+{{- if and (not (empty $allowList)) (not (empty $blockList)) }}
+        "prefix_filter": [{{- range $index, $value := concat $allowList $blockList -}}
+    "{{- if (has $value $allowList) }}{{ printf "+%s" ($value | trimAll "\"") }}{{- else }}{{ printf "-%s" ($value | trimAll "\"") }}{{- end }}"{{- if lt $index (sub (len (concat $allowList $blockList)) 1) -}},{{- end -}}
+  {{- end -}}],
+{{- else if not (empty $allowList) }}
+        "prefix_filter": [{{- range $index, $value := $allowList -}}
+    "{{ printf "+%s" ($value | trimAll "\"") }}"{{- if lt $index (sub (len $allowList) 1) -}},{{- end -}}
+  {{- end -}}],
+{{- else if not (empty $blockList) }}
+        "prefix_filter": [{{- range $index, $value := $blockList -}}
+    "{{ printf "-%s" ($value | trimAll "\"") }}"{{- if lt $index (sub (len $blockList) 1) -}},{{- end -}}
+  {{- end -}}],
+{{- end }}
+{{- end -}}
+
+{{/*
+Retrieves the global consul/consul-enterprise version string for use with labels or tags.
+Requirements for valid labels:
+ -  a valid label must be an empty string or consist of
+    =>  alphanumeric characters
+    =>  '-', '_' or '.'
+    =>  must start and end with an alphanumeric character
+        (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is
+        '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')
+
+Usage: {{ template "consul.versionInfo" }}
+*/}}
+{{- define "consul.versionInfo" -}}
+{{- $imageVersion := regexSplit ":" .Values.global.image -1 }}
+{{- $versionInfo := printf "%s" (index $imageVersion 1 ) | trimSuffix "\"" }}
+{{- $sanitizedVersion := "" }}
+{{- $pattern := "^([A-Za-z0-9][-A-Za-z0-9_.]*[A-Za-z0-9])?$" }}
+{{- if not (regexMatch $pattern $versionInfo) -}}
+    {{- $sanitizedVersion = regexReplaceAll "[^A-Za-z0-9-_.]|sha256" $versionInfo "" }}
+    {{- $sanitizedVersion = printf "%s" (trimSuffix "-" (trimPrefix "-" $sanitizedVersion)) -}}
+{{- else }}
+    {{- $sanitizedVersion = $versionInfo }}
+{{- end -}}
+{{- printf "%s" $sanitizedVersion | quote }}
+{{- end -}}

charts/consul/templates/datadog-agent-role.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.global.metrics.datadogIntegration.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "consul.fullname" . }}-datadog-metrics-agent
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: datadog
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: agent
+{{- if (or (and .Values.global.openshift.enabled .Values.server.exposeGossipAndRPCPorts) .Values.global.enablePodSecurityPolicies) }}
+{{- if .Values.global.enablePodSecurityPolicies }}
+rules:
+  - apiGroups: ["policy"]
+    resources: ["podsecuritypolicies"]
+    resourceNames:
+      - {{ template "consul.fullname" . }}-datadog-metrics-agent
+    verbs:
+      - use
+{{- end }}
+{{- if (and .Values.global.openshift.enabled .Values.server.exposeGossipAndRPCPorts ) }}
+  - apiGroups: ["security.openshift.io"]
+    resources: ["securitycontextconstraints"]
+    resourceNames:
+      - {{ template "consul.fullname" . }}-datadog-metrics-agent
+    verbs:
+      - use
+{{- end }}
+{{- else}}
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "secrets" ]
+    resourceNames:
+      - {{ .Release.Namespace }}-datadog-agent-metrics-acl-token
+    verbs: [ "get", "watch", "list" ]
+{{- end }}
+{{- end }}

charts/consul/templates/datadog-agent-rolebinding.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.global.metrics.datadogIntegration.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: datadog-read-secrets
+  namespace: consul
+  labels:
+    component: agent
+subjects:
+  - kind: ServiceAccount
+    apiGroup: ""
+    name: datadog-agent
+    namespace: datadog
+  - kind: ServiceAccount
+    apiGroup: ""
+    name: datadog-cluster-agent
+    namespace: datadog
+roleRef:
+  kind: Role
+  name: {{ template "consul.fullname" . }}-datadog-metrics-agent
+  apiGroup: ""
+{{- end }}

charts/consul/templates/server-acl-init-job.yaml

@@ -2,6 +2,9 @@
 {{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
 {{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if and .Values.global.acls.createReplicationToken (not .Values.global.acls.manageSystemACLs) }}{{ fail "if global.acls.createReplicationToken is true, global.acls.manageSystemACLs must be true" }}{{ end -}}
+{{- if .Values.global.metrics.enableDatadogIntegration }}
+{{- if and .Values.global.metrics.enableDatadogIntegration.createAgentToken (not .Values.global.acls.manageSystemACLs) }}{{ fail "if global.metrics.enableDatadogIntegration.createAgentToken is true, global.acls.manageSystemACLs must be true" }}{{ end -}}
+{{- end }}
 {{- if .Values.global.bootstrapACLs }}{{ fail "global.bootstrapACLs was removed, use global.acls.manageSystemACLs instead" }}{{ end -}}
 {{- if .Values.global.acls.manageSystemACLs }}
 {{- if or (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.acls.bootstrapToken.secretKey))  (and .Values.global.acls.bootstrapToken.secretKey (not .Values.global.acls.bootstrapToken.secretName))}}{{ fail "both global.acls.bootstrapToken.secretKey and global.acls.bootstrapToken.secretName must be set if one of them is provided" }}{{ end -}}
@@ -273,6 +276,10 @@ spec:
             -create-enterprise-license-token=true \
             {{- end }}
 
+            {{- if (and (not .Values.global.metrics.datadogIntegration.dogstatsd.enabled) .Values.global.metrics.datadogIntegration.enabled .Values.global.acls.manageSystemACLs) }}
+            -create-dd-agent-token=true \
+            {{- end }}
+
             {{- if .Values.server.snapshotAgent.enabled }}
             -snapshot-agent=true \
             {{- end }}

charts/consul/templates/server-config-configmap.yaml

@@ -30,6 +30,7 @@ data:
       {{- if .Values.server.logLevel }}
       "log_level": "{{ .Values.server.logLevel | upper }}",
       {{- end }}
+      "enable_debug": {{ .Values.global.metrics.enableConsulAgentDebug }},
       "domain": "{{ .Values.global.domain }}",
       "limits": {
         "request_limits": {
@@ -187,7 +188,13 @@ data:
   telemetry-config.json: |-
     {
       "telemetry": {
-        "prometheus_retention_time": "{{ .Values.global.metrics.agentMetricsRetentionTime }}"
+        "prometheus_retention_time": "{{ .Values.global.metrics.agentMetricsRetentionTime }}",
+        "disable_hostname": {{ .Values.global.metrics.disableAgentHostName }},{{ template "consul.metricsPrefixFiltering" . }}
+        "enable_host_metrics": {{ .Values.global.metrics.enableHostMetrics }}{{- if .Values.global.metrics.datadogIntegration.dogstatsd.enabled }},{{ template "consul.dogstatsdAaddressInfo" . }}
+        {{- if .Values.global.metrics.datadogIntegration.dogstatsd.enabled }}
+        "dogstatsd_tags": {{ .Values.global.metrics.datadogIntegration.dogstatsd.dogstatsdTags | toJson }}
+        {{- end }}
+        {{- end }}
       }
     }
   {{- end }}