diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 849f7ae2d53..b8e6c065fda 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -30,6 +30,8 @@ changelog/fragments/ /internal/pkg/otel/samples @elastic/ingest-otel-data @elastic/ingest-docs /internal/pkg/otel/components.yml @elastic/ingest-otel-leads /internal/pkg/composable/providers/kubernetes @elastic/elastic-agent-control-plane -/internal/pkg/otel/samples/darwin/autoops_es*.yml @elastic/opex -/internal/pkg/otel/samples/linux/autoops_es*.yml @elastic/opex -/internal/pkg/otel/samples/windows/autoops_es*.yml @elastic/opex +/internal/pkg/otel/samples/darwin/autoops_es.yml @elastic/opex +/internal/pkg/otel/samples/linux/autoops_es.yml @elastic/opex +/internal/pkg/otel/samples/windows/autoops_es.yml @elastic/opex +/deploy/helm/elastic-agent/templates/integrations/_auto_ops @elastic.opex + diff --git a/changelog/fragments/1762856188-added-opex-to-elastic-agent-helm-chart.yaml b/changelog/fragments/1762856188-added-opex-to-elastic-agent-helm-chart.yaml new file mode 100644 index 00000000000..dc3908018fb --- /dev/null +++ b/changelog/fragments/1762856188-added-opex-to-elastic-agent-helm-chart.yaml @@ -0,0 +1,45 @@ +# REQUIRED +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# REQUIRED for all kinds +# Change summary; a 80ish characters long description of the change. +summary: added opex to elastic-agent helm chart, This change will add the Opex-CCM support to the offical elastic-agent helm chart deployment. + +# REQUIRED for breaking-change, deprecation, known-issue +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# description: + +# REQUIRED for breaking-change, deprecation, known-issue +# impact: + +# REQUIRED for breaking-change, deprecation, known-issue +# action: + +# REQUIRED for all kinds +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# AUTOMATED +# OPTIONAL to manually add other PR URLs +# PR URL: A link the PR that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/9363 + +# AUTOMATED +# OPTIONAL to manually add other issue URLs +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +# issue: https://github.com/owner/repo/1234 diff --git a/deploy/helm/elastic-agent/examples/autoops-agent/README.md b/deploy/helm/elastic-agent/examples/autoops-agent/README.md new file mode 100644 index 00000000000..23c09a33c29 --- /dev/null +++ b/deploy/helm/elastic-agent/examples/autoops-agent/README.md @@ -0,0 +1,35 @@ +# Example: Managed by Opex Agent + +In this example we deploy an Elastic AutoOps Agent that is managed by Opex team . + + +## Run: + +There are 2 kinds of installations, related to auth method + +### for API_KEY installation + +```console +helm install ./deploy/helm/elastic-agent -n kube-system \ + --set kube-state-metrics.enabled=false \ + --set kubernetes.enabled=false \ + --set autoOps.enabled=true \ + --set-string autoOps.autoops_token="tok-123" \ + --set-string autoOps.autoops_otel_url="https://otel.example.com:4318" \ + --set-string autoOps.autoops_temp_resource_id="res-abc" \ + --set-string autoOps.es_api_key="API_KEY_123" +``` + +### for Username:password (Basic auth) installation + +```console +helm install ./deploy/helm/elastic-agent -n kube-system \ + --set kube-state-metrics.enabled=false \ + --set kubernetes.enabled=false \ + --set autoOps.enabled=true \ + --set-string autoOps.autoops_token="tok-123" \ + --set-string autoOps.autoops_otel_url="https://otel.example.com:4318" \ + --set-string autoOps.autoops_temp_resource_id="res-abc" \ + --set-string autoOps.es_username="elastic" \ + --set-string autoOps.es_username="es_pass" +``` diff --git a/deploy/helm/elastic-agent/examples/autoops-agent/autoops-values.yaml b/deploy/helm/elastic-agent/examples/autoops-agent/autoops-values.yaml new file mode 100644 index 00000000000..b64c0eb213d --- /dev/null +++ b/deploy/helm/elastic-agent/examples/autoops-agent/autoops-values.yaml @@ -0,0 +1,15 @@ +# Turn off everything except the AutoOps preset +kubernetes: + enabled: false +kube-state-metrics: + enabled: false + +autoOps: + enabled: true + autoops_token: "REPLACE_ME_TOKEN" + autoops_otel_url: "https://otel.example.com:4318" + autoops_temp_resource_id: "REPLACE_ME_RESOURCE_ID" + es_api_key: "REPLACE_ME_API_KEY" + elastic_cloud_connected_mode_api_key: "CCM-API-KEY" + elastic_cloud_connected_mode_api_url: "CCM-API-URL" + autoops_es_url: "https://ccm-test-eu-west-1.es.us-east-2.aws.elastic-cloud.com" diff --git a/deploy/helm/elastic-agent/examples/autoops-agent/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/autoops-agent/rendered/manifest.yaml new file mode 100644 index 00000000000..548fccb18ed --- /dev/null +++ b/deploy/helm/elastic-agent/examples/autoops-agent/rendered/manifest.yaml @@ -0,0 +1,194 @@ +--- +# Source: elastic-agent/templates/agent/service-account.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: agent-autoops-example + namespace: "default" + labels: + helm.sh/chart: elastic-agent-9.3.0-beta + app.kubernetes.io/name: elastic-agent + app.kubernetes.io/instance: example + app.kubernetes.io/version: 9.3.0 +--- +# Source: elastic-agent/templates/agent/k8s/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: agent-autoops-example + namespace: "default" + labels: + helm.sh/chart: elastic-agent-9.3.0-beta + app.kubernetes.io/name: elastic-agent + app.kubernetes.io/instance: example + app.kubernetes.io/version: 9.3.0 +stringData: + + agent.yml: |- + exporters: + otlphttp: + endpoint: ${env:AUTOOPS_OTEL_URL} + headers: + Authorization: AutoOpsToken ${env:AUTOOPS_TOKEN} + receivers: + metricbeatreceiver: + metricbeat: + modules: + - hosts: ${env:AUTOOPS_ES_URL} + metricsets: + - cat_shards + - cluster_health + - cluster_settings + - license + - node_stats + - tasks_management + module: autoops_es + period: 10s + - hosts: ${env:AUTOOPS_ES_URL} + metricsets: + - cat_template + - component_template + - index_template + module: autoops_es + period: 24h + output: + otelconsumer: null + processors: + - add_fields: + fields: + temp_resource_id: ${env:AUTOOPS_TEMP_RESOURCE_ID} + token: ${env:AUTOOPS_TOKEN} + target: autoops_es + telemetry_types: + - logs + service: + pipelines: + logs: + exporters: + - otlphttp + receivers: + - metricbeatreceiver + telemetry: + logs: + encoding: json +--- +# Source: elastic-agent/templates/integrations/_auto_ops/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: agent-autoops-example-autoops + namespace: "default" + labels: + helm.sh/chart: elastic-agent-9.3.0-beta + app.kubernetes.io/name: elastic-agent + app.kubernetes.io/instance: example + app.kubernetes.io/version: 9.3.0 +stringData: + autoops-token: "REPLACE_ME_TOKEN" + autoops-es-url: "https://ccm-test-eu-west-1.es.us-east-2.aws.elastic-cloud.com" + temp-resource-id: "REPLACE_ME_RESOURCE_ID" + otel-url: "https://otel.example.com:4318" + es-api-key: "REPLACE_ME_API_KEY" + cloud-connected-mode-api-key: "CCM-API-KEY" + cloud-connected-mode-api-url: "CCM-API-URL" +--- +# Source: elastic-agent/templates/agent/k8s/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-autoops-example + namespace: "default" + labels: + helm.sh/chart: elastic-agent-9.3.0-beta + app.kubernetes.io/name: elastic-agent + app.kubernetes.io/instance: example + app.kubernetes.io/version: 9.3.0 +spec: + selector: + matchLabels: + name: agent-autoops-example + template: + metadata: + labels: + name: agent-autoops-example + annotations: + checksum/config: e89ad87aab4abdf0b266914a815ddd71744340acbec3678f63b018cfe85334df + spec: + automountServiceAccountToken: true + containers: + - args: + - otel + - --config + - /etc/elastic-agent/agent.yml + command: + - elastic-agent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: STATE_PATH + value: /usr/share/elastic-agent/state + - name: AUTOOPS_TOKEN + valueFrom: + secretKeyRef: + key: autoops-token + name: agent-autoops-example-autoops + - name: AUTOOPS_TEMP_RESOURCE_ID + valueFrom: + secretKeyRef: + key: temp-resource-id + name: agent-autoops-example-autoops + - name: AUTOOPS_OTEL_URL + valueFrom: + secretKeyRef: + key: otel-url + name: agent-autoops-example-autoops + - name: AUTOOPS_ES_URL + valueFrom: + secretKeyRef: + key: autoops-es-url + name: agent-autoops-example-autoops + - name: ELASTICSEARCH_READ_API_KEY + valueFrom: + secretKeyRef: + key: es-api-key + name: agent-autoops-example-autoops + - name: ELASTIC_CLOUD_CONNECTED_MODE_API_KEY + valueFrom: + secretKeyRef: + key: cloud-connected-mode-api-key + name: agent-autoops-example-autoops + - name: ELASTIC_CLOUD_CONNECTED_MODE_API_URL + valueFrom: + secretKeyRef: + key: cloud-connected-mode-api-url + name: agent-autoops-example-autoops + optional: true + image: docker.elastic.co/elastic-agent/elastic-agent:9.3.0-SNAPSHOT + imagePullPolicy: IfNotPresent + name: agent + securityContext: + runAsUser: 0 + volumeMounts: + - mountPath: /usr/share/elastic-agent/state + name: agent-data + - mountPath: /etc/elastic-agent/agent.yml + name: config + readOnly: true + subPath: agent.yml + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: agent-autoops-example + volumes: + - emptyDir: {} + name: agent-data + - name: config + secret: + defaultMode: 292 + secretName: agent-autoops-example diff --git a/deploy/helm/elastic-agent/templates/agent/_helpers.tpl b/deploy/helm/elastic-agent/templates/agent/_helpers.tpl index 590c41357a8..621074a0781 100644 --- a/deploy/helm/elastic-agent/templates/agent/_helpers.tpl +++ b/deploy/helm/elastic-agent/templates/agent/_helpers.tpl @@ -89,6 +89,7 @@ Initialise input templates if we are not deploying as managed as they change the k8s configuration of presets e.g. necessary volume mounts, etc. */}} {{- include "elasticagent.kubernetes.init" $ -}} {{- include "elasticagent.system.init" $ -}} +{{- include "elasticagent.autoops.init" $ -}} {{/* initialise inputs the custom integrations only if fleet is disabled */}} {{- if eq $.Values.agent.fleet.enabled false -}} {{- range $customInputName, $customInputVal := $.Values.extraIntegrations -}} @@ -351,7 +352,7 @@ app.kubernetes.io/version: {{ .Values.agent.version}} {{- $presetVal := index . 1 -}} {{- $otelConfigVal := index . 2 -}} {{- $presetOtelConfig := dig "otelConfig" (dict) $presetVal -}} -{{- $presetOtelConfig = uniq (deepCopy $presetOtelConfig | merge $otelConfigVal) -}} +{{- $presetOtelConfig = (deepCopy $presetOtelConfig | merge $otelConfigVal) -}} {{- $_ := set $presetVal "otelConfig" $presetOtelConfig -}} {{- end -}} diff --git a/deploy/helm/elastic-agent/templates/integrations/_auto_ops/_auto_ops.tpl b/deploy/helm/elastic-agent/templates/integrations/_auto_ops/_auto_ops.tpl new file mode 100644 index 00000000000..583098e3748 --- /dev/null +++ b/deploy/helm/elastic-agent/templates/integrations/_auto_ops/_auto_ops.tpl @@ -0,0 +1,131 @@ +{{- define "elasticagent.autoops.init" -}} +{{- if eq $.Values.autoOps.enabled true -}} +{{- $presetVal := $.Values.agent.presets.autoOps -}} +{{- $autoOpsConfig := ((include "elasticagent.autoops.config" $) | fromYaml) -}} +{{- include "elasticagent.preset.mutate.otelConfig" (list $ $presetVal $autoOpsConfig) -}} +{{- $autoOpsEnvVars := ((include "elasticagent.autoops.envVars" $) | fromYaml) -}} +{{- include "elasticagent.preset.mutate.envs" (list $presetVal $autoOpsEnvVars)}} +{{- end -}} +{{- end -}} + +{{- define "elasticagent.autoops.config" -}} +receivers: + metricbeatreceiver: + metricbeat: + modules: + # Metrics + - module: autoops_es + hosts: ${env:AUTOOPS_ES_URL} + period: 10s + metricsets: + - cat_shards + - cluster_health + - cluster_settings + - license + - node_stats + - tasks_management + # Templates + - module: autoops_es + hosts: ${env:AUTOOPS_ES_URL} + period: 24h + metricsets: + - cat_template + - component_template + - index_template + processors: + - add_fields: + target: autoops_es + fields: + temp_resource_id: ${env:AUTOOPS_TEMP_RESOURCE_ID} + token: ${env:AUTOOPS_TOKEN} + output: + otelconsumer: + telemetry_types: ["logs"] + +exporters: + otlphttp: + headers: + Authorization: "AutoOpsToken ${env:AUTOOPS_TOKEN}" + endpoint: ${env:AUTOOPS_OTEL_URL} + +service: + pipelines: + logs: + receivers: [metricbeatreceiver] + exporters: [otlphttp] + telemetry: + logs: + encoding: json +{{- end -}} + +{{- define "elasticagent.autoops.envVars" -}} +{{- $presetName := "autoOps" -}} +{{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName) -}} +extraEnvs: + # Always present + - name: AUTOOPS_TOKEN + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: autoops-token + + - name: AUTOOPS_TEMP_RESOURCE_ID + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: temp-resource-id + + - name: AUTOOPS_OTEL_URL + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: otel-url + + {{- if $.Values.autoOps.autoops_es_url }} + - name: AUTOOPS_ES_URL + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: autoops-es-url + {{- end }} + + # Only if API key provided + {{- if $.Values.autoOps.es_api_key }} + - name: ELASTICSEARCH_READ_API_KEY + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: es-api-key + {{- end }} + + # Only if BOTH username & password provided + {{- if and $.Values.autoOps.es_username $.Values.autoOps.es_password }} + - name: ELASTICSEARCH_READ_USERNAME + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: es-username + - name: ELASTICSEARCH_READ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: es-password + {{- end }} + +{{- if $.Values.autoOps.elastic_cloud_connected_mode_api_key }} + - name: ELASTIC_CLOUD_CONNECTED_MODE_API_KEY + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: cloud-connected-mode-api-key +{{- end }} + +{{- if $.Values.autoOps.elastic_cloud_connected_mode_api_url }} + - name: ELASTIC_CLOUD_CONNECTED_MODE_API_URL + valueFrom: + secretKeyRef: + name: {{ $agentName }}-autoops + key: cloud-connected-mode-api-url + optional: true +{{- end }} +{{- end -}} diff --git a/deploy/helm/elastic-agent/templates/integrations/_auto_ops/secret.yaml b/deploy/helm/elastic-agent/templates/integrations/_auto_ops/secret.yaml new file mode 100644 index 00000000000..9c75713acbc --- /dev/null +++ b/deploy/helm/elastic-agent/templates/integrations/_auto_ops/secret.yaml @@ -0,0 +1,39 @@ +{{- if eq $.Values.autoOps.enabled true }} +{{- $presetName := "autoOps" }} +{{- $presetVal := $.Values.agent.presets.autoOps }} +{{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $agentName }}-autoops + namespace: {{ .namespace | default $.Release.Namespace | quote }} + labels: + {{- include "elasticagent.labels" $ | nindent 4 }} + {{- with ($presetVal).labels -}} + {{ toYaml . | nindent 4 }} + {{- end }} + {{- with ($presetVal).annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +stringData: + autoops-token: {{ $.Values.autoOps.autoops_token | quote }} + {{- if and $.Values.autoOps.es_username $.Values.autoOps.es_password }} + es-username: {{ $.Values.autoOps.es_username | quote }} + es-password: {{ $.Values.autoOps.es_password | quote }} + {{- end }} + {{- if $.Values.autoOps.autoops_es_url }} + autoops-es-url: {{ $.Values.autoOps.autoops_es_url | quote }} + {{- end }} + temp-resource-id: {{ $.Values.autoOps.autoops_temp_resource_id | quote }} + otel-url: {{ $.Values.autoOps.autoops_otel_url | quote }} + {{- if $.Values.autoOps.es_api_key }} + es-api-key: {{ $.Values.autoOps.es_api_key | quote }} + {{- end }} + {{- if $.Values.autoOps.elastic_cloud_connected_mode_api_key }} + cloud-connected-mode-api-key: {{ $.Values.autoOps.elastic_cloud_connected_mode_api_key | quote }} + {{- end }} + {{- if $.Values.autoOps.elastic_cloud_connected_mode_api_url }} + cloud-connected-mode-api-url: {{ $.Values.autoOps.elastic_cloud_connected_mode_api_url | quote }} + {{- end }} +{{- end }} diff --git a/deploy/helm/elastic-agent/values.schema.json b/deploy/helm/elastic-agent/values.schema.json index c78319b05f8..bb5be015d04 100644 --- a/deploy/helm/elastic-agent/values.schema.json +++ b/deploy/helm/elastic-agent/values.schema.json @@ -331,6 +331,88 @@ } } }, + "autoOps": { + "type": "object", + "description": "Configuration for AutoOps integration.", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean", "description": "Enable AutoOps integration." }, + + "autoops_token": { "type": "string", "description": "Token to register into AutoOps" }, + "autoops_otel_url": { "type": "string", "description": "OTel server URL" }, + "autoops_temp_resource_id": { "type": "string", "description": "Temporary resource ID" }, + + "es_api_key": { "type": "string", "description": "Monitored ES cluster API key" }, + "es_username": { "type": "string", "description": "Monitored ES cluster username" }, + "es_password": { "type": "string", "description": "Monitored ES cluster password" }, + "autoops_es_url": { "type": "string", "description": "Monitored ES cluster URL" }, + + + "elastic_cloud_connected_mode_api_key": { "type": "string", "description": "Cloud Connected Mode API key" }, + "elastic_cloud_connected_mode_api_url": { "type": "string", "description": "Cloud Connected Mode API URL" } + }, + + "if": { + "properties": { "enabled": { "const": true } }, + "required": ["enabled"] + }, + "then": { + "required": ["autoops_token", "autoops_otel_url", "autoops_temp_resource_id"], + "allOf": [ + { + "properties": { + "autoops_token": { "type": "string", "minLength": 1 }, + "autoops_otel_url": { "type": "string", "minLength": 1 }, + "autoops_temp_resource_id": { "type": "string", "minLength": 1 }, + "autoops_es_url": { "type": "string", "description": "Monitored ES cluster URL" } + } + } + ], + "oneOf": [ + { + "description": "Auth via API key only", + "required": ["es_api_key"], + "properties": { + "es_api_key": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "not": { + "anyOf": [ + { + "required": ["es_username"], + "properties": { "es_username": { "type": "string", "minLength": 1 } } + }, + { + "required": ["es_password"], + "properties": { "es_password": { "type": "string", "minLength": 1 } } + } + ] + } + } + ] + }, + { + "description": "Auth via username + password only", + "required": ["es_username", "es_password"], + "properties": { + "es_username": { "type": "string", "minLength": 1 }, + "es_password": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "not": { + "allOf": [ + { "required": ["es_api_key"] }, + { "properties": { "es_api_key": { "type": "string", "minLength": 1 } } } + ] + } + } + ] + } + ] + } + }, "extraIntegrations": { "type": "object", "description": "Configuration for extra integrations.", diff --git a/deploy/helm/elastic-agent/values.yaml b/deploy/helm/elastic-agent/values.yaml index 6d9f336fe5c..0dc348f4e8f 100644 --- a/deploy/helm/elastic-agent/values.yaml +++ b/deploy/helm/elastic-agent/values.yaml @@ -300,6 +300,17 @@ kubernetes: # -- system metric stream vars # @section -- 2 - Kubernetes integration vars: {} +autoOps: + enabled: false + autoops_token: "" + autoops_otel_url: "" + autoops_temp_resource_id: "" + es_api_key: "" + es_username: "" + es_password: "" + elastic_cloud_connected_mode_api_key: "" + elastic_cloud_connected_mode_api_url: "" + autoops_es_url: "" system: # -- enable System integration. # @section -- 4 - System integration @@ -530,6 +541,14 @@ agent: kubernetes: node: ${NODE_NAME} scope: node + autoOps: + mode: deployment + serviceAccount: + create: true + clusterRole: + create: false + nodeSelector: + kubernetes.io/os: linux kube-state-metrics: enabled: true fullnameOverride: "kube-state-metrics"