From 38719960a41221b740e6a4bd24a94f16aa0b9d37 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 19 Sep 2023 10:33:28 -0400 Subject: [PATCH 1/4] [elastic_agent] Fix system.process.cpu.{system,user,total}.time.ms mapping Fix mapping and description for the `system.process.cpu.{system,user,total}.time.ms` fields. Fixes #4245 --- .../apm_server_metrics/fields/fields.yml | 15 +++++++++------ .../auditbeat_metrics/fields/fields.yml | 15 +++++++++------ .../cloudbeat_metrics/fields/fields.yml | 15 +++++++++------ .../elastic_agent_metrics/fields/fields.yml | 6 +++--- .../filebeat_metrics/fields/fields.yml | 15 +++++++++------ .../fleet_server_metrics/fields/fields.yml | 15 +++++++++------ .../heartbeat_metrics/fields/fields.yml | 15 +++++++++------ .../metricbeat_metrics/fields/fields.yml | 15 +++++++++------ .../osquerybeat_metrics/fields/fields.yml | 15 +++++++++------ .../packetbeat_metrics/fields/fields.yml | 15 +++++++++------ 10 files changed, 84 insertions(+), 57 deletions(-) diff --git a/packages/elastic_agent/data_stream/apm_server_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/apm_server_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/apm_server_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/apm_server_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/auditbeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/auditbeat_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/auditbeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/auditbeat_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/cloudbeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/cloudbeat_metrics/fields/fields.yml index 78b0cec8df7..5302fe09d73 100644 --- a/packages/elastic_agent/data_stream/cloudbeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/cloudbeat_metrics/fields/fields.yml @@ -45,17 +45,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/fields.yml index 9cc6627c7f7..fbbdf6701ac 100644 --- a/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/fields.yml @@ -53,17 +53,17 @@ type: long metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms type: long metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms type: long metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/filebeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/filebeat_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/filebeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/filebeat_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/fleet_server_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/fleet_server_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/fleet_server_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/fleet_server_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/heartbeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/heartbeat_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/heartbeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/heartbeat_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/metricbeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/metricbeat_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/metricbeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/metricbeat_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/osquerybeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/osquerybeat_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/osquerybeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/osquerybeat_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: diff --git a/packages/elastic_agent/data_stream/packetbeat_metrics/fields/fields.yml b/packages/elastic_agent/data_stream/packetbeat_metrics/fields/fields.yml index 4aa74dca2b4..127066f5971 100644 --- a/packages/elastic_agent/data_stream/packetbeat_metrics/fields/fields.yml +++ b/packages/elastic_agent/data_stream/packetbeat_metrics/fields/fields.yml @@ -50,17 +50,20 @@ description: | The total CPU time spent by the process. - name: total.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The total CPU time spent by the process. - name: user.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in user space. - name: system.time.ms - type: date + type: long + metric_type: counter description: | - The time when the process was started. + The amount of CPU time the process spent in kernel space. - name: memory type: group fields: From c360d5c46535f00c651aa5c407fbe89d17c79ab9 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 19 Sep 2023 10:36:40 -0400 Subject: [PATCH 2/4] [elastic_agent] Change beat.stats.libbeat.config.{running,starts,stops} to long Align mapping for the `beat.stats.libbeat.config.{running,starts,stops}` fields with the `beat` integration. Fixes #4245 --- .../elastic_agent_metrics/fields/beat-stats-fields.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/beat-stats-fields.yml b/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/beat-stats-fields.yml index 8fd2649426a..6ea4b078eca 100644 --- a/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/beat-stats-fields.yml +++ b/packages/elastic_agent/data_stream/elastic_agent_metrics/fields/beat-stats-fields.yml @@ -160,11 +160,11 @@ type: group fields: - name: running - type: short + type: long - name: starts - type: short + type: long - name: stops - type: short + type: long - name: output type: group description: > From 7afc8d98a4aa7f0ec1dae19b1a5af689c30d50c8 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 19 Sep 2023 10:41:51 -0400 Subject: [PATCH 3/4] [elastic_agent] Use ECS definition for 'message' field type Fixes #7846 --- .../data_stream/apm_server_logs/fields/fields.yml | 3 +-- .../elastic_agent/data_stream/auditbeat_logs/fields/fields.yml | 3 +-- .../data_stream/cloud_defend_logs/fields/fields.yml | 3 +-- .../elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml | 3 +-- .../data_stream/elastic_agent_logs/fields/fields.yml | 3 +-- .../data_stream/endpoint_sercurity_logs/fields/fields.yml | 3 +-- .../elastic_agent/data_stream/filebeat_logs/fields/fields.yml | 3 +-- .../data_stream/fleet_server_logs/fields/fields.yml | 3 +-- .../elastic_agent/data_stream/heartbeat_logs/fields/fields.yml | 3 +-- .../data_stream/metricbeat_logs/fields/fields.yml | 3 +-- .../data_stream/osquerybeat_logs/fields/fields.yml | 3 +-- .../data_stream/packetbeat_logs/fields/fields.yml | 3 +-- 12 files changed, 12 insertions(+), 24 deletions(-) diff --git a/packages/elastic_agent/data_stream/apm_server_logs/fields/fields.yml b/packages/elastic_agent/data_stream/apm_server_logs/fields/fields.yml index 24771ec5046..504e0bb9581 100644 --- a/packages/elastic_agent/data_stream/apm_server_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/apm_server_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml index ce6082a4b9c..012b7bf3c5b 100644 --- a/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/cloud_defend_logs/fields/fields.yml b/packages/elastic_agent/data_stream/cloud_defend_logs/fields/fields.yml index 1acd8e61492..3e634873125 100644 --- a/packages/elastic_agent/data_stream/cloud_defend_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/cloud_defend_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: decision_id type: text title: Decision ID diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml index aaca02caed7..61fe833aa96 100644 --- a/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: decision_id type: text title: Decision ID diff --git a/packages/elastic_agent/data_stream/elastic_agent_logs/fields/fields.yml b/packages/elastic_agent/data_stream/elastic_agent_logs/fields/fields.yml index e31bc791110..f8fb04af2f5 100644 --- a/packages/elastic_agent/data_stream/elastic_agent_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/elastic_agent_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/endpoint_sercurity_logs/fields/fields.yml b/packages/elastic_agent/data_stream/endpoint_sercurity_logs/fields/fields.yml index 24771ec5046..504e0bb9581 100644 --- a/packages/elastic_agent/data_stream/endpoint_sercurity_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/endpoint_sercurity_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml index ce6082a4b9c..012b7bf3c5b 100644 --- a/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/fleet_server_logs/fields/fields.yml b/packages/elastic_agent/data_stream/fleet_server_logs/fields/fields.yml index 5dad28b927c..2c3028a180d 100644 --- a/packages/elastic_agent/data_stream/fleet_server_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/fleet_server_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml index 006352cf8d7..5044a016cd6 100644 --- a/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml index ce6082a4b9c..012b7bf3c5b 100644 --- a/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml index ce6082a4b9c..012b7bf3c5b 100644 --- a/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml index ce6082a4b9c..012b7bf3c5b 100644 --- a/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml @@ -1,6 +1,5 @@ - name: message - type: text - title: Log Message + external: ecs - name: elastic_agent title: Elastic Agent description: Fields related to the Elastic Agents From 4e6f2dc12bed8437801a7faad18476c5486f1594 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 19 Sep 2023 10:42:07 -0400 Subject: [PATCH 4/4] Add changelog --- packages/elastic_agent/changelog.yml | 11 +++++++++++ packages/elastic_agent/manifest.yml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/elastic_agent/changelog.yml b/packages/elastic_agent/changelog.yml index 5acea2f322e..5b9773a5051 100644 --- a/packages/elastic_agent/changelog.yml +++ b/packages/elastic_agent/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "1.13.1" + changes: + - description: Fix mapping and description for the `system.process.cpu.{system,user,total}.time.ms` fields. + type: bugfix + link: https://github.com/elastic/integrations/pull/7872 + - description: Align mapping for the `beat.stats.libbeat.config.{running,starts,stops}` fields with the `beat` integration. + type: bugfix + link: https://github.com/elastic/integrations/pull/7872 + - description: For the `message` field, consistently use the ECS defined mapping type of `match_only_text`. + type: bugfix + link: https://github.com/elastic/integrations/pull/7872 - version: "1.13.0" changes: - description: Remove metric mappings from the filebeat_input_logs data stream diff --git a/packages/elastic_agent/manifest.yml b/packages/elastic_agent/manifest.yml index dd51fab5812..52b893401c8 100644 --- a/packages/elastic_agent/manifest.yml +++ b/packages/elastic_agent/manifest.yml @@ -1,6 +1,6 @@ name: elastic_agent title: Elastic Agent -version: 1.13.0 +version: 1.13.1 description: Collect logs and metrics from Elastic Agents. type: integration format_version: 1.0.0