Add processors and tags to all system data streams

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.17.0"
+  changes:
+    - description: Add processor and tag fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3562
 - version: "1.16.2"
   changes:
     - description: Update documentation with additional context for new users.

packages/system/data_stream/core/agent/stream/stream.yml.hbs

@@ -5,4 +5,14 @@ core.metrics:
 {{/each}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/core/manifest.yml

@@ -23,5 +23,18 @@ streams:
 
         default:
           - percentages
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System core metrics
     description: Collect System core metrics

packages/system/data_stream/cpu/agent/stream/stream.yml.hbs

@@ -6,4 +6,14 @@ cpu.metrics:
 period: {{period}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/cpu/manifest.yml

@@ -23,5 +23,18 @@ streams:
         default:
           - percentages
           - normalized_percentages
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System cpu metrics
     description: Collect System cpu metrics

packages/system/data_stream/diskio/agent/stream/stream.yml.hbs

@@ -6,4 +6,14 @@ diskio.include_devices:
 period: {{period}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/diskio/manifest.yml

@@ -20,5 +20,18 @@ streams:
         description: >
           Provide a specific list of devices to monitor. By default, all devices are monitored.
 
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System diskio metrics
     description: Collect System diskio metrics

packages/system/data_stream/filesystem/agent/stream/stream.yml.hbs

@@ -6,4 +6,10 @@ filesystem.ignore_types: {{filesystem.ignore_types}}
 {{/if}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/filesystem/manifest.yml

@@ -12,26 +12,31 @@ streams:
         required: true
         show_user: true
         default: 1m
+      - name: filesystem.ignore_types
+        type: text
+        title: List of filesystem types to ignore
+        multi: true
+        required: false
+        show_user: true
+        description: >
+          The filesystem datastream will ignore any filesystems with a matching type as specified here. By default, this will exclude any filesystems marked as "nodev" in /proc/filesystems on linux.
+
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
       - name: processors
         type: yaml
         title: Processors
         multi: false
         required: true
-        show_user: true
+        show_user: false
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with external metadata.
 
         default: |
           - drop_event.when.regexp:
               system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
-      - name: filesystem.ignore_types
-        type: text
-        title: List of filesystem types to ignore
-        multi: true
-        required: false
-        show_user: true
-        description: >
-          The filesystem datastream will ignore any filesystems with a matching type as specified here. By default, this will exclude any filesystems marked as "nodev" in /proc/filesystems on linux.
-
     title: System filesystem metrics
     description: Collect System filesystem metrics

packages/system/data_stream/fsstat/agent/stream/stream.yml.hbs

@@ -3,4 +3,10 @@ period: {{period}}
 processors: {{processors}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/fsstat/manifest.yml

@@ -12,6 +12,11 @@ streams:
         required: true
         show_user: true
         default: 1m
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
       - name: processors
         type: yaml
         title: Processors

packages/system/data_stream/load/agent/stream/stream.yml.hbs

@@ -1,3 +1,13 @@
 metricsets: ["load"]
 condition: ${host.platform} != 'windows'
-period: {{period}}
+period: {{period}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
+{{/if}}

packages/system/data_stream/load/manifest.yml

@@ -11,5 +11,18 @@ streams:
         required: true
         show_user: true
         default: 10s
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System load metrics
     description: Collect System load metrics

packages/system/data_stream/memory/agent/stream/stream.yml.hbs

@@ -2,4 +2,14 @@ metricsets: ["memory"]
 period: {{period}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/memory/manifest.yml

@@ -11,5 +11,18 @@ streams:
         required: true
         show_user: true
         default: 10s
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System memory metrics
     description: Collect System memory metrics

packages/system/data_stream/network/agent/stream/stream.yml.hbs

@@ -2,5 +2,15 @@ metricsets: ["network"]
 period: {{period}}
 network.interfaces:
 {{#each network.interfaces}}
-  - {{this}}
+- {{this}}
 {{/each}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
+{{/if}}

packages/system/data_stream/network/manifest.yml

@@ -20,5 +20,18 @@ streams:
         description: >
           List of interfaces to monitor. Will monitor all by default.
 
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System network metrics
     description: Collect System network metrics

packages/system/data_stream/process/agent/stream/stream.yml.hbs

@@ -8,13 +8,23 @@ process.include_cpu_ticks: {{process.include_cpu_ticks}}
 {{#if process.env.whitelist}}
 process.env.whitelist:
 {{#each process.env.whitelist}}
-  - {{this}}
+- {{this}}
 {{/each}}
 {{/if}}
 processes:
 {{#each processes}}
-  - {{this}}
+- {{this}}
 {{/each}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
 {{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
+{{/if}}

packages/system/data_stream/process/manifest.yml

@@ -81,5 +81,18 @@ streams:
 
         default:
           - .*
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System process metrics
     description: Collect System process metrics

packages/system/data_stream/process_summary/agent/stream/stream.yml.hbs

@@ -2,4 +2,14 @@ metricsets: ["process_summary"]
 period: {{period}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}

packages/system/data_stream/process_summary/manifest.yml

@@ -12,5 +12,18 @@ streams:
         required: true
         show_user: true
         default: 10s
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: System process_summary metrics
     description: Collect System process_summary metrics

packages/system/data_stream/socket_summary/agent/stream/stream.yml.hbs

@@ -2,4 +2,14 @@ metricsets: ["socket_summary"]
 period: {{period}}
 {{#if system.hostfs}}
 system.hostfs: {{system.hostfs}}
+{{/if}}
+{{#if processors.length}}
+processors:
+{{processors}}
+{{/if}}
+{{#if tags.length}}
+tags:
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
 {{/if}}