Add ignore_older to remaining nginx and system data streams

packages/nginx/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.0"
+  changes:
+    - description: Add ignore_older to log data streams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3690
 - version: "1.4.1"
   changes:
     - description: Update documentation with additional context for new users.

packages/nginx/data_stream/access/agent/stream/stream.yml.hbs

@@ -1,3 +1,6 @@
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 paths:
 {{#each paths}}
   - {{this}}

packages/nginx/data_stream/access/manifest.yml

@@ -36,6 +36,14 @@ streams:
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
 
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        default: 72h
+        required: false
+        show_user: false
+        description: >-
+          If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
     title: Nginx access logs
     description: Collect Nginx access logs
   - input: httpjson

packages/nginx/data_stream/error/agent/stream/stream.yml.hbs

@@ -1,3 +1,6 @@
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 paths:
 {{#each paths}}
   - {{this}}

packages/nginx/data_stream/error/manifest.yml

@@ -36,6 +36,14 @@ streams:
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
 
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        default: 72h
+        required: false
+        show_user: false
+        description: >-
+          If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
     title: Nginx error logs
     description: Collect Nginx error logs
   - input: httpjson

packages/nginx/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: nginx
 title: Nginx
-version: 1.4.1
+version: 1.5.0
 license: basic
 description: Collect logs and metrics from Nginx HTTP servers with Elastic Agent.
 type: integration

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.18.0"
+  changes:
+    - description: Add ignore_older to remaining logs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3690
 - version: "1.17.0"
   changes:
     - description: Add processor and tag fields

packages/system/data_stream/auth/agent/stream/log.yml.hbs

@@ -1,10 +1,13 @@
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 paths:
 {{#each paths as |path i|}}
- - {{path}}
+- {{path}}
 {{/each}}
 exclude_files: [".gz$"]
 multiline:
-  pattern: "^\\s"
-  match: after
+pattern: "^\\s"
+match: after
 processors:
-  - add_locale: ~
+- add_locale: ~

packages/system/data_stream/auth/manifest.yml

@@ -4,6 +4,14 @@ type: logs
 streams:
   - input: logfile
     vars:
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        default: 72h
+        required: false
+        show_user: false
+        description: >-
+          If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
       - name: paths
         type: text
         title: Paths

packages/system/data_stream/syslog/agent/stream/log.yml.hbs

@@ -1,13 +1,13 @@
 paths:
 {{#each paths as |path i|}}
- - {{path}}
+- {{path}}
 {{/each}}
 exclude_files: [".gz$"]
 multiline:
-  pattern: "^\\s"
-  match: after
+pattern: "^\\s"
+match: after
 processors:
-  - add_locale: ~
+- add_locale: ~
 {{#if processors.length}}
 processors:
 {{processors}}
@@ -17,4 +17,7 @@ tags:
 {{#each tags as |tag i|}}
 - {{tag}}
 {{/each}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
 {{/if}}

packages/system/data_stream/syslog/manifest.yml

@@ -26,6 +26,14 @@ streams:
         show_user: false
         description: >-
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        default: 72h
+        required: false
+        show_user: false
+        description: >-
+          If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
     template_path: log.yml.hbs
     title: System syslog logs (log)
     description: Collect System syslog logs using log input

packages/system/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: system
 title: System
-version: 1.17.0
+version: 1.18.0
 license: basic
 description: Collect system logs and metrics from your servers with Elastic Agent.
 type: integration