From 0415cf077f5ecd95cf7129674e5cb48a7393f871 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Tue, 12 Jul 2022 15:24:02 -0700 Subject: [PATCH] add ignore_older to remaining nginx and system data streams --- packages/nginx/changelog.yml | 5 +++++ .../data_stream/access/agent/stream/stream.yml.hbs | 3 +++ packages/nginx/data_stream/access/manifest.yml | 8 ++++++++ .../data_stream/error/agent/stream/stream.yml.hbs | 3 +++ packages/nginx/data_stream/error/manifest.yml | 8 ++++++++ packages/nginx/manifest.yml | 2 +- packages/system/changelog.yml | 5 +++++ .../system/data_stream/auth/agent/stream/log.yml.hbs | 11 +++++++---- packages/system/data_stream/auth/manifest.yml | 8 ++++++++ .../data_stream/syslog/agent/stream/log.yml.hbs | 11 +++++++---- packages/system/data_stream/syslog/manifest.yml | 8 ++++++++ packages/system/manifest.yml | 2 +- 12 files changed, 64 insertions(+), 10 deletions(-) diff --git a/packages/nginx/changelog.yml b/packages/nginx/changelog.yml index 360892bac5b..b321ab7fffa 100644 --- a/packages/nginx/changelog.yml +++ b/packages/nginx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.5.0" + changes: + - description: Add ignore_older to log data streams + type: enhancement + link: https://github.com/elastic/integrations/pull/3690 - version: "1.4.1" changes: - description: Update documentation with additional context for new users. diff --git a/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs b/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs index 531705c5f3f..8fb8db0c708 100644 --- a/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs +++ b/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs @@ -1,3 +1,6 @@ +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} paths: {{#each paths}} - {{this}} diff --git a/packages/nginx/data_stream/access/manifest.yml b/packages/nginx/data_stream/access/manifest.yml index 4892e1f828c..137825e3843 100644 --- a/packages/nginx/data_stream/access/manifest.yml +++ b/packages/nginx/data_stream/access/manifest.yml @@ -36,6 +36,14 @@ streams: description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". title: Nginx access logs description: Collect Nginx access logs - input: httpjson diff --git a/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs b/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs index e860976a65e..dc6349ad322 100644 --- a/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs +++ b/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs @@ -1,3 +1,6 @@ +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} paths: {{#each paths}} - {{this}} diff --git a/packages/nginx/data_stream/error/manifest.yml b/packages/nginx/data_stream/error/manifest.yml index 1a741682a6c..70e51bec93b 100644 --- a/packages/nginx/data_stream/error/manifest.yml +++ b/packages/nginx/data_stream/error/manifest.yml @@ -36,6 +36,14 @@ streams: description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". title: Nginx error logs description: Collect Nginx error logs - input: httpjson diff --git a/packages/nginx/manifest.yml b/packages/nginx/manifest.yml index ca01603cf30..84fb05f31c7 100644 --- a/packages/nginx/manifest.yml +++ b/packages/nginx/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: nginx title: Nginx -version: 1.4.1 +version: 1.5.0 license: basic description: Collect logs and metrics from Nginx HTTP servers with Elastic Agent. type: integration diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml index cad6ce29af8..f143ffb6074 100644 --- a/packages/system/changelog.yml +++ b/packages/system/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.0" + changes: + - description: Add ignore_older to remaining logs + type: enhancement + link: https://github.com/elastic/integrations/pull/3690 - version: "1.17.0" changes: - description: Add processor and tag fields diff --git a/packages/system/data_stream/auth/agent/stream/log.yml.hbs b/packages/system/data_stream/auth/agent/stream/log.yml.hbs index 2cb3f087334..a32c68fbf20 100644 --- a/packages/system/data_stream/auth/agent/stream/log.yml.hbs +++ b/packages/system/data_stream/auth/agent/stream/log.yml.hbs @@ -1,10 +1,13 @@ +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} paths: {{#each paths as |path i|}} - - {{path}} +- {{path}} {{/each}} exclude_files: [".gz$"] multiline: - pattern: "^\\s" - match: after +pattern: "^\\s" +match: after processors: - - add_locale: ~ +- add_locale: ~ \ No newline at end of file diff --git a/packages/system/data_stream/auth/manifest.yml b/packages/system/data_stream/auth/manifest.yml index 428764ece1d..a59a984c41a 100644 --- a/packages/system/data_stream/auth/manifest.yml +++ b/packages/system/data_stream/auth/manifest.yml @@ -4,6 +4,14 @@ type: logs streams: - input: logfile vars: + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: paths type: text title: Paths diff --git a/packages/system/data_stream/syslog/agent/stream/log.yml.hbs b/packages/system/data_stream/syslog/agent/stream/log.yml.hbs index 13835734a55..42c8594f63a 100644 --- a/packages/system/data_stream/syslog/agent/stream/log.yml.hbs +++ b/packages/system/data_stream/syslog/agent/stream/log.yml.hbs @@ -1,13 +1,13 @@ paths: {{#each paths as |path i|}} - - {{path}} +- {{path}} {{/each}} exclude_files: [".gz$"] multiline: - pattern: "^\\s" - match: after +pattern: "^\\s" +match: after processors: - - add_locale: ~ +- add_locale: ~ {{#if processors.length}} processors: {{processors}} @@ -17,4 +17,7 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} {{/if}} \ No newline at end of file diff --git a/packages/system/data_stream/syslog/manifest.yml b/packages/system/data_stream/syslog/manifest.yml index 23363957f5b..db2fb291b20 100644 --- a/packages/system/data_stream/syslog/manifest.yml +++ b/packages/system/data_stream/syslog/manifest.yml @@ -26,6 +26,14 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". template_path: log.yml.hbs title: System syslog logs (log) description: Collect System syslog logs using log input diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml index a3eb781eb0d..0d1dd066e93 100644 --- a/packages/system/manifest.yml +++ b/packages/system/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: system title: System -version: 1.17.0 +version: 1.18.0 license: basic description: Collect system logs and metrics from your servers with Elastic Agent. type: integration