From ae43ca35951ee8625ef6a01160d169a8edc83f00 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 4 Jun 2019 10:39:40 +0200 Subject: [PATCH 1/3] Remove `convert_timezone` option from filebeat system module Timezone is now obtained from the system where filebeat is running, and timezone conversion when parsing the timestamp is always done. This timezone information is included in the `event.timezone` field, it can be overriden with the `add_fields` processor, or removed with the `remove_fields` processor. --- CHANGELOG.next.asciidoc | 1 + filebeat/filebeat.reference.yml | 6 ------ filebeat/module/system/_meta/config.reference.yml | 6 ------ filebeat/module/system/_meta/config.yml | 6 ------ filebeat/module/system/auth/config/auth.yml | 2 -- filebeat/module/system/auth/manifest.yml | 7 ------- filebeat/module/system/syslog/config/syslog.yml | 2 -- filebeat/module/system/syslog/manifest.yml | 7 ------- filebeat/modules.d/system.yml.disabled | 6 ------ 9 files changed, 1 insertion(+), 42 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b2b4bc6c3c68..8f956a070cc1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810] - Add read_buffer configuration option. {pull}11739[11739] +- `convert_timezone` option is removed and locale is always added to the event so timezone is used when parsing the timestamp, this behaviour can be overriden with processors. {pull}12410[12410] *Heartbeat* diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 4df44bbe093e..71ecb1f09e0e 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -21,9 +21,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -36,9 +33,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml index b4121ca80815..3c7a0b43d499 100644 --- a/filebeat/module/system/_meta/config.reference.yml +++ b/filebeat/module/system/_meta/config.reference.yml @@ -7,9 +7,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -22,9 +19,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml index 34d3bb35df23..f76dd905b4d7 100644 --- a/filebeat/module/system/_meta/config.yml +++ b/filebeat/module/system/_meta/config.yml @@ -7,9 +7,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Authorization logs auth: enabled: true @@ -17,6 +14,3 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml index 3834faa1c9a4..c691d47a3561 100644 --- a/filebeat/module/system/auth/config/auth.yml +++ b/filebeat/module/system/auth/config/auth.yml @@ -7,7 +7,5 @@ exclude_files: [".gz$"] multiline: pattern: "^\\s" match: after -{{ if .convert_timezone }} processors: - add_locale: ~ -{{ end }} diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml index 0e7cc747b9b0..ade9e03a69af 100644 --- a/filebeat/module/system/auth/manifest.yml +++ b/filebeat/module/system/auth/manifest.yml @@ -10,13 +10,6 @@ var: # ssh logs to files - /var/log/secure.log* os.windows: [] - - name: convert_timezone - default: false - # if ES < 6.1.0, this flag switches to false automatically when evaluating the - # pipeline - min_elasticsearch_version: - version: 6.1.0 - value: false ingest_pipeline: ingest/pipeline.json input: config/auth.yml diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml index 3834faa1c9a4..c691d47a3561 100644 --- a/filebeat/module/system/syslog/config/syslog.yml +++ b/filebeat/module/system/syslog/config/syslog.yml @@ -7,7 +7,5 @@ exclude_files: [".gz$"] multiline: pattern: "^\\s" match: after -{{ if .convert_timezone }} processors: - add_locale: ~ -{{ end }} diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml index 5d4bebb28077..fa0ec0491351 100644 --- a/filebeat/module/system/syslog/manifest.yml +++ b/filebeat/module/system/syslog/manifest.yml @@ -8,13 +8,6 @@ var: os.darwin: - /var/log/system.log* os.windows: [] - - name: convert_timezone - default: false - # if ES < 6.1.0, this flag switches to false automatically when evaluating the - # pipeline - min_elasticsearch_version: - version: 6.1.0 - value: false ingest_pipeline: ingest/pipeline.json input: config/syslog.yml diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled index 32ddd6b6d9cc..49e5c9c4d984 100644 --- a/filebeat/modules.d/system.yml.disabled +++ b/filebeat/modules.d/system.yml.disabled @@ -10,9 +10,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Authorization logs auth: enabled: true @@ -20,6 +17,3 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false From aae341c0b49d89ce45312c8c9301baef08cda50c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 4 Jun 2019 11:08:40 +0200 Subject: [PATCH 2/3] Update filebeat x-pack reference config --- x-pack/filebeat/filebeat.reference.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f1a4d580e9a0..b1d412da7b11 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -21,9 +21,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -36,9 +33,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: From c0e0053f22a3cf6b4079e0d00631685fd522eb5c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 4 Jun 2019 12:23:17 +0200 Subject: [PATCH 3/3] Add documentation for timezones --- filebeat/docs/include/timezone-support.asciidoc | 17 +++++++++++++++++ filebeat/docs/modules/system.asciidoc | 4 +--- filebeat/module/system/_meta/docs.asciidoc | 4 +--- 3 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 filebeat/docs/include/timezone-support.asciidoc diff --git a/filebeat/docs/include/timezone-support.asciidoc b/filebeat/docs/include/timezone-support.asciidoc new file mode 100644 index 000000000000..e49bc860e731 --- /dev/null +++ b/filebeat/docs/include/timezone-support.asciidoc @@ -0,0 +1,17 @@ +[float] +==== Timezone support + +This module parses logs that don't contain timezone information. For these logs, +Filebeat reads the local timezone and uses it when parsing to convert the +timestamp to UTC. The timezone to be used for parsing is included in the event +in the `event.timezone` field. + +To disable this conversion, the `event.timezone` field can be removed with +the `drop_fields` processor. + +If logs are originated from systems or applications with a different timezone to +the local one, the `event.timezone` field can be overwritten with the original +timezone using the `add_fields` processor. + +See <> for information about specifying +processors in your config. diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc index af750bb63e45..7d5ed4c4d6a2 100644 --- a/filebeat/docs/modules/system.asciidoc +++ b/filebeat/docs/modules/system.asciidoc @@ -69,14 +69,12 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] - [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] +include::../include/timezone-support.asciidoc[] :has-dashboards!: diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc index 06a1f912cde5..3fcfa5773fd2 100644 --- a/filebeat/module/system/_meta/docs.asciidoc +++ b/filebeat/module/system/_meta/docs.asciidoc @@ -64,14 +64,12 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] - [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] +include::../include/timezone-support.asciidoc[] :has-dashboards!: