diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b2b4bc6c3c68..8f956a070cc1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810] - Add read_buffer configuration option. {pull}11739[11739] +- `convert_timezone` option is removed and locale is always added to the event so timezone is used when parsing the timestamp, this behaviour can be overriden with processors. {pull}12410[12410] *Heartbeat* diff --git a/filebeat/docs/include/timezone-support.asciidoc b/filebeat/docs/include/timezone-support.asciidoc new file mode 100644 index 000000000000..e49bc860e731 --- /dev/null +++ b/filebeat/docs/include/timezone-support.asciidoc @@ -0,0 +1,17 @@ +[float] +==== Timezone support + +This module parses logs that don't contain timezone information. For these logs, +Filebeat reads the local timezone and uses it when parsing to convert the +timestamp to UTC. The timezone to be used for parsing is included in the event +in the `event.timezone` field. + +To disable this conversion, the `event.timezone` field can be removed with +the `drop_fields` processor. + +If logs are originated from systems or applications with a different timezone to +the local one, the `event.timezone` field can be overwritten with the original +timezone using the `add_fields` processor. + +See <> for information about specifying +processors in your config. diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc index af750bb63e45..7d5ed4c4d6a2 100644 --- a/filebeat/docs/modules/system.asciidoc +++ b/filebeat/docs/modules/system.asciidoc @@ -69,14 +69,12 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] - [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] +include::../include/timezone-support.asciidoc[] :has-dashboards!: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 4df44bbe093e..71ecb1f09e0e 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -21,9 +21,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -36,9 +33,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml index b4121ca80815..3c7a0b43d499 100644 --- a/filebeat/module/system/_meta/config.reference.yml +++ b/filebeat/module/system/_meta/config.reference.yml @@ -7,9 +7,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -22,9 +19,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml index 34d3bb35df23..f76dd905b4d7 100644 --- a/filebeat/module/system/_meta/config.yml +++ b/filebeat/module/system/_meta/config.yml @@ -7,9 +7,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Authorization logs auth: enabled: true @@ -17,6 +14,3 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc index 06a1f912cde5..3fcfa5773fd2 100644 --- a/filebeat/module/system/_meta/docs.asciidoc +++ b/filebeat/module/system/_meta/docs.asciidoc @@ -64,14 +64,12 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] - [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] -include::../include/var-convert-timezone.asciidoc[] +include::../include/timezone-support.asciidoc[] :has-dashboards!: diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml index 3834faa1c9a4..c691d47a3561 100644 --- a/filebeat/module/system/auth/config/auth.yml +++ b/filebeat/module/system/auth/config/auth.yml @@ -7,7 +7,5 @@ exclude_files: [".gz$"] multiline: pattern: "^\\s" match: after -{{ if .convert_timezone }} processors: - add_locale: ~ -{{ end }} diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml index 0e7cc747b9b0..ade9e03a69af 100644 --- a/filebeat/module/system/auth/manifest.yml +++ b/filebeat/module/system/auth/manifest.yml @@ -10,13 +10,6 @@ var: # ssh logs to files - /var/log/secure.log* os.windows: [] - - name: convert_timezone - default: false - # if ES < 6.1.0, this flag switches to false automatically when evaluating the - # pipeline - min_elasticsearch_version: - version: 6.1.0 - value: false ingest_pipeline: ingest/pipeline.json input: config/auth.yml diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml index 3834faa1c9a4..c691d47a3561 100644 --- a/filebeat/module/system/syslog/config/syslog.yml +++ b/filebeat/module/system/syslog/config/syslog.yml @@ -7,7 +7,5 @@ exclude_files: [".gz$"] multiline: pattern: "^\\s" match: after -{{ if .convert_timezone }} processors: - add_locale: ~ -{{ end }} diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml index 5d4bebb28077..fa0ec0491351 100644 --- a/filebeat/module/system/syslog/manifest.yml +++ b/filebeat/module/system/syslog/manifest.yml @@ -8,13 +8,6 @@ var: os.darwin: - /var/log/system.log* os.windows: [] - - name: convert_timezone - default: false - # if ES < 6.1.0, this flag switches to false automatically when evaluating the - # pipeline - min_elasticsearch_version: - version: 6.1.0 - value: false ingest_pipeline: ingest/pipeline.json input: config/syslog.yml diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled index 32ddd6b6d9cc..49e5c9c4d984 100644 --- a/filebeat/modules.d/system.yml.disabled +++ b/filebeat/modules.d/system.yml.disabled @@ -10,9 +10,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Authorization logs auth: enabled: true @@ -20,6 +17,3 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f1a4d580e9a0..b1d412da7b11 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -21,9 +21,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -36,9 +33,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - #var.convert_timezone: false - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: