From 2fc27492524c5e55fb5c497f4f095a8fa41dd7e1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 14 Jan 2019 09:52:25 -0700 Subject: [PATCH 1/3] Add grok pattern to support different timestamp fmt in redis log (#10033) * Add grok pattern to support different timestamp fmt in redis log * Add changelog * Fix rebase issue (cherry picked from commit 2ec7b550a523fd4ca9c0b7ef87e6330e8e9b3def) --- CHANGELOG.next.asciidoc | 1 + filebeat/module/redis/log/ingest/pipeline.json | 7 +++++-- filebeat/module/redis/log/test/redis-5.0.3.log | 1 + .../redis/log/test/redis-5.0.3.log-expected.json | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 filebeat/module/redis/log/test/redis-5.0.3.log create mode 100644 filebeat/module/redis/log/test/redis-5.0.3.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0f171f23455e..8fc6f12b4e3b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -20,6 +20,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Allow beats to blacklist certain part of the configuration while using Central Management. {pull}9099[9099] - Filesets with multiple ingest pipelines added in {pull}8914[8914] only work with Elasticsearch >= 6.5.0 {pull}10001[10001] +- Add grok pattern to support redis 5.0.3 log timestamp. {issue}9819[9819] {pull}10033[10033] *Heartbeat* diff --git a/filebeat/module/redis/log/ingest/pipeline.json b/filebeat/module/redis/log/ingest/pipeline.json index 637b309c11ed..b3708623d234 100644 --- a/filebeat/module/redis/log/ingest/pipeline.json +++ b/filebeat/module/redis/log/ingest/pipeline.json @@ -5,12 +5,14 @@ "grok": { "field": "message", "patterns": [ - "(%{POSINT:redis.log.pid:long}:%{CHAR:redis.log.role} )?%{REDISTIMESTAMP:redis.log.timestamp} %{REDISLEVEL:redis.log.level} %{GREEDYDATA:redis.log.message}", + "(%{POSINT:redis.log.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{GREEDYDATA:redis.log.message}", "%{POSINT:redis.log.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:redis.log.message}" ], "pattern_definitions": { "CHAR": "[a-zA-Z]", - "REDISLEVEL": "[.\\-*#]" + "REDISLEVEL": "[.\\-*#]", + "REDISTIMESTAMP1": "%{MONTHDAY} %{MONTH} %{TIME}", + "REDISTIMESTAMP2": "%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}" } } }, @@ -62,6 +64,7 @@ "field": "redis.log.timestamp", "target_field": "@timestamp", "formats": [ + "dd MMM YYYY H:m:s.SSS", "dd MMM H:m:s.SSS", "dd MMM H:m:s", "UNIX" diff --git a/filebeat/module/redis/log/test/redis-5.0.3.log b/filebeat/module/redis/log/test/redis-5.0.3.log new file mode 100644 index 000000000000..e59b6eeab43e --- /dev/null +++ b/filebeat/module/redis/log/test/redis-5.0.3.log @@ -0,0 +1 @@ +26571:M 27 Dec 2018 11:19:18.874 * Synchronization with replica 10.114.208.18:6023 succeeded diff --git a/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json new file mode 100644 index 000000000000..50883fdf85e6 --- /dev/null +++ b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json @@ -0,0 +1,14 @@ +[ + { + "ecs.version": "1.0.0-beta2", + "event.dataset": "redis.log", + "event.module": "redis", + "fileset.name": "log", + "input.type": "log", + "log.level": "notice", + "log.offset": 0, + "message": "Synchronization with replica 10.114.208.18:6023 succeeded", + "process.pid": 26571, + "redis.log.role": "master" + } +] \ No newline at end of file From 8322fdd9fc278bea072c9f736af9a9d446a5b133 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 14 Jan 2019 10:04:50 -0700 Subject: [PATCH 2/3] Rerun GENEREATE=1 with 6.x test environment --- .../redis/log/test/redis-5.0.3.log-expected.json | 11 +++++------ filebeat/module/redis/log/test/test.log-expected.json | 9 +++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json index 50883fdf85e6..4d53fef10a3f 100644 --- a/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json +++ b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json @@ -1,14 +1,13 @@ [ { - "ecs.version": "1.0.0-beta2", "event.dataset": "redis.log", - "event.module": "redis", + "fileset.module": "redis", "fileset.name": "log", "input.type": "log", - "log.level": "notice", - "log.offset": 0, - "message": "Synchronization with replica 10.114.208.18:6023 succeeded", - "process.pid": 26571, + "offset": 0, + "prospector.type": "log", + "redis.log.message": "* Synchronization with replica 10.114.208.18:6023 succeeded", + "redis.log.pid": "26571", "redis.log.role": "master" } ] \ No newline at end of file diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json index 8d9449d6d82d..a09235916a6f 100644 --- a/filebeat/module/redis/log/test/test.log-expected.json +++ b/filebeat/module/redis/log/test/test.log-expected.json @@ -6,8 +6,7 @@ "input.type": "log", "offset": 0, "prospector.type": "log", - "redis.log.level": "notice", - "redis.log.message": "Saving the final RDB snapshot before exiting.", + "redis.log.message": "* Saving the final RDB snapshot before exiting.", "redis.log.pid": "98738", "redis.log.role": "master" }, @@ -18,8 +17,7 @@ "input.type": "log", "offset": 76, "prospector.type": "log", - "redis.log.level": "debug", - "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." + "redis.log.message": ". 0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." }, { "event.dataset": "redis.log", @@ -28,8 +26,7 @@ "input.type": "log", "offset": 165, "prospector.type": "log", - "redis.log.level": "notice", - "redis.log.message": "The server is now ready to accept connections on port 6379\"" + "redis.log.message": "31 May 04:32:08 * The server is now ready to accept connections on port 6379\"" }, { "event.dataset": "redis.log", From 59bf18caa41a1573ed5d794cdb009b40cdd28dc0 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 14 Jan 2019 13:50:26 -0700 Subject: [PATCH 3/3] Deleted log level by mistake --- filebeat/module/redis/log/ingest/pipeline.json | 2 +- .../module/redis/log/test/redis-5.0.3.log-expected.json | 3 ++- filebeat/module/redis/log/test/test.log-expected.json | 9 ++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/filebeat/module/redis/log/ingest/pipeline.json b/filebeat/module/redis/log/ingest/pipeline.json index b3708623d234..2701d3486559 100644 --- a/filebeat/module/redis/log/ingest/pipeline.json +++ b/filebeat/module/redis/log/ingest/pipeline.json @@ -5,7 +5,7 @@ "grok": { "field": "message", "patterns": [ - "(%{POSINT:redis.log.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{GREEDYDATA:redis.log.message}", + "(%{POSINT:redis.log.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{REDISLEVEL:redis.log.level} %{GREEDYDATA:redis.log.message}", "%{POSINT:redis.log.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:redis.log.message}" ], "pattern_definitions": { diff --git a/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json index 4d53fef10a3f..3aaafd38544f 100644 --- a/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json +++ b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json @@ -6,7 +6,8 @@ "input.type": "log", "offset": 0, "prospector.type": "log", - "redis.log.message": "* Synchronization with replica 10.114.208.18:6023 succeeded", + "redis.log.level": "notice", + "redis.log.message": "Synchronization with replica 10.114.208.18:6023 succeeded", "redis.log.pid": "26571", "redis.log.role": "master" } diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json index a09235916a6f..8d9449d6d82d 100644 --- a/filebeat/module/redis/log/test/test.log-expected.json +++ b/filebeat/module/redis/log/test/test.log-expected.json @@ -6,7 +6,8 @@ "input.type": "log", "offset": 0, "prospector.type": "log", - "redis.log.message": "* Saving the final RDB snapshot before exiting.", + "redis.log.level": "notice", + "redis.log.message": "Saving the final RDB snapshot before exiting.", "redis.log.pid": "98738", "redis.log.role": "master" }, @@ -17,7 +18,8 @@ "input.type": "log", "offset": 76, "prospector.type": "log", - "redis.log.message": ". 0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." + "redis.log.level": "debug", + "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." }, { "event.dataset": "redis.log", @@ -26,7 +28,8 @@ "input.type": "log", "offset": 165, "prospector.type": "log", - "redis.log.message": "31 May 04:32:08 * The server is now ready to accept connections on port 6379\"" + "redis.log.level": "notice", + "redis.log.message": "The server is now ready to accept connections on port 6379\"" }, { "event.dataset": "redis.log",