Cherry-pick to 6.x: Ignore timestamp in redis, haproxy and system filebeat module (#9855)

filebeat/module/haproxy/log/test/default.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-09-20T15:42:59.000Z",
         "event.dataset": "haproxy.log",
         "fileset.module": "haproxy",
         "fileset.name": "log",

filebeat/module/haproxy/log/test/haproxy.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-07-30T09:03:52.726Z",
         "event.dataset": "haproxy.log",
         "fileset.module": "haproxy",
         "fileset.name": "log",

filebeat/module/haproxy/log/test/tcplog.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-09-20T15:44:23.285Z",
         "event.dataset": "haproxy.log",
         "fileset.module": "haproxy",
         "fileset.name": "log",

filebeat/module/icinga/startup/test/test.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-12-28T19:56:51.277Z",
         "event.dataset": "icinga.startup",
         "fileset.module": "icinga",
         "fileset.name": "startup",
@@ -12,7 +11,6 @@
         "prospector.type": "log"
     },
     {
-        "@timestamp": "2018-12-28T19:56:51.277Z",
         "event.dataset": "icinga.startup",
         "fileset.module": "icinga",
         "fileset.name": "startup",

filebeat/module/redis/log/test/test.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-05-30T12:23:52.442Z",
         "event.dataset": "redis.log",
         "fileset.module": "redis",
         "fileset.name": "log",
@@ -13,7 +12,6 @@
         "redis.log.role": "master"
     },
     {
-        "@timestamp": "2018-05-30T10:05:20.000Z",
         "event.dataset": "redis.log",
         "fileset.module": "redis",
         "fileset.name": "log",
@@ -24,7 +22,6 @@
         "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
     },
     {
-        "@timestamp": "2018-05-31T04:32:08.000Z",
         "event.dataset": "redis.log",
         "fileset.module": "redis",
         "fileset.name": "log",
@@ -35,7 +32,6 @@
         "redis.log.message": "The server is now ready to accept connections on port 6379\""
     },
     {
-        "@timestamp": "2017-05-30T10:57:24.000Z",
         "event.dataset": "redis.log",
         "fileset.module": "redis",
         "fileset.name": "log",

filebeat/module/system/auth/test/test.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-02-21T21:54:44.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -18,7 +17,6 @@
         "system.auth.user": "vagrant"
     },
     {
-        "@timestamp": "2018-02-23T00:13:35.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -35,7 +33,6 @@
         "system.auth.user": "vagrant"
     },
     {
-        "@timestamp": "2018-02-21T21:56:12.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -50,7 +47,6 @@
         "system.auth.user": "test"
     },
     {
-        "@timestamp": "2018-02-20T08:35:22.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -73,7 +69,6 @@
         "system.auth.user": "root"
     },
     {
-        "@timestamp": "2018-02-21T23:35:33.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -89,7 +84,6 @@
         "system.auth.user": "vagrant"
     },
     {
-        "@timestamp": "2018-02-19T15:30:04.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -102,7 +96,6 @@
         "system.auth.timestamp": "Feb 19 15:30:04"
     },
     {
-        "@timestamp": "2018-02-23T00:08:48.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -118,7 +111,6 @@
         "system.auth.user": "vagrant"
     },
     {
-        "@timestamp": "2018-02-24T00:13:02.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -135,7 +127,6 @@
         "system.auth.user": "tsg"
     },
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",
@@ -149,7 +140,6 @@
         "system.auth.timestamp": "Feb 22 11:47:05"
     },
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z",
         "event.dataset": "system.auth",
         "fileset.module": "system",
         "fileset.name": "auth",

filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z",
         "event.dataset": "system.syslog",
         "fileset.module": "system",
         "fileset.name": "syslog",
@@ -17,7 +16,6 @@
         "system.syslog.timestamp": "Dec 13 11:35:28"
     },
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z",
         "event.dataset": "system.syslog",
         "fileset.module": "system",
         "fileset.name": "syslog",
@@ -31,7 +29,6 @@
         "system.syslog.timestamp": "Dec 13 11:35:28"
     },
     {
-        "@timestamp": "2018-04-04T03:39:57.000Z",
         "event.dataset": "system.syslog",
         "fileset.module": "system",
         "fileset.name": "syslog",

filebeat/tests/system/test_modules.py

@@ -176,11 +176,6 @@ def _test_expected_events(self, test_file, objects):
                 obj = self.flatten_object(obj, {}, "")
                 clean_keys(obj)
 
-                # Remove timestamp for comparison where timestamp is not part of the log line
-                if obj["fileset.module"] == "icinga" and obj["fileset.name"] == "startup":
-                    delete_key(obj, "@timestamp")
-                    delete_key(ev, "@timestamp")
-
                 if ev == obj:
                     found = True
                     break
@@ -200,6 +195,10 @@ def clean_keys(obj):
     for key in host_keys + time_keys + other_keys:
         delete_key(obj, key)
 
+    # Remove timestamp for comparison where timestamp is not part of the log line
+    if obj["event.dataset"] in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog"]:
+        delete_key(obj, "@timestamp")
+
 
 def delete_key(obj, key):
     if key in obj: