From f3740f8c5f4e5296aa5e4a0004cb0fdefed5d714 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 2 Jan 2019 16:56:21 -0700 Subject: [PATCH 1/5] Ignore timestamp in redis, haproxy and system filebeat module (#9855) * Ignore timestamp in redis, haproxy and system filebeat module * Fixing formatting (cherry picked from commit b9a20f7ba0b42c98c223a9d34216957a1922372e) --- filebeat/tests/system/test_modules.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 4a3b39aa28a2..f210104ffb0a 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -169,6 +169,7 @@ def _test_expected_events(self, test_file, objects): for ev in expected: found = False + clean_keys(ev) for obj in objects: # Flatten objects for easier comparing @@ -199,6 +200,12 @@ def clean_keys(obj): for key in host_keys + time_keys + other_keys: delete_key(obj, key) + # Remove timestamp for comparison where timestamp is not part of the log line + if (obj["event.module"] == "icinga" and obj["event.dataset"] == "startup") or \ + (obj["event.module"] in ["redis", "haproxy"] and obj["event.dataset"] == "log") or \ + (obj["event.module"] == "system" and obj["event.dataset"] in ["auth", "syslog"]): + delete_key(obj, "@timestamp") + def delete_key(obj, key): if key in obj: From b3ef624230e92e4c1e31197f089072ee9cd2669c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 2 Jan 2019 19:20:25 -0700 Subject: [PATCH 2/5] Use event.dataset in if statement --- filebeat/tests/system/test_modules.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index f210104ffb0a..b8f4f8c4bae5 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -201,9 +201,7 @@ def clean_keys(obj): delete_key(obj, key) # Remove timestamp for comparison where timestamp is not part of the log line - if (obj["event.module"] == "icinga" and obj["event.dataset"] == "startup") or \ - (obj["event.module"] in ["redis", "haproxy"] and obj["event.dataset"] == "log") or \ - (obj["event.module"] == "system" and obj["event.dataset"] in ["auth", "syslog"]): + if obj["event.dataset"] in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog"]: delete_key(obj, "@timestamp") From 8e5c294d3328e4fb66a3919314f49188707d8032 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 12 Dec 2018 13:38:04 +0100 Subject: [PATCH 3/5] Update golden files for icinga/startup test to exclude timestamp (#9506) The icing/startup log file does not contain a timestamp. Because of this the timestamp from filebeat is taken. During the generation of golden files still a timestamp was added but every skipped on comparison. Instead now the timestamp is not added to the generated file anymore to now show a diff each time GENERATE is run. --- .../icinga/startup/test/test.log-expected.json | 18 +++++++----------- filebeat/tests/system/test_modules.py | 6 +----- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index be7a016f527c..c17b42388d5a 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -1,10 +1,8 @@ [ { - "@timestamp": "2018-12-12T11:22:05.182Z", - "event.dataset": "icinga.startup", - "fileset.module": "icinga", - "fileset.name": "startup", - "icinga.startup.facility": "cli", + "event.dataset": "startup", + "event.module": "icinga", + "icinga.startup.facility": "cli", "icinga.startup.message": "Icinga application loader (version: r2.6.3-1)", "icinga.startup.severity": "information", "input.type": "log", @@ -12,15 +10,13 @@ "prospector.type": "log" }, { - "@timestamp": "2018-12-12T11:22:05.182Z", - "event.dataset": "icinga.startup", - "fileset.module": "icinga", - "fileset.name": "startup", - "icinga.startup.facility": "cli", + "event.dataset": "startup", + "event.module": "icinga", + "icinga.startup.facility": "cli", "icinga.startup.message": "Loading configuration file(s).", "icinga.startup.severity": "information", "input.type": "log", "offset": 63, "prospector.type": "log" } -] \ No newline at end of file +] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index b8f4f8c4bae5..ff0ab6f6c620 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -159,6 +159,7 @@ def _test_expected_events(self, test_file, objects): for k, obj in enumerate(objects): objects[k] = self.flatten_object(obj, {}, "") clean_keys(objects[k]) + json.dump(objects, f, indent=4, sort_keys=True) with open(test_file + "-expected.json", "r") as f: @@ -176,11 +177,6 @@ def _test_expected_events(self, test_file, objects): obj = self.flatten_object(obj, {}, "") clean_keys(obj) - # Remove timestamp for comparison where timestamp is not part of the log line - if obj["fileset.module"] == "icinga" and obj["fileset.name"] == "startup": - delete_key(obj, "@timestamp") - delete_key(ev, "@timestamp") - if ev == obj: found = True break From ae8131ca23901997f1efa4805c3be3db666f2cff Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 3 Jan 2019 16:46:38 +0100 Subject: [PATCH 4/5] Regenerate expectations for logs without timestamps (#9862) --- .../log/test/default.log-expected.json | 41 ++- .../log/test/haproxy.log-expected.json | 38 +-- .../haproxy/log/test/tcplog.log-expected.json | 55 ++-- .../redis/log/test/test.log-expected.json | 72 ++--- .../system/auth/test/test.log-expected.json | 287 ++++++++---------- .../darwin-syslog-sample.log-expected.json | 50 ++- filebeat/tests/system/test_modules.py | 1 - 7 files changed, 248 insertions(+), 296 deletions(-) diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json index 0ee6f23b074d..5433e48b8244 100644 --- a/filebeat/module/haproxy/log/test/default.log-expected.json +++ b/filebeat/module/haproxy/log/test/default.log-expected.json @@ -1,24 +1,23 @@ [ { - "@timestamp": "2018-09-20T15:42:59.000Z", - "event.dataset": "haproxy.log", - "fileset.module": "haproxy", - "fileset.name": "log", - "haproxy.client.ip": "1.2.3.4", - "haproxy.client.port": "40780", - "haproxy.destination.ip": "1.2.3.4", - "haproxy.destination.port": "5000", - "haproxy.frontend_name": "main", - "haproxy.geoip.continent_name": "North America", - "haproxy.geoip.country_iso_code": "US", - "haproxy.geoip.location.lat": 37.751, - "haproxy.geoip.location.lon": -97.822, - "haproxy.mode": "HTTP", - "haproxy.pid": "24551", - "haproxy.process_name": "haproxy", - "haproxy.source": "1.2.3.4", - "input.type": "log", - "offset": 0, - "prospector.type": "log" + "destination.ip": "1.2.3.4", + "destination.port": 5000, + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "haproxy", + "haproxy.client.ip": "1.2.3.4", + "haproxy.frontend_name": "main", + "haproxy.mode": "HTTP", + "haproxy.source": "1.2.3.4", + "input.type": "log", + "log.offset": 0, + "process.name": "haproxy", + "process.pid": 24551, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "1.2.3.4", + "source.port": 40780 } -] \ No newline at end of file +] diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index 96d6630de4ee..44e98feb0c95 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -1,26 +1,20 @@ [ { - "@timestamp": "2018-07-30T09:03:52.726Z", - "event.dataset": "haproxy.log", - "fileset.module": "haproxy", - "fileset.name": "log", - "haproxy.backend_name": "docs_microservice", - "haproxy.backend_queue": 0, - "haproxy.bytes_read": 168, - "haproxy.client.ip": "1.2.3.4", - "haproxy.client.port": 38862, - "haproxy.connection_wait_time_ms": 1, - "haproxy.connections.active": 6, - "haproxy.connections.backend": 0, - "haproxy.connections.frontend": 6, - "haproxy.connections.retries": 0, - "haproxy.connections.server": 0, - "haproxy.frontend_name": "incoming~", - "haproxy.geoip.continent_name": "North America", - "haproxy.geoip.country_iso_code": "US", - "haproxy.geoip.location.lat": 37.751, - "haproxy.geoip.location.lon": -97.822, - "haproxy.http.request.captured_cookie": "-", + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "haproxy", + "haproxy.backend_name": "docs_microservice", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 168, + "haproxy.client.ip": "1.2.3.4", + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 6, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 6, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "incoming~", + "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.captured_headers": [ "docs.example.internal" ], @@ -41,4 +35,4 @@ "offset": 0, "prospector.type": "log" } -] \ No newline at end of file +] diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index b0a2ff08d925..b1dbebcc80e4 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -1,31 +1,30 @@ [ { - "@timestamp": "2018-09-20T15:44:23.285Z", - "event.dataset": "haproxy.log", - "fileset.module": "haproxy", - "fileset.name": "log", - "haproxy.backend_name": "app", - "haproxy.backend_queue": 0, - "haproxy.bytes_read": 212, - "haproxy.client.ip": "127.0.0.1", - "haproxy.client.port": 40962, - "haproxy.connection_wait_time_ms": -1, - "haproxy.connections.active": 1, - "haproxy.connections.backend": 0, - "haproxy.connections.frontend": 1, - "haproxy.connections.retries": 0, - "haproxy.connections.server": 0, - "haproxy.frontend_name": "main", - "haproxy.pid": 25457, - "haproxy.process_name": "haproxy", - "haproxy.server_name": "", - "haproxy.server_queue": 0, - "haproxy.source": "127.0.0.1", - "haproxy.tcp.processing_time_ms": 0, - "haproxy.termination_state": "SC", - "haproxy.total_waiting_time_ms": -1, - "input.type": "log", - "offset": 0, - "prospector.type": "log" + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "haproxy", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 212, + "haproxy.client.ip": "127.0.0.1", + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "main", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.source": "127.0.0.1", + "haproxy.tcp.processing_time_ms": 0, + "haproxy.termination_state": "SC", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "log.offset": 0, + "process.name": "haproxy", + "process.pid": 25457, + "source.ip": "127.0.0.1", + "source.port": 40962 } -] \ No newline at end of file +] diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json index 649114bbf6eb..f065cf5dd796 100644 --- a/filebeat/module/redis/log/test/test.log-expected.json +++ b/filebeat/module/redis/log/test/test.log-expected.json @@ -1,48 +1,40 @@ [ { - "@timestamp": "2018-05-30T12:23:52.442Z", - "event.dataset": "redis.log", - "fileset.module": "redis", - "fileset.name": "log", - "input.type": "log", - "offset": 0, - "prospector.type": "log", - "redis.log.level": "notice", - "redis.log.message": "Saving the final RDB snapshot before exiting.", - "redis.log.pid": "98738", + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "redis", + "input.type": "log", + "log.level": "notice", + "log.offset": 0, + "message": "Saving the final RDB snapshot before exiting.", + "process.pid": 98738, "redis.log.role": "master" }, { - "@timestamp": "2018-05-30T10:05:20.000Z", - "event.dataset": "redis.log", - "fileset.module": "redis", - "fileset.name": "log", - "input.type": "log", - "offset": 76, - "prospector.type": "log", - "redis.log.level": "debug", - "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "redis", + "input.type": "log", + "log.level": "debug", + "log.offset": 76, + "message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." + }, { - "@timestamp": "2018-05-31T04:32:08.000Z", - "event.dataset": "redis.log", - "fileset.module": "redis", - "fileset.name": "log", - "input.type": "log", - "offset": 165, - "prospector.type": "log", - "redis.log.level": "notice", - "redis.log.message": "The server is now ready to accept connections on port 6379\"" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "redis", + "input.type": "log", + "log.level": "notice", + "log.offset": 165, + "message": "The server is now ready to accept connections on port 6379\"" + }, { - "@timestamp": "2017-05-30T10:57:24.000Z", - "event.dataset": "redis.log", - "fileset.module": "redis", - "fileset.name": "log", - "input.type": "log", - "offset": 250, - "prospector.type": "log", - "redis.log.message": "Received SIGINT scheduling shutdown...", - "redis.log.pid": "5092" + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "redis", + "input.type": "log", + "log.offset": 250, + "message": "Received SIGINT scheduling shutdown...", + "process.pid": 5092 } -] \ No newline at end of file +] diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index 83018d65dbe2..c3482656ef6d 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -1,168 +1,143 @@ [ { - "@timestamp": "2018-02-21T21:54:44.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 0, - "prospector.type": "log", - "system.auth.hostname": "localhost", - "system.auth.pid": "3402", - "system.auth.ssh.event": "Accepted", - "system.auth.ssh.ip": "10.0.2.2", - "system.auth.ssh.method": "publickey", - "system.auth.ssh.port": "63673", - "system.auth.ssh.signature": "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", - "system.auth.timestamp": "Feb 21 21:54:44", - "system.auth.user": "vagrant" - }, + "ecs.version": "1.0.0-beta2", + "event.action": "Accepted", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 0, + "process.pid": 3402, + "source.ip": "10.0.2.2", + "source.port": 63673, + "system.auth.ssh.method": "publickey", + "system.auth.ssh.signature": "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", + "user.name": "vagrant" + }, { - "@timestamp": "2018-02-23T00:13:35.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 152, - "prospector.type": "log", - "system.auth.hostname": "localhost", - "system.auth.pid": "7483", - "system.auth.ssh.event": "Accepted", - "system.auth.ssh.ip": "192.168.33.1", - "system.auth.ssh.method": "password", - "system.auth.ssh.port": "58803", - "system.auth.timestamp": "Feb 23 00:13:35", - "system.auth.user": "vagrant" - }, + "ecs.version": "1.0.0-beta2", + "event.action": "Accepted", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 152, + "process.pid": 7483, + "source.ip": "192.168.33.1", + "source.port": 58803, + "system.auth.ssh.method": "password", + "user.name": "vagrant" + }, { - "@timestamp": "2018-02-21T21:56:12.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 254, - "prospector.type": "log", - "system.auth.hostname": "localhost", - "system.auth.pid": "3430", - "system.auth.ssh.event": "Invalid", - "system.auth.ssh.ip": "10.0.2.2", - "system.auth.timestamp": "Feb 21 21:56:12", - "system.auth.user": "test" - }, + "ecs.version": "1.0.0-beta2", + "event.action": "Invalid", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 254, + "process.pid": 3430, + "source.ip": "10.0.2.2", + "user.name": "test" + }, { - "@timestamp": "2018-02-20T08:35:22.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 324, - "prospector.type": "log", - "system.auth.hostname": "slave22", - "system.auth.pid": "5774", - "system.auth.ssh.event": "Failed", - "system.auth.ssh.geoip.continent_name": "Asia", - "system.auth.ssh.geoip.country_iso_code": "CN", - "system.auth.ssh.geoip.location.lat": 23.1167, - "system.auth.ssh.geoip.location.lon": 113.25, - "system.auth.ssh.geoip.region_iso_code": "CN-GD", - "system.auth.ssh.geoip.region_name": "Guangdong", - "system.auth.ssh.ip": "116.31.116.24", - "system.auth.ssh.method": "password", - "system.auth.ssh.port": "29160", - "system.auth.timestamp": "Feb 20 08:35:22", - "system.auth.user": "root" - }, + "ecs.version": "1.0.0-beta2", + "event.action": "Failed", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "slave22", + "input.type": "log", + "log.offset": 324, + "process.pid": 5774, + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.location.lat": 23.1167, + "source.geo.location.lon": 113.25, + "source.geo.region_iso_code": "CN-GD", + "source.geo.region_name": "Guangdong", + "source.ip": "116.31.116.24", + "source.port": 29160, + "system.auth.ssh.method": "password", + "user.name": "root" + }, { - "@timestamp": "2018-02-21T23:35:33.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 420, - "prospector.type": "log", - "system.auth.hostname": "localhost", - "system.auth.sudo.command": "/bin/ls", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/0", - "system.auth.sudo.user": "root", - "system.auth.timestamp": "Feb 21 23:35:33", - "system.auth.user": "vagrant" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 420, + "system.auth.sudo.command": "/bin/ls", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/0", + "system.auth.sudo.user": "root", + "user.name": "vagrant" + }, { - "@timestamp": "2018-02-19T15:30:04.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 522, - "prospector.type": "log", - "system.auth.hostname": "slave22", - "system.auth.pid": "18406", - "system.auth.ssh.dropped_ip": "123.57.245.163", - "system.auth.timestamp": "Feb 19 15:30:04" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "slave22", + "input.type": "log", + "log.offset": 522, + "process.pid": 18406, + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.location.lat": 34.7725, + "source.geo.location.lon": 113.7266, + "source.ip": "123.57.245.163", + "system.auth.ssh.dropped_ip": "123.57.245.163" + }, { - "@timestamp": "2018-02-23T00:08:48.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 617, - "prospector.type": "log", - "system.auth.hostname": "localhost", - "system.auth.sudo.command": "/bin/cat /var/log/secure", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/1", - "system.auth.sudo.user": "root", - "system.auth.timestamp": "Feb 23 00:08:48", - "system.auth.user": "vagrant" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 617, + "system.auth.sudo.command": "/bin/cat /var/log/secure", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/1", + "system.auth.sudo.user": "root", + "user.name": "vagrant" + }, { - "@timestamp": "2018-02-24T00:13:02.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 736, - "prospector.type": "log", - "system.auth.hostname": "precise32", - "system.auth.sudo.command": "/bin/ls", - "system.auth.sudo.error": "user NOT in sudoers", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/1", - "system.auth.sudo.user": "root", - "system.auth.timestamp": "Feb 24 00:13:02", - "system.auth.user": "tsg" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "auth", + "event.module": "system", + "host.hostname": "precise32", + "input.type": "log", + "log.offset": 736, + "system.auth.sudo.command": "/bin/ls", + "system.auth.sudo.error": "user NOT in sudoers", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/1", + "system.auth.sudo.user": "root", + "user.name": "tsg" + }, { - "@timestamp": "2018-02-22T11:47:05.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 861, - "prospector.type": "log", - "system.auth.groupadd.gid": "48", - "system.auth.groupadd.name": "apache", - "system.auth.hostname": "localhost", - "system.auth.pid": "6991", - "system.auth.timestamp": "Feb 22 11:47:05" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "auth", + "event.module": "system", + "group.id": 48, + "group.name": "apache", + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 861, + "process.pid": 6991 + }, { - "@timestamp": "2018-02-22T11:47:05.000Z", - "event.dataset": "system.auth", - "fileset.module": "system", - "fileset.name": "auth", - "input.type": "log", - "offset": 934, - "prospector.type": "log", - "system.auth.hostname": "localhost", - "system.auth.pid": "6995", - "system.auth.timestamp": "Feb 22 11:47:05", - "system.auth.useradd.gid": "48", - "system.auth.useradd.home": "/usr/share/httpd", - "system.auth.useradd.name": "apache", - "system.auth.useradd.shell": "/sbin/nologin", - "system.auth.useradd.uid": "48" + "ecs.version": "1.0.0-beta2", + "event.dataset": "auth", + "event.module": "system", + "group.id": 48, + "host.hostname": "localhost", + "input.type": "log", + "log.offset": 934, + "process.pid": 6995, + "system.auth.useradd.home": "/usr/share/httpd", + "system.auth.useradd.shell": "/sbin/nologin", + "user.id": 48, + "user.name": "apache" } -] \ No newline at end of file +] diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index a17409df62b7..e929fdf03a5b 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -1,10 +1,10 @@ [ { - "@timestamp": "2018-12-13T11:35:28.000Z", - "event.dataset": "system.syslog", - "fileset.module": "system", - "fileset.name": "syslog", - "input.type": "log", + "ecs.version": "1.0.0-beta2", + "event.dataset": "syslog", + "event.module": "system", + "host.hostname": "a-mac-with-esc-key", + "input.type": "log", "log.flags": [ "multiline" ], @@ -17,28 +17,22 @@ "system.syslog.timestamp": "Dec 13 11:35:28" }, { - "@timestamp": "2018-12-13T11:35:28.000Z", - "event.dataset": "system.syslog", - "fileset.module": "system", - "fileset.name": "syslog", - "input.type": "log", - "offset": 907, - "prospector.type": "log", - "system.syslog.hostname": "a-mac-with-esc-key", - "system.syslog.message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", - "system.syslog.pid": "21412", - "system.syslog.program": "GoogleSoftwareUpdateAgent", - "system.syslog.timestamp": "Dec 13 11:35:28" - }, + "ecs.version": "1.0.0-beta2", + "event.dataset": "syslog", + "event.module": "system", + "host.hostname": "a-mac-with-esc-key", + "input.type": "log", + "log.offset": 907, + "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", + "process.name": "GoogleSoftwareUpdateAgent", + "process.pid": 21412 + }, { - "@timestamp": "2018-04-04T03:39:57.000Z", - "event.dataset": "system.syslog", - "fileset.module": "system", - "fileset.name": "syslog", - "input.type": "log", - "offset": 1176, - "prospector.type": "log", - "system.syslog.message": "--- last message repeated 1 time ---", - "system.syslog.timestamp": "Apr 4 03:39:57" + "ecs.version": "1.0.0-beta2", + "event.dataset": "syslog", + "event.module": "system", + "input.type": "log", + "log.offset": 1176, + "message": "--- last message repeated 1 time ---" } -] \ No newline at end of file +] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index ff0ab6f6c620..ba5b4fd785ac 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -170,7 +170,6 @@ def _test_expected_events(self, test_file, objects): for ev in expected: found = False - clean_keys(ev) for obj in objects: # Flatten objects for easier comparing From 8b8039efaacc43b4166fc17abd23b5ec28e393ce Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 3 Jan 2019 10:01:38 -0700 Subject: [PATCH 5/5] Rerun GENERATE=1 to update expected json files --- .../log/test/default.log-expected.json | 40 +-- .../log/test/haproxy.log-expected.json | 37 ++- .../haproxy/log/test/tcplog.log-expected.json | 54 ++-- .../startup/test/test.log-expected.json | 16 +- .../redis/log/test/test.log-expected.json | 68 +++-- .../system/auth/test/test.log-expected.json | 277 +++++++++--------- .../darwin-syslog-sample.log-expected.json | 47 +-- 7 files changed, 284 insertions(+), 255 deletions(-) diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json index 5433e48b8244..5771d863b29b 100644 --- a/filebeat/module/haproxy/log/test/default.log-expected.json +++ b/filebeat/module/haproxy/log/test/default.log-expected.json @@ -1,23 +1,23 @@ [ { - "destination.ip": "1.2.3.4", - "destination.port": 5000, - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "haproxy", - "haproxy.client.ip": "1.2.3.4", - "haproxy.frontend_name": "main", - "haproxy.mode": "HTTP", - "haproxy.source": "1.2.3.4", - "input.type": "log", - "log.offset": 0, - "process.name": "haproxy", - "process.pid": 24551, - "source.geo.continent_name": "North America", - "source.geo.country_iso_code": "US", - "source.geo.location.lat": 37.751, - "source.geo.location.lon": -97.822, - "source.ip": "1.2.3.4", - "source.port": 40780 + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.client.ip": "1.2.3.4", + "haproxy.client.port": "40780", + "haproxy.destination.ip": "1.2.3.4", + "haproxy.destination.port": "5000", + "haproxy.frontend_name": "main", + "haproxy.geoip.continent_name": "North America", + "haproxy.geoip.country_iso_code": "US", + "haproxy.geoip.location.lat": 37.751, + "haproxy.geoip.location.lon": -97.822, + "haproxy.mode": "HTTP", + "haproxy.pid": "24551", + "haproxy.process_name": "haproxy", + "haproxy.source": "1.2.3.4", + "input.type": "log", + "offset": 0, + "prospector.type": "log" } -] +] \ No newline at end of file diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index 44e98feb0c95..6c4b0d4fa8e2 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -1,20 +1,25 @@ [ { - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "haproxy", - "haproxy.backend_name": "docs_microservice", - "haproxy.backend_queue": 0, - "haproxy.bytes_read": 168, - "haproxy.client.ip": "1.2.3.4", - "haproxy.connection_wait_time_ms": 1, - "haproxy.connections.active": 6, - "haproxy.connections.backend": 0, - "haproxy.connections.frontend": 6, - "haproxy.connections.retries": 0, - "haproxy.connections.server": 0, - "haproxy.frontend_name": "incoming~", - "haproxy.http.request.captured_cookie": "-", + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "docs_microservice", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 168, + "haproxy.client.ip": "1.2.3.4", + "haproxy.client.port": 38862, + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 6, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 6, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "incoming~", + "haproxy.geoip.continent_name": "North America", + "haproxy.geoip.country_iso_code": "US", + "haproxy.geoip.location.lat": 37.751, + "haproxy.geoip.location.lon": -97.822, + "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.captured_headers": [ "docs.example.internal" ], @@ -35,4 +40,4 @@ "offset": 0, "prospector.type": "log" } -] +] \ No newline at end of file diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index b1dbebcc80e4..3c84eec09b15 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -1,30 +1,30 @@ [ { - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "haproxy", - "haproxy.backend_name": "app", - "haproxy.backend_queue": 0, - "haproxy.bytes_read": 212, - "haproxy.client.ip": "127.0.0.1", - "haproxy.connection_wait_time_ms": -1, - "haproxy.connections.active": 1, - "haproxy.connections.backend": 0, - "haproxy.connections.frontend": 1, - "haproxy.connections.retries": 0, - "haproxy.connections.server": 0, - "haproxy.frontend_name": "main", - "haproxy.server_name": "", - "haproxy.server_queue": 0, - "haproxy.source": "127.0.0.1", - "haproxy.tcp.processing_time_ms": 0, - "haproxy.termination_state": "SC", - "haproxy.total_waiting_time_ms": -1, - "input.type": "log", - "log.offset": 0, - "process.name": "haproxy", - "process.pid": 25457, - "source.ip": "127.0.0.1", - "source.port": 40962 + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 212, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 40962, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "main", + "haproxy.pid": 25457, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.source": "127.0.0.1", + "haproxy.tcp.processing_time_ms": 0, + "haproxy.termination_state": "SC", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 0, + "prospector.type": "log" } -] +] \ No newline at end of file diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index c17b42388d5a..04e34bd0814b 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -1,8 +1,9 @@ [ { - "event.dataset": "startup", - "event.module": "icinga", - "icinga.startup.facility": "cli", + "event.dataset": "icinga.startup", + "fileset.module": "icinga", + "fileset.name": "startup", + "icinga.startup.facility": "cli", "icinga.startup.message": "Icinga application loader (version: r2.6.3-1)", "icinga.startup.severity": "information", "input.type": "log", @@ -10,13 +11,14 @@ "prospector.type": "log" }, { - "event.dataset": "startup", - "event.module": "icinga", - "icinga.startup.facility": "cli", + "event.dataset": "icinga.startup", + "fileset.module": "icinga", + "fileset.name": "startup", + "icinga.startup.facility": "cli", "icinga.startup.message": "Loading configuration file(s).", "icinga.startup.severity": "information", "input.type": "log", "offset": 63, "prospector.type": "log" } -] +] \ No newline at end of file diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json index f065cf5dd796..1f5766993084 100644 --- a/filebeat/module/redis/log/test/test.log-expected.json +++ b/filebeat/module/redis/log/test/test.log-expected.json @@ -1,40 +1,44 @@ [ { - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "redis", - "input.type": "log", - "log.level": "notice", - "log.offset": 0, - "message": "Saving the final RDB snapshot before exiting.", - "process.pid": 98738, + "event.dataset": "redis.log", + "fileset.module": "redis", + "fileset.name": "log", + "input.type": "log", + "offset": 0, + "prospector.type": "log", + "redis.log.level": "notice", + "redis.log.message": "Saving the final RDB snapshot before exiting.", + "redis.log.pid": "98738", "redis.log.role": "master" }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "redis", - "input.type": "log", - "log.level": "debug", - "log.offset": 76, - "message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." - }, + "event.dataset": "redis.log", + "fileset.module": "redis", + "fileset.name": "log", + "input.type": "log", + "offset": 76, + "prospector.type": "log", + "redis.log.level": "debug", + "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "redis", - "input.type": "log", - "log.level": "notice", - "log.offset": 165, - "message": "The server is now ready to accept connections on port 6379\"" - }, + "event.dataset": "redis.log", + "fileset.module": "redis", + "fileset.name": "log", + "input.type": "log", + "offset": 165, + "prospector.type": "log", + "redis.log.level": "notice", + "redis.log.message": "The server is now ready to accept connections on port 6379\"" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "redis", - "input.type": "log", - "log.offset": 250, - "message": "Received SIGINT scheduling shutdown...", - "process.pid": 5092 + "event.dataset": "redis.log", + "fileset.module": "redis", + "fileset.name": "log", + "input.type": "log", + "offset": 250, + "prospector.type": "log", + "redis.log.message": "Received SIGINT scheduling shutdown...", + "redis.log.pid": "5092" } -] +] \ No newline at end of file diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index c3482656ef6d..9b003fd4cdd8 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -1,143 +1,158 @@ [ { - "ecs.version": "1.0.0-beta2", - "event.action": "Accepted", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 0, - "process.pid": 3402, - "source.ip": "10.0.2.2", - "source.port": 63673, - "system.auth.ssh.method": "publickey", - "system.auth.ssh.signature": "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", - "user.name": "vagrant" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 0, + "prospector.type": "log", + "system.auth.hostname": "localhost", + "system.auth.pid": "3402", + "system.auth.ssh.event": "Accepted", + "system.auth.ssh.ip": "10.0.2.2", + "system.auth.ssh.method": "publickey", + "system.auth.ssh.port": "63673", + "system.auth.ssh.signature": "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", + "system.auth.timestamp": "Feb 21 21:54:44", + "system.auth.user": "vagrant" + }, { - "ecs.version": "1.0.0-beta2", - "event.action": "Accepted", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 152, - "process.pid": 7483, - "source.ip": "192.168.33.1", - "source.port": 58803, - "system.auth.ssh.method": "password", - "user.name": "vagrant" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 152, + "prospector.type": "log", + "system.auth.hostname": "localhost", + "system.auth.pid": "7483", + "system.auth.ssh.event": "Accepted", + "system.auth.ssh.ip": "192.168.33.1", + "system.auth.ssh.method": "password", + "system.auth.ssh.port": "58803", + "system.auth.timestamp": "Feb 23 00:13:35", + "system.auth.user": "vagrant" + }, { - "ecs.version": "1.0.0-beta2", - "event.action": "Invalid", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 254, - "process.pid": 3430, - "source.ip": "10.0.2.2", - "user.name": "test" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 254, + "prospector.type": "log", + "system.auth.hostname": "localhost", + "system.auth.pid": "3430", + "system.auth.ssh.event": "Invalid", + "system.auth.ssh.ip": "10.0.2.2", + "system.auth.timestamp": "Feb 21 21:56:12", + "system.auth.user": "test" + }, { - "ecs.version": "1.0.0-beta2", - "event.action": "Failed", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "slave22", - "input.type": "log", - "log.offset": 324, - "process.pid": 5774, - "source.geo.continent_name": "Asia", - "source.geo.country_iso_code": "CN", - "source.geo.location.lat": 23.1167, - "source.geo.location.lon": 113.25, - "source.geo.region_iso_code": "CN-GD", - "source.geo.region_name": "Guangdong", - "source.ip": "116.31.116.24", - "source.port": 29160, - "system.auth.ssh.method": "password", - "user.name": "root" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 324, + "prospector.type": "log", + "system.auth.hostname": "slave22", + "system.auth.pid": "5774", + "system.auth.ssh.event": "Failed", + "system.auth.ssh.geoip.continent_name": "Asia", + "system.auth.ssh.geoip.country_iso_code": "CN", + "system.auth.ssh.geoip.location.lat": 23.1167, + "system.auth.ssh.geoip.location.lon": 113.25, + "system.auth.ssh.geoip.region_iso_code": "CN-GD", + "system.auth.ssh.geoip.region_name": "Guangdong", + "system.auth.ssh.ip": "116.31.116.24", + "system.auth.ssh.method": "password", + "system.auth.ssh.port": "29160", + "system.auth.timestamp": "Feb 20 08:35:22", + "system.auth.user": "root" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 420, - "system.auth.sudo.command": "/bin/ls", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/0", - "system.auth.sudo.user": "root", - "user.name": "vagrant" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 420, + "prospector.type": "log", + "system.auth.hostname": "localhost", + "system.auth.sudo.command": "/bin/ls", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/0", + "system.auth.sudo.user": "root", + "system.auth.timestamp": "Feb 21 23:35:33", + "system.auth.user": "vagrant" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "slave22", - "input.type": "log", - "log.offset": 522, - "process.pid": 18406, - "source.geo.continent_name": "Asia", - "source.geo.country_iso_code": "CN", - "source.geo.location.lat": 34.7725, - "source.geo.location.lon": 113.7266, - "source.ip": "123.57.245.163", - "system.auth.ssh.dropped_ip": "123.57.245.163" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 522, + "prospector.type": "log", + "system.auth.hostname": "slave22", + "system.auth.pid": "18406", + "system.auth.ssh.dropped_ip": "123.57.245.163", + "system.auth.timestamp": "Feb 19 15:30:04" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 617, - "system.auth.sudo.command": "/bin/cat /var/log/secure", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/1", - "system.auth.sudo.user": "root", - "user.name": "vagrant" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 617, + "prospector.type": "log", + "system.auth.hostname": "localhost", + "system.auth.sudo.command": "/bin/cat /var/log/secure", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/1", + "system.auth.sudo.user": "root", + "system.auth.timestamp": "Feb 23 00:08:48", + "system.auth.user": "vagrant" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "auth", - "event.module": "system", - "host.hostname": "precise32", - "input.type": "log", - "log.offset": 736, - "system.auth.sudo.command": "/bin/ls", - "system.auth.sudo.error": "user NOT in sudoers", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/1", - "system.auth.sudo.user": "root", - "user.name": "tsg" - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 736, + "prospector.type": "log", + "system.auth.hostname": "precise32", + "system.auth.sudo.command": "/bin/ls", + "system.auth.sudo.error": "user NOT in sudoers", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/1", + "system.auth.sudo.user": "root", + "system.auth.timestamp": "Feb 24 00:13:02", + "system.auth.user": "tsg" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "auth", - "event.module": "system", - "group.id": 48, - "group.name": "apache", - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 861, - "process.pid": 6991 - }, + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 861, + "prospector.type": "log", + "system.auth.groupadd.gid": "48", + "system.auth.groupadd.name": "apache", + "system.auth.hostname": "localhost", + "system.auth.pid": "6991", + "system.auth.timestamp": "Feb 22 11:47:05" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "auth", - "event.module": "system", - "group.id": 48, - "host.hostname": "localhost", - "input.type": "log", - "log.offset": 934, - "process.pid": 6995, - "system.auth.useradd.home": "/usr/share/httpd", - "system.auth.useradd.shell": "/sbin/nologin", - "user.id": 48, - "user.name": "apache" + "event.dataset": "system.auth", + "fileset.module": "system", + "fileset.name": "auth", + "input.type": "log", + "offset": 934, + "prospector.type": "log", + "system.auth.hostname": "localhost", + "system.auth.pid": "6995", + "system.auth.timestamp": "Feb 22 11:47:05", + "system.auth.useradd.gid": "48", + "system.auth.useradd.home": "/usr/share/httpd", + "system.auth.useradd.name": "apache", + "system.auth.useradd.shell": "/sbin/nologin", + "system.auth.useradd.uid": "48" } -] +] \ No newline at end of file diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index e929fdf03a5b..cc4364084b1b 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -1,10 +1,9 @@ [ { - "ecs.version": "1.0.0-beta2", - "event.dataset": "syslog", - "event.module": "system", - "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "event.dataset": "system.syslog", + "fileset.module": "system", + "fileset.name": "syslog", + "input.type": "log", "log.flags": [ "multiline" ], @@ -17,22 +16,26 @@ "system.syslog.timestamp": "Dec 13 11:35:28" }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "syslog", - "event.module": "system", - "host.hostname": "a-mac-with-esc-key", - "input.type": "log", - "log.offset": 907, - "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", - "process.name": "GoogleSoftwareUpdateAgent", - "process.pid": 21412 - }, + "event.dataset": "system.syslog", + "fileset.module": "system", + "fileset.name": "syslog", + "input.type": "log", + "offset": 907, + "prospector.type": "log", + "system.syslog.hostname": "a-mac-with-esc-key", + "system.syslog.message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", + "system.syslog.pid": "21412", + "system.syslog.program": "GoogleSoftwareUpdateAgent", + "system.syslog.timestamp": "Dec 13 11:35:28" + }, { - "ecs.version": "1.0.0-beta2", - "event.dataset": "syslog", - "event.module": "system", - "input.type": "log", - "log.offset": 1176, - "message": "--- last message repeated 1 time ---" + "event.dataset": "system.syslog", + "fileset.module": "system", + "fileset.name": "syslog", + "input.type": "log", + "offset": 1176, + "prospector.type": "log", + "system.syslog.message": "--- last message repeated 1 time ---", + "system.syslog.timestamp": "Apr 4 03:39:57" } -] +] \ No newline at end of file