diff --git a/internal/install/install.go b/internal/install/install.go index fab0ec297e..48619f1774 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -166,32 +166,25 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error { // Install GeoIP database ingestGeoIPDir := filepath.Join(elasticPackagePath.StackDir(), "ingest-geoip") - // This directory is intended to be empty as we include GeoIP databases only in the 8x stack family. ingestGeoIPDefaultDir := filepath.Join(ingestGeoIPDir, "default") err = os.MkdirAll(ingestGeoIPDefaultDir, 0755) if err != nil { return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIPDefaultDir) } - ingestGeoIP8xDir := filepath.Join(ingestGeoIPDir, "8x") - err = os.MkdirAll(ingestGeoIP8xDir, 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIP8xDir) - } - - geoIpAsnMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-ASN.mmdb") + geoIpAsnMmdbPath := filepath.Join(ingestGeoIPDefaultDir, "GeoLite2-ASN.mmdb") err = writeStaticResource(err, geoIpAsnMmdbPath, geoIpAsnMmdb) if err != nil { return errors.Wrapf(err, "copying GeoIP ASN database failed (%s)", geoIpAsnMmdbPath) } - geoIpCityMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-City.mmdb") + geoIpCityMmdbPath := filepath.Join(ingestGeoIPDefaultDir, "GeoLite2-City.mmdb") err = writeStaticResource(err, geoIpCityMmdbPath, geoIpCityMmdb) if err != nil { return errors.Wrapf(err, "copying GeoIP city database failed (%s)", geoIpCityMmdbPath) } - geoIpCountryMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-Country.mmdb") + geoIpCountryMmdbPath := filepath.Join(ingestGeoIPDefaultDir, "GeoLite2-Country.mmdb") err = writeStaticResource(err, geoIpCountryMmdbPath, geoIpCountryMmdb) if err != nil { return errors.Wrapf(err, "copying GeoIP country database failed (%s)", geoIpCountryMmdbPath) diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 481ed5c4e5..12b25eb194 100644 --- a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -25,7 +25,7 @@ "ip": "::1" }, "event": { - "ingested": "2021-12-09T13:30:29.903774500Z", + "ingested": "2021-12-14T10:30:19.171259100Z", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "category": "web", "kind": "event", @@ -45,17 +45,6 @@ ] }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/hello", - "original": "/hello" - }, - "tags": [ - "preserve_original_event" - ], "apache": { "access": {} }, @@ -76,8 +65,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:29.903783200Z", + "ingested": "2021-12-14T10:30:19.171272300Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -87,6 +80,10 @@ "user": { "name": "-" }, + "url": { + "path": "/hello", + "original": "/hello" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -99,7 +96,10 @@ "name": "Mac" }, "version": "50.0." - } + }, + "tags": [ + "preserve_original_event" + ] }, { "apache": { @@ -119,7 +119,7 @@ "ip": "::1" }, "event": { - "ingested": "2021-12-09T13:30:29.903788600Z", + "ingested": "2021-12-14T10:30:19.171276600Z", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "category": "web", "kind": "event", @@ -134,17 +134,6 @@ ] }, { - "source": { - "address": "172.17.0.1", - "ip": "172.17.0.1" - }, - "url": { - "path": "/stringpatch", - "original": "/stringpatch" - }, - "tags": [ - "preserve_original_event" - ], "apache": { "access": {} }, @@ -165,8 +154,12 @@ "status_code": 404 } }, + "source": { + "address": "172.17.0.1", + "ip": "172.17.0.1" + }, "event": { - "ingested": "2021-12-09T13:30:29.903792500Z", + "ingested": "2021-12-14T10:30:19.171281Z", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "category": "web", "kind": "event", @@ -176,6 +169,10 @@ "user": { "name": "-" }, + "url": { + "path": "/stringpatch", + "original": "/stringpatch" + }, "user_agent": { "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -188,20 +185,12 @@ "name": "Other" }, "version": "15.0.a2" - } - }, - { - "source": { - "address": "monitoring-server", - "domain": "monitoring-server" - }, - "url": { - "path": "/status", - "original": "/status" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -222,8 +211,12 @@ "status_code": 200 } }, + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, "event": { - "ingested": "2021-12-09T13:30:29.903797600Z", + "ingested": "2021-12-14T10:30:19.171285300Z", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "category": "web", "kind": "event", @@ -233,6 +226,10 @@ "user": { "name": "-" }, + "url": { + "path": "/status", + "original": "/status" + }, "user_agent": { "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -245,7 +242,10 @@ "name": "Other" }, "version": "15.0.a2" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "apache": { @@ -271,7 +271,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-12-09T13:30:29.903803900Z", + "ingested": "2021-12-14T10:30:19.171289700Z", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "category": "web", "kind": "event", @@ -293,18 +293,6 @@ ] }, { - "source": { - "address": "monitoring-server", - "domain": "monitoring-server" - }, - "url": { - "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", - "extension": "mp4", - "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" - }, - "tags": [ - "preserve_original_event" - ], "apache": { "access": {} }, @@ -325,8 +313,12 @@ "status_code": 200 } }, + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, "event": { - "ingested": "2021-12-09T13:30:29.903809300Z", + "ingested": "2021-12-14T10:30:19.171328400Z", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "category": "web", "kind": "event", @@ -336,6 +328,11 @@ "user": { "name": "-" }, + "url": { + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" + }, "user_agent": { "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -348,7 +345,10 @@ "name": "Other" }, "version": "15.0.a2" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index c893102f46..1094c79d09 100644 --- a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -25,7 +25,7 @@ "ip": "::1" }, "event": { - "ingested": "2021-12-09T13:30:30.879403900Z", + "ingested": "2021-12-14T10:30:20.126148600Z", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "category": "web", "kind": "event", @@ -68,7 +68,7 @@ "ip": "::1" }, "event": { - "ingested": "2021-12-09T13:30:30.879409400Z", + "ingested": "2021-12-14T10:30:20.126162400Z", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "category": "web", "kind": "event", @@ -105,7 +105,7 @@ "ip": "::1" }, "event": { - "ingested": "2021-12-09T13:30:30.879413800Z", + "ingested": "2021-12-14T10:30:20.126170700Z", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "category": "web", "kind": "event", @@ -142,14 +142,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -162,7 +162,7 @@ "ip": "89.160.20.156" }, "event": { - "ingested": "2021-12-09T13:30:30.879418Z", + "ingested": "2021-12-14T10:30:20.126178600Z", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "category": "web", "kind": "event", @@ -203,14 +203,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -223,7 +223,7 @@ "ip": "89.160.20.156" }, "event": { - "ingested": "2021-12-09T13:30:30.879422100Z", + "ingested": "2021-12-14T10:30:20.126186500Z", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "category": "web", "kind": "event", @@ -264,14 +264,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -284,7 +284,7 @@ "ip": "89.160.20.156" }, "event": { - "ingested": "2021-12-09T13:30:30.879427100Z", + "ingested": "2021-12-14T10:30:20.126194300Z", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "category": "web", "kind": "event", diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 0964c3ba0d..3f8442649f 100644 --- a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -34,7 +34,7 @@ "ip": "172.30.0.119" }, "event": { - "ingested": "2021-12-09T13:30:31.533065900Z", + "ingested": "2021-12-14T10:30:20.756861200Z", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", "category": "web", "kind": "event", @@ -77,14 +77,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -97,7 +97,7 @@ "ip": "89.160.20.156" }, "event": { - "ingested": "2021-12-09T13:30:31.533074100Z", + "ingested": "2021-12-14T10:30:20.756875500Z", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", "category": "web", "kind": "event", diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 92c297c4b3..77a81ce005 100644 --- a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -1,17 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], "apache": { "access": {} }, @@ -32,8 +21,12 @@ "status_code": 200 } }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835525800Z", + "ingested": "2021-12-14T10:30:21.081782700Z", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "category": "web", "kind": "event", @@ -43,6 +36,10 @@ "user": { "name": "-" }, + "url": { + "path": "/", + "original": "/" + }, "user_agent": { "name": "Wget", "original": "Wget/1.13.4 (linux-gnu)", @@ -53,20 +50,12 @@ "name": "Other" }, "version": "1.13.4" - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/", - "original": "/" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -87,8 +76,12 @@ "status_code": 200 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835534600Z", + "ingested": "2021-12-14T10:30:21.081797300Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "category": "web", "kind": "event", @@ -98,6 +91,10 @@ "user": { "name": "-" }, + "url": { + "path": "/", + "original": "/" + }, "user_agent": { "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -110,21 +107,12 @@ "name": "Mac" }, "version": "54.0.2840.98" - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -145,8 +133,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835540100Z", + "ingested": "2021-12-14T10:30:21.081805900Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "category": "web", "kind": "event", @@ -156,6 +148,11 @@ "user": { "name": "-" }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, "user_agent": { "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -168,20 +165,12 @@ "name": "Mac" }, "version": "54.0.2840.98" - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/", - "original": "/" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -202,8 +191,12 @@ "status_code": 200 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835543600Z", + "ingested": "2021-12-14T10:30:21.081814Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -213,6 +206,10 @@ "user": { "name": "-" }, + "url": { + "path": "/", + "original": "/" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -225,21 +222,12 @@ "name": "Mac" }, "version": "50.0." - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -260,8 +248,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835548Z", + "ingested": "2021-12-14T10:30:21.081822Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -271,6 +263,11 @@ "user": { "name": "-" }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -283,21 +280,12 @@ "name": "Mac" }, "version": "50.0." - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -318,8 +306,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835553700Z", + "ingested": "2021-12-14T10:30:21.081829900Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -329,6 +321,11 @@ "user": { "name": "-" }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -341,20 +338,12 @@ "name": "Mac" }, "version": "50.0." - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/test", - "original": "/test" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -375,8 +364,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835559600Z", + "ingested": "2021-12-14T10:30:21.081837900Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -386,6 +379,10 @@ "user": { "name": "-" }, + "url": { + "path": "/test", + "original": "/test" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -398,20 +395,12 @@ "name": "Mac" }, "version": "50.0." - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/hello", - "original": "/hello" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -432,8 +421,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835563600Z", + "ingested": "2021-12-14T10:30:21.081845900Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -443,6 +436,10 @@ "user": { "name": "-" }, + "url": { + "path": "/hello", + "original": "/hello" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -455,20 +452,12 @@ "name": "Mac" }, "version": "50.0." - } - }, - { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/crap", - "original": "/crap" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "apache": { "access": {} }, @@ -489,8 +478,12 @@ "status_code": 404 } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:31.835568100Z", + "ingested": "2021-12-14T10:30:21.081853800Z", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", @@ -500,6 +493,10 @@ "user": { "name": "-" }, + "url": { + "path": "/crap", + "original": "/crap" + }, "user_agent": { "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", @@ -512,7 +509,10 @@ "name": "Mac" }, "version": "50.0." - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 8b6a8cbbef..3f778d516b 100644 --- a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -36,7 +36,7 @@ } }, "event": { - "ingested": "2021-12-09T13:30:33.387841500Z", + "ingested": "2021-12-14T10:30:22.626765300Z", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "category": "web", "kind": "event", diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index 21df2d32d2..d3a1f6021a 100644 --- a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -19,7 +19,7 @@ "ip": "192.168.33.1" }, "event": { - "ingested": "2021-12-09T13:30:33.868254100Z", + "ingested": "2021-12-14T10:30:23.084440800Z", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "category": "web", "type": "error", @@ -48,7 +48,7 @@ "level": "notice" }, "event": { - "ingested": "2021-12-09T13:30:33.868263600Z", + "ingested": "2021-12-14T10:30:23.084454100Z", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "category": "web", "type": "info", @@ -67,20 +67,32 @@ "id": 4328636416 } }, + "apache": { + "error": { + "module": "core" + } + }, + "file": { + "path": "/usr/local/apache2/htdocs/favicon.ico" + }, + "@timestamp": "2011-09-09T10:42:29.902+02:00", + "ecs": { + "version": "1.12.0" + }, "log": { "level": "error" }, "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -92,30 +104,18 @@ "address": "89.160.20.156", "ip": "89.160.20.156" }, - "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", - "tags": [ - "preserve_original_event" - ], - "apache": { - "error": { - "module": "core" - } - }, - "file": { - "path": "/usr/local/apache2/htdocs/favicon.ico" - }, - "@timestamp": "2011-09-09T10:42:29.902+02:00", - "ecs": { - "version": "1.12.0" - }, "event": { - "ingested": "2021-12-09T13:30:33.868270Z", + "ingested": "2021-12-14T10:30:23.084463800Z", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "category": "web", "type": "error", "timezone": "GMT+2", "kind": "event" - } + }, + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "tags": [ + "preserve_original_event" + ] }, { "process": { @@ -136,14 +136,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -157,7 +157,7 @@ "ip": "89.160.20.156" }, "event": { - "ingested": "2021-12-09T13:30:33.868275800Z", + "ingested": "2021-12-14T10:30:23.084472500Z", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "category": "web", "type": "error", diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 15400a6c97..e7dec80ece 100644 --- a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -17,7 +17,7 @@ "level": "notice" }, "event": { - "ingested": "2021-12-09T13:30:34.149405700Z", + "ingested": "2021-12-14T10:30:23.334072200Z", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "category": "web", "type": "info", @@ -46,7 +46,7 @@ "level": "notice" }, "event": { - "ingested": "2021-12-09T13:30:34.149429600Z", + "ingested": "2021-12-14T10:30:23.334081Z", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "category": "web", "type": "info", diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 7b5f6e1b4b..eb72ece497 100644 --- a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -20,7 +20,7 @@ "level": "trace3" }, "event": { - "ingested": "2021-12-09T13:30:34.228018700Z", + "ingested": "2021-12-14T10:30:23.413830300Z", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "category": "web", "type": "info", diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 92c843f590..2a8a76e17d 100644 --- a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -12,7 +12,7 @@ "level": "notice" }, "event": { - "ingested": "2021-12-09T13:30:34.283841100Z", + "ingested": "2021-12-14T10:30:23.471847700Z", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "category": "web", "type": "info", @@ -25,17 +25,6 @@ ] }, { - "log": { - "level": "error" - }, - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", - "tags": [ - "preserve_original_event" - ], "apache": { "error": {} }, @@ -46,19 +35,30 @@ "ecs": { "version": "1.12.0" }, + "log": { + "level": "error" + }, "http": { "request": { "referrer": "http://192.168.33.72/" } }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, "event": { - "ingested": "2021-12-09T13:30:34.283849400Z", + "ingested": "2021-12-14T10:30:23.471861500Z", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "category": "web", "type": "error", "timezone": "GMT+2", "kind": "event" - } + }, + "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "tags": [ + "preserve_original_event" + ] }, { "apache": { @@ -79,7 +79,7 @@ "ip": "192.168.33.1" }, "event": { - "ingested": "2021-12-09T13:30:34.283853Z", + "ingested": "2021-12-14T10:30:23.471870100Z", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "category": "web", "type": "error", @@ -110,7 +110,7 @@ "ip": "192.168.33.1" }, "event": { - "ingested": "2021-12-09T13:30:34.283857200Z", + "ingested": "2021-12-14T10:30:23.471878Z", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "category": "web", "type": "error", @@ -141,7 +141,7 @@ "ip": "192.168.33.1" }, "event": { - "ingested": "2021-12-09T13:30:34.283862600Z", + "ingested": "2021-12-14T10:30:23.471885700Z", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "category": "web", "type": "error", @@ -172,7 +172,7 @@ "ip": "192.168.33.1" }, "event": { - "ingested": "2021-12-09T13:30:34.283867900Z", + "ingested": "2021-12-14T10:30:23.471889200Z", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "category": "web", "type": "error", @@ -203,7 +203,7 @@ "ip": "192.168.33.1" }, "event": { - "ingested": "2021-12-09T13:30:34.283873300Z", + "ingested": "2021-12-14T10:30:23.471894500Z", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "category": "web", "type": "error", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json index 6231a4dd28..e9b520569f 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json @@ -14,20 +14,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "GB-OXF", - "city_name": "Abingdon", + "region_iso_code": "GB-ENG", + "city_name": "London", "country_iso_code": "GB", "country_name": "United Kingdom", - "region_name": "Oxfordshire", + "region_name": "England", "location": { - "lon": -1.3614, - "lat": 51.7095 - } - }, - "as": { - "number": 20712, - "organization": { - "name": "Andrews \u0026 Arnold Ltd" + "lon": -0.0931, + "lat": 51.5142 } }, "address": "81.2.69.144", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json index 9dcab5261a..6b314e7ee9 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:09:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"AccessDeniedException\",\"errorMessage\":\"An unknown error occurred\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE\",\"eventID\":\"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -65,16 +62,12 @@ "name": "Spider" }, "version": "1.16.310" - } - }, - { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "cloud": { "region": "us-east-1", "account": { @@ -90,6 +83,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:03:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5c16-4eda-9724-EXAMPLE\",\"eventID\":\"EXAMPLE-35a7-4c25-9fc7-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -130,7 +127,10 @@ "name": "Spider" }, "version": "1.16.310" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json index 69fec10edc..5163692301 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json @@ -1,17 +1,32 @@ { "expected": [ { + "cloud": { + "region": "us-east-2", + "account": { + "id": "111122223333" + } + }, + "@timestamp": "2014-07-16T15:49:27.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "JohnDoe" + ] + }, "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -23,24 +38,6 @@ "address": "89.160.20.156", "ip": "89.160.20.156" }, - "tags": [ - "preserve_original_event" - ], - "cloud": { - "region": "us-east-2", - "account": { - "id": "111122223333" - } - }, - "@timestamp": "2014-07-16T15:49:27.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "user": [ - "JohnDoe" - ] - }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JohnDoe\",\"accountId\":\"111122223333\",\"userName\":\"JohnDoe\"},\"eventTime\":\"2014-07-16T15:49:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Success\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/s3/\",\"MFAUsed\":\"No\"},\"eventID\":\"3fcfb182-98f8-4744-bd45-10aEXAMPLE\"}", "provider": "signin.amazonaws.com", @@ -100,20 +97,38 @@ "name": "Other" }, "version": "24.0." - } + }, + "tags": [ + "preserve_original_event" + ] }, { + "cloud": { + "region": "us-east-2", + "account": { + "id": "111122223333" + } + }, + "@timestamp": "2014-07-08T17:35:27.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "JaneDoe" + ] + }, "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -125,24 +140,6 @@ "address": "89.160.20.156", "ip": "89.160.20.156" }, - "tags": [ - "preserve_original_event" - ], - "cloud": { - "region": "us-east-2", - "account": { - "id": "111122223333" - } - }, - "@timestamp": "2014-07-08T17:35:27.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "user": [ - "JaneDoe" - ] - }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JaneDoe\",\"accountId\":\"111122223333\",\"userName\":\"JaneDoe\"},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", "provider": "signin.amazonaws.com", @@ -203,7 +200,10 @@ "name": "Other" }, "version": "24.0." - } + }, + "tags": [ + "preserve_original_event" + ] }, { "cloud": { @@ -219,14 +219,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json index fe958a8e1b..0ffe55825e 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:43:06Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"accessKey\":{\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"status\":\"Active\",\"userName\":\"Bob\",\"createDate\":\"Jan 8, 2020 8:43:06 PM\"}},\"requestID\":\"EXAMPLE-823a-48dc-8fa9-EXAMPLE\",\"eventID\":\"EXAMPLE-3cab-40f8-938b-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -85,7 +82,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json index 00bd7e5645..ce1c161bf2 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json @@ -1,17 +1,32 @@ { "expected": [ { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2014-03-06T17:10:34.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "Alice" + ] + }, "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -23,24 +38,6 @@ "address": "89.160.20.156", "ip": "89.160.20.156" }, - "tags": [ - "preserve_original_event" - ], - "cloud": { - "region": "us-east-2", - "account": { - "id": "123456789012" - } - }, - "@timestamp": "2014-03-06T17:10:34.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "user": [ - "Alice" - ] - }, "event": { "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-06T15:15:06Z\"}}},\"eventTime\":\"2014-03-06T17:10:34Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"CreateKeyPair\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx\",\"requestParameters\":{\"keyName\":\"mykeypair\"},\"responseElements\":{\"keyName\":\"mykeypair\",\"keyFingerprint\":\"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21\",\"keyMaterial\":\"\u003csensitiveDataRemoved\u003e\"}}", "provider": "ec2.amazonaws.com", @@ -95,7 +92,10 @@ "device": { "name": "Other" } - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json index 1839c2927b..b86259807b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-west-2", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"CreateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"enableLogFileValidation\":true,\"kmsKeyId\":\"\",\"isOrganizationTrail\":false},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":true,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-5149-4cf2-be99-EXAMPLE\",\"eventID\":\"EXAMPLE-d04b-4eff-833a-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", @@ -82,7 +79,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json index 5fc1e895b7..413dc250e7 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-2", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2014-03-24T21:11:59Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateUser\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.3.2 Python/2.7.5 Windows/7\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"user\":{\"createDate\":\"Mar 24, 2014 9:11:59 PM\",\"userName\":\"Bob\",\"arn\":\"arn:aws:iam::123456789012:user/Bob\",\"path\":\"/\",\"userId\":\"EXAMPLEUSERID\"}}}", "provider": "iam.amazonaws.com", @@ -83,7 +80,10 @@ "name": "Other" }, "version": "1.3.2" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json index 4a9c4f4240..1884ebe14d 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:10:15Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"virtualMFADeviceName\":\"Alice\",\"path\":\"/\"},\"responseElements\":{\"virtualMFADevice\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"}},\"requestID\":\"EXAMPLE-303b-4b0e-a8c7-EXAMPLE\",\"eventID\":\"EXAMPLE-351c-472a-b089-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -78,7 +75,10 @@ "name": "Other" }, "original": "console.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json index 764cc3d905..b30f8843ba 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeactivateMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Alice\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-801a-4624-8fa0-EXAMPLE\",\"eventID\":\"EXAMPLE-1889-416b-ace9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -76,7 +73,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json index 885f9b97c9..aa9e820ad8 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T19:09:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"accessKeyId\":\"EXAMPLE_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-3bea-41fa-a0b4-EXAMPLE\",\"eventID\":\"EXAMPLE-0698-46bd-998d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -77,7 +74,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json index 4ed161acea..ce237efc69 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json @@ -14,14 +14,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json index 6ffa98520d..253d4c6b7b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:07:08Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7b34-44ae-a22f-EXAMPLE\",\"eventID\":\"EXAMPLE-72ff-4d4f-9a8d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -77,7 +74,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json index 2dd33e9881..ae6e0fb9e5 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-west-2", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T20:09:51Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"DeleteTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-d44f-4a2a-966f-EXAMPLE\",\"eventID\":\"EXAMPLE-3f9d-4634-8ff1-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", @@ -63,7 +60,10 @@ "name": "Spider" }, "version": "1.16.310" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json index 6bb1d94f1e..6026a18f90 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-03T15:26:38Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-03T15:50:52Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"0e794d53-cdb5-4f7d-b7db-5EXAMPLE\",\"eventID\":\"b89eb34b-8fcb-4cba-8439-d4EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "iam.amazonaws.com", @@ -76,7 +73,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json index a482b81222..86319a3d4b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-af91-4d1a-aaf2-EXAMPLE\",\"eventID\":\"EXAMPLE-f8e6-4d5f-8525-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -72,7 +69,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json index 5a66ad5660..9ac93b9661 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:11:09Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"EnableMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-adea-490a-a806-EXAMPLE\",\"eventID\":\"EXAMPLE-3fdc-4b2a-9885-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -76,7 +73,10 @@ "name": "Other" }, "original": "console.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json index 3101de44c8..39c81506b7 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-west-2", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StartLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-1c30-4f43-9763-EXAMPLE\",\"eventID\":\"EXAMPLE-aa78-4a84-a27f-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", @@ -67,7 +64,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json index 79d7439e30..47da079038 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-west-2", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T16:46:16Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StopLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-869f-4fec-86f9-EXAMPLE\",\"eventID\":\"EXAMPLE-8cc3-42db-9a0d-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", @@ -67,7 +64,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json index 0d57cca9cf..5ae9bff983 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T15:01:23Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7d0c-45f4-b25b-EXAMPLE\",\"eventID\":\"EXAMPLE-0ef0-42cd-8551-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -78,7 +75,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json index cdc24c9d2f..27ceafa8f3 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:05:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccountPasswordPolicy\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"requireLowercaseCharacters\":true,\"requireSymbols\":true,\"requireNumbers\":true,\"minimumPasswordLength\":12,\"requireUppercaseCharacters\":true,\"allowUsersToChangePassword\":true},\"responseElements\":null,\"requestID\":\"EXAMPLE-5ebf-4bc3-a349-EXAMPLE\",\"eventID\":\"EXAMPLE-91f9-49f3-948c-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -77,7 +74,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json index 54e213526e..a4d54bfee5 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:25:42Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateLoginProfile\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0dc6-447a-8859-EXAMPLE\",\"eventID\":\"EXAMPLE-c3b6-4498-b818-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -76,7 +73,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json index c6ee9087cd..913be656a4 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -24,6 +17,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -78,16 +75,12 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } - }, - { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "cloud": { "region": "us-east-1", "account": { @@ -104,6 +97,10 @@ "Bob" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -158,7 +155,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json index 754605022e..2b62a062c5 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json @@ -1,17 +1,32 @@ { "expected": [ { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2016-07-14T19:15:45.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "Alice" + ] + }, "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -23,24 +38,6 @@ "address": "89.160.20.156", "ip": "89.160.20.156" }, - "tags": [ - "preserve_original_event" - ], - "cloud": { - "region": "us-east-2", - "account": { - "id": "123456789012" - } - }, - "@timestamp": "2016-07-14T19:15:45.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "user": [ - "Alice" - ] - }, "event": { "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2016-07-14T19:15:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22\",\"errorCode\":\"TrailNotFoundException\",\"errorMessage\":\"Unknown trail: myTrail2 for the user: 123456789012\",\"requestParameters\":{\"name\":\"myTrail2\"},\"responseElements\":null,\"requestID\":\"5d40662a-49f7-11e6-97e4-dEXAMPLE\",\"eventID\":\"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "cloudtrail.amazonaws.com", @@ -85,16 +82,12 @@ "name": "Spider" }, "version": "1.10.32" - } - }, - { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "cloud": { "region": "us-west-2", "account": { @@ -110,6 +103,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:58:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"isMultiRegionTrail\":true,\"enableLogFileValidation\":false,\"kmsKeyId\":\"\"},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"snsTopicARN\":\"\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":false,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-f3da-42d1-84f5-EXAMPLE\",\"eventID\":\"EXAMPLE-b5e9-4846-8407-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", @@ -167,7 +164,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json index e324600a27..b9fcec54df 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -25,6 +18,10 @@ "Robert" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "iam.amazonaws.com", @@ -77,7 +74,10 @@ "name": "Spider" }, "version": "1.16.310" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json index 63e08a3cb3..9d41f0e746 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json @@ -1,13 +1,6 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "tags": [ - "preserve_original_event" - ], "cloud": { "region": "us-east-1", "account": { @@ -23,6 +16,10 @@ "Alice" ] }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, "event": { "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:40Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UploadSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\",\"userName\":\"Alice\"},\"responseElements\":{\"sSHPublicKey\":{\"fingerprint\":\"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\",\"status\":\"Active\",\"uploadDate\":\"Jan 10, 2020 4:06:40 PM\",\"userName\":\"Alice\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\"}},\"requestID\":\"EXAMPLE-44b9-41cd-90f2-EXAMPLE\",\"eventID\":\"EXAMPLE-9a9d-4da4-9998-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", @@ -81,7 +78,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json index 8e4cce3a5f..3dfb5a8313 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json +++ b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json @@ -6,7 +6,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.525004600Z", + "ingested": "2021-12-14T10:30:54.939936200Z", "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." }, "aws": { @@ -24,7 +24,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.525012700Z", + "ingested": "2021-12-14T10:30:54.939952100Z", "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." }, "aws": { @@ -42,7 +42,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.525017900Z", + "ingested": "2021-12-14T10:30:54.939961600Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" }, "aws": { @@ -60,7 +60,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.525022500Z", + "ingested": "2021-12-14T10:30:54.940028500Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" }, "aws": { @@ -78,7 +78,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.525027400Z", + "ingested": "2021-12-14T10:30:54.940036600Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." }, "aws": { @@ -96,7 +96,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.525032300Z", + "ingested": "2021-12-14T10:30:54.940044900Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" }, "aws": { diff --git a/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json index 4298569cb3..645fed7657 100644 --- a/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json +++ b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json @@ -9,7 +9,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.684169900Z", + "ingested": "2021-12-14T10:30:56.125028Z", "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." }, "aws": { @@ -31,7 +31,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.684178100Z", + "ingested": "2021-12-14T10:30:56.125040800Z", "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." }, "aws": { @@ -53,7 +53,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.684183300Z", + "ingested": "2021-12-14T10:30:56.125047800Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" }, "aws": { @@ -75,7 +75,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.684188400Z", + "ingested": "2021-12-14T10:30:56.125052700Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" }, "aws": { @@ -97,7 +97,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.684193500Z", + "ingested": "2021-12-14T10:30:56.125057100Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." }, "aws": { @@ -119,7 +119,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:11:58.684198500Z", + "ingested": "2021-12-14T10:30:56.125063600Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" }, "aws": { diff --git a/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json b/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json index baf96cc0f7..6cd73ba7ae 100644 --- a/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json +++ b/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json @@ -43,7 +43,7 @@ } }, "event": { - "ingested": "2021-12-09T16:11:58.868846100Z", + "ingested": "2021-12-14T10:30:56.337187600Z", "original": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward,redirect\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"", "kind": "event", "start": "2018-07-02T22:22:48.364000Z", diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json index abd3a9e475..77f299fddd 100644 --- a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json @@ -11,14 +11,14 @@ ], "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "cloud": { @@ -61,7 +61,7 @@ }, "event": { "duration": 17000000, - "ingested": "2021-12-09T16:11:59.134194800Z", + "ingested": "2021-12-14T10:30:56.619660100Z", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.LOCATION", @@ -118,14 +118,14 @@ ], "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "cloud": { @@ -168,7 +168,7 @@ }, "event": { "duration": 3000000, - "ingested": "2021-12-09T16:11:59.134198700Z", + "ingested": "2021-12-14T10:30:56.619676Z", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.LOCATION", @@ -225,14 +225,14 @@ ], "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "cloud": { @@ -275,7 +275,7 @@ }, "event": { "duration": 2000000, - "ingested": "2021-12-09T16:11:59.134204100Z", + "ingested": "2021-12-14T10:30:56.619685700Z", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.BUCKET", @@ -333,14 +333,14 @@ ], "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "cloud": { @@ -383,7 +383,7 @@ }, "event": { "duration": 4000000, - "ingested": "2021-12-09T16:11:59.134208400Z", + "ingested": "2021-12-14T10:30:56.619695Z", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.LOCATION", @@ -430,19 +430,16 @@ } }, { - "tags": [ - "preserve_original_event" - ], "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "cloud": { @@ -477,7 +474,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-12-09T16:11:59.134212900Z", + "ingested": "2021-12-14T10:30:56.619704800Z", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2", "kind": "event", "action": "BATCH.DELETE.OBJECT", @@ -506,22 +503,22 @@ "key": "jolokia-war-1.5.0.war", "object_size": 344017 } - } - }, - { + }, "tags": [ "preserve_original_event" - ], + ] + }, + { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "cloud": { @@ -556,7 +553,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-12-09T16:11:59.134217300Z", + "ingested": "2021-12-14T10:30:56.619767600Z", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "BATCH.DELETE.OBJECT", @@ -585,17 +582,12 @@ "key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png", "object_size": 57138 } - } - }, - { - "url": { - "path": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz", - "extension": "gz", - "original": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz" }, "tags": [ "preserve_original_event" - ], + ] + }, + { "cloud": { "provider": "aws" }, @@ -628,7 +620,7 @@ }, "event": { "duration": 103000000, - "ingested": "2021-12-09T16:11:59.134221Z", + "ingested": "2021-12-14T10:30:56.619777300Z", "original": "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b flow-log-test [14/Jul/2021:18:57:31 +0000] - svc:delivery.logs.amazonaws.com MVGXZXEVN3IG9S24 REST.PUT.OBJECT AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz \"PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1\" 200 - - 773 103 13 \"-\" \"-\" - 02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader flow-log-test.s3.us-gov-west-1.amazonaws.com TLSv1.2 -", "kind": "event", "action": "REST.PUT.OBJECT", @@ -659,7 +651,15 @@ "key": "AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz", "object_size": 773 } - } + }, + "url": { + "path": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz", + "extension": "gz", + "original": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz" + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json index 77a00b72a0..6c89adae0f 100644 --- a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json @@ -1,21 +1,31 @@ { "expected": [ { + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2016-10-31T11:37:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + ] + }, "destination": { "geo": { "continent_name": "Europe", - "country_name": "Denmark", + "country_name": "Norway", "location": { "lon": 10.0, - "lat": 56.0 + "lat": 62.0 }, - "country_iso_code": "DK" - }, - "as": { - "number": 62121, - "organization": { - "name": "Christian Ebsen ApS" - } + "country_iso_code": "NO" }, "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", "port": 22, @@ -24,54 +34,21 @@ "source": { "geo": { "continent_name": "Europe", - "country_name": "Denmark", + "country_name": "Norway", "location": { "lon": 10.0, - "lat": 56.0 + "lat": 62.0 }, - "country_iso_code": "DK" - }, - "as": { - "number": 62121, - "organization": { - "name": "Christian Ebsen ApS" - } + "country_iso_code": "NO" }, "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", "port": 34892, "bytes": 8855, - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", - "packets": 54 - }, - "tags": [ - "preserve_original_event" - ], - "network": { - "community_id": "1:3piNHoW0DjbrWkF//BeRomCaOZQ=", - "transport": "tcp", - "type": "ipv6", - "bytes": 8855, - "iana_number": "6", - "packets": 54 - }, - "cloud": { - "provider": "aws", - "account": { - "id": "123456789010" - } - }, - "@timestamp": "2016-10-31T11:37:00.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "ip": [ - "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", - "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" - ] + "packets": 54, + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "event": { - "ingested": "2021-12-09T16:12:00.503382700Z", + "ingested": "2021-12-14T10:30:58.023227600Z", "original": "2 123456789010 eni-1235b8ca123456789 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", "kind": "event", "start": "2016-10-31T11:35:08.000Z", @@ -88,6 +65,17 @@ "interface_id": "eni-1235b8ca123456789", "version": "2" } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:3piNHoW0DjbrWkF//BeRomCaOZQ=", + "transport": "tcp", + "type": "ipv6", + "bytes": 8855, + "iana_number": "6", + "packets": 54 } }, { @@ -102,7 +90,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:12:00.503391600Z", + "ingested": "2021-12-14T10:30:58.023240100Z", "original": "2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA", "kind": "event", "start": "2015-05-10T18:01:16.000Z", @@ -134,7 +122,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:12:00.503397800Z", + "ingested": "2021-12-14T10:30:58.023249300Z", "original": "2 123456789010 eni-89.160.20.1561aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA", "kind": "event", "start": "2015-05-10T18:01:16.000Z", @@ -155,17 +143,33 @@ ] }, { + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2014-12-14T04:07:50.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156", + "89.160.20.156" + ] + }, "destination": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -181,14 +185,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -203,6 +207,25 @@ "ip": "89.160.20.156", "packets": 20 }, + "event": { + "ingested": "2021-12-14T10:30:58.023257700Z", + "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK", + "kind": "event", + "start": "2014-12-14T04:06:50.000Z", + "end": "2014-12-14T04:07:50.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + }, "tags": [ "preserve_original_event" ], @@ -213,7 +236,9 @@ "bytes": 4249, "iana_number": "6", "packets": 20 - }, + } + }, + { "cloud": { "provider": "aws", "account": { @@ -230,38 +255,17 @@ "89.160.20.156" ] }, - "event": { - "ingested": "2021-12-09T16:12:00.503403700Z", - "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK", - "kind": "event", - "start": "2014-12-14T04:06:50.000Z", - "end": "2014-12-14T04:07:50.000Z", - "type": "flow", - "category": "network_traffic", - "outcome": "allow" - }, - "aws": { - "vpcflow": { - "action": "ACCEPT", - "account_id": "123456789010", - "log_status": "OK", - "interface_id": "eni-1235b8ca123456789", - "version": "2" - } - } - }, - { "destination": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -277,14 +281,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -299,6 +303,25 @@ "ip": "89.160.20.156", "packets": 20 }, + "event": { + "ingested": "2021-12-14T10:30:58.023265800Z", + "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK", + "kind": "event", + "start": "2014-12-14T04:06:50.000Z", + "end": "2014-12-14T04:07:50.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "deny" + }, + "aws": { + "vpcflow": { + "action": "REJECT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + }, "tags": [ "preserve_original_event" ], @@ -309,44 +332,25 @@ "bytes": 4249, "iana_number": "6", "packets": 20 - }, + } + }, + { "cloud": { "provider": "aws", "account": { "id": "123456789010" } }, - "@timestamp": "2014-12-14T04:07:50.000Z", + "@timestamp": "2015-05-29T16:32:22.000Z", "ecs": { "version": "1.12.0" }, "related": { "ip": [ "89.160.20.156", - "89.160.20.156" + "172.31.16.139" ] }, - "event": { - "ingested": "2021-12-09T16:12:00.503409900Z", - "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK", - "kind": "event", - "start": "2014-12-14T04:06:50.000Z", - "end": "2014-12-14T04:07:50.000Z", - "type": "flow", - "category": "network_traffic", - "outcome": "deny" - }, - "aws": { - "vpcflow": { - "action": "REJECT", - "account_id": "123456789010", - "log_status": "OK", - "interface_id": "eni-1235b8ca123456789", - "version": "2" - } - } - }, - { "destination": { "port": 0, "address": "172.31.16.139", @@ -355,14 +359,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -377,6 +381,25 @@ "ip": "89.160.20.156", "packets": 4 }, + "event": { + "ingested": "2021-12-14T10:30:58.023274Z", + "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK", + "kind": "event", + "start": "2015-05-29T16:30:27.000Z", + "end": "2015-05-29T16:32:22.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + }, "tags": [ "preserve_original_event" ], @@ -386,7 +409,9 @@ "bytes": 336, "iana_number": "1", "packets": 4 - }, + } + }, + { "cloud": { "provider": "aws", "account": { @@ -399,42 +424,21 @@ }, "related": { "ip": [ - "89.160.20.156", - "172.31.16.139" + "172.31.16.139", + "89.160.20.156" ] }, - "event": { - "ingested": "2021-12-09T16:12:00.503416200Z", - "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK", - "kind": "event", - "start": "2015-05-29T16:30:27.000Z", - "end": "2015-05-29T16:32:22.000Z", - "type": "flow", - "category": "network_traffic", - "outcome": "allow" - }, - "aws": { - "vpcflow": { - "action": "ACCEPT", - "account_id": "123456789010", - "log_status": "OK", - "interface_id": "eni-1235b8ca123456789", - "version": "2" - } - } - }, - { "destination": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -454,34 +458,8 @@ "packets": 4, "ip": "172.31.16.139" }, - "tags": [ - "preserve_original_event" - ], - "network": { - "community_id": "1:XiVZKra6oEtIAPBi9QgeQL4Hp6M=", - "type": "ipv4", - "bytes": 336, - "iana_number": "1", - "packets": 4 - }, - "cloud": { - "provider": "aws", - "account": { - "id": "123456789010" - } - }, - "@timestamp": "2015-05-29T16:32:22.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "ip": [ - "172.31.16.139", - "89.160.20.156" - ] - }, "event": { - "ingested": "2021-12-09T16:12:00.503420100Z", + "ingested": "2021-12-14T10:30:58.023282200Z", "original": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 89.160.20.156 0 0 1 4 336 1432917094 1432917142 REJECT OK", "kind": "event", "start": "2015-05-29T16:31:34.000Z", @@ -498,6 +476,16 @@ "interface_id": "eni-1235b8ca123456789", "version": "2" } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:XiVZKra6oEtIAPBi9QgeQL4Hp6M=", + "type": "ipv4", + "bytes": 336, + "iana_number": "1", + "packets": 4 } } ] diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json index cca3b2323c..145d7546ec 100644 --- a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json @@ -1,6 +1,25 @@ { "expected": [ { + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "@timestamp": "2019-08-26T19:48:53.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156", + "10.0.0.62" + ] + }, "destination": { "port": 5001, "address": "10.0.0.62", @@ -9,14 +28,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -31,38 +50,8 @@ "ip": "89.160.20.156", "packets": 8 }, - "tags": [ - "preserve_original_event" - ], - "network": { - "community_id": "1:dF5WY79X1yVncj+yH8q27Q5Bnpk=", - "transport": "tcp", - "type": "ipv4", - "bytes": 568, - "iana_number": "6", - "packets": 8 - }, - "cloud": { - "provider": "aws", - "account": { - "id": "123456789010" - }, - "instance": { - "id": "i-01234567890123456" - } - }, - "@timestamp": "2019-08-26T19:48:53.000Z", - "ecs": { - "version": "1.12.0" - }, - "related": { - "ip": [ - "89.160.20.156", - "10.0.0.62" - ] - }, "event": { - "ingested": "2021-12-09T16:12:01.346119700Z", + "ingested": "2021-12-14T10:30:58.834121700Z", "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 89.160.20.156 10.0.0.62 43416 5001 89.160.20.156 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK", "kind": "event", "start": "2019-08-26T19:47:55.000Z", @@ -89,6 +78,17 @@ "action": "ACCEPT", "pkt_dstaddr": "10.0.0.62" } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:dF5WY79X1yVncj+yH8q27Q5Bnpk=", + "transport": "tcp", + "type": "ipv4", + "bytes": 568, + "iana_number": "6", + "packets": 8 } }, { @@ -106,7 +106,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:12:01.346125500Z", + "ingested": "2021-12-14T10:30:58.834135900Z", "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA", "kind": "event", "start": "2019-08-26T19:47:55.000Z", @@ -144,7 +144,7 @@ "version": "1.12.0" }, "event": { - "ingested": "2021-12-09T16:12:01.346129200Z", + "ingested": "2021-12-14T10:30:58.834144Z", "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA", "kind": "event", "start": "2019-08-26T19:47:55.000Z", diff --git a/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json index b2240aa9ad..1342df1c84 100644 --- a/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json +++ b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json @@ -8,14 +8,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -63,7 +63,7 @@ }, "event": { "action": "BLOCK", - "ingested": "2021-12-09T16:12:01.711621Z", + "ingested": "2021-12-14T10:30:59.169780800Z", "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"x-stm-test\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", "category": "web", "type": [ @@ -106,14 +106,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -161,7 +161,7 @@ }, "event": { "action": "ALLOW", - "ingested": "2021-12-09T16:12:01.711626900Z", + "ingested": "2021-12-14T10:30:59.169794200Z", "original": "{\"timestamp\":1592357192516,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9\",\"terminatingRuleId\":\"Default_Action\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"ALLOW\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[{\"ruleId\":\"TestRule\",\"action\":\"COUNT\",\"ruleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"and\",\"1\"]}]}],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"foo\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", "category": "web", "type": [ @@ -210,14 +210,14 @@ "source": { "geo": { "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", + "region_iso_code": "SE-E", + "city_name": "Linköping", "country_iso_code": "SE", "country_name": "Sweden", - "region_name": "Stockholm", + "region_name": "Östergötland County", "location": { - "lon": 17.8167, - "lat": 59.2 + "lon": 15.6167, + "lat": 58.4167 } }, "as": { @@ -265,7 +265,7 @@ }, "event": { "action": "BLOCK", - "ingested": "2021-12-09T16:12:01.711632900Z", + "ingested": "2021-12-14T10:30:59.169805500Z", "original": "{\"timestamp\":1592361810888,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9\",\"terminatingRuleId\":\"RG-Reference\",\"terminatingRuleType\":\"GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"XSS\",\"location\":\"HEADER\",\"matchedData\":[\"\u003c\",\"frameset\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[{\"ruleGroupId\":\"arn:aws:wafv2:us-east-1:123456789012:global/rulegroup/hello-world/c05lb698-1f11-4m41-aef4-99a506d53f4b\",\"terminatingRule\":{\"ruleId\":\"RuleA-XSS\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[{\"ruleId\":\"RuleB-SQLi\",\"action\":\"COUNT\",\"ruleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"and\",\"1\"]}]}],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"xssfoo\",\"value\":\"\u003cframeset onload=alert(1)\u003e\"},{\"name\":\"bar\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", "category": "web", "type": [ @@ -327,38 +327,6 @@ } }, { - "rule": { - "ruleset": "REGULAR", - "id": "STMTest_SQLi_XSS" - }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-AB", - "city_name": "Tumba", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Stockholm", - "location": { - "lon": 17.8167, - "lat": 59.2 - } - }, - "as": { - "number": 29518, - "organization": { - "name": "Bredband2 AB" - } - }, - "ip": "89.160.20.156" - }, - "tags": [ - "preserve_original_event" - ], - "network": { - "protocol": "http", - "transport": "tcp" - }, "cloud": { "region": "ap-southeast-2", "provider": "aws", @@ -377,6 +345,10 @@ "89.160.20.156" ] }, + "rule": { + "ruleset": "REGULAR", + "id": "STMTest_SQLi_XSS" + }, "http": { "request": { "method": "POST", @@ -384,9 +356,30 @@ }, "version": "1.1" }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-E", + "city_name": "Linköping", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Östergötland County", + "location": { + "lon": 15.6167, + "lat": 58.4167 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "ip": "89.160.20.156" + }, "event": { "action": "BLOCK", - "ingested": "2021-12-09T16:12:01.711639Z", + "ingested": "2021-12-14T10:30:59.169813400Z", "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"UNKNOWN\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"alb\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[],\"uri\":\"\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"POST\",\"requestId\":\"null\"},\"labels\":[{\"name\":\"value\"}]}", "category": "web", "type": [ @@ -415,6 +408,13 @@ }, "arn": "arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111" } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" } } ] diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json index 778ef7889f..2f07154767 100644 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json @@ -9,6 +9,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -42,7 +54,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653051900Z", + "ingested": "2021-12-14T10:41:41.359783700Z", "original": "67.43.156.13 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -77,6 +89,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -112,7 +136,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653067Z", + "ingested": "2021-12-14T10:41:41.359794100Z", "original": "67.43.156.13 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -147,6 +171,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -180,7 +216,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653072600Z", + "ingested": "2021-12-14T10:41:41.359798300Z", "original": "67.43.156.13 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -215,6 +251,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -248,7 +296,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653079800Z", + "ingested": "2021-12-14T10:41:41.359802800Z", "original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -283,6 +331,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -318,7 +378,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653088800Z", + "ingested": "2021-12-14T10:41:41.359862200Z", "original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -353,6 +413,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -386,7 +458,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653098400Z", + "ingested": "2021-12-14T10:41:41.359871600Z", "original": "67.43.156.13 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -421,6 +493,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -454,7 +538,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653105800Z", + "ingested": "2021-12-14T10:41:41.359880100Z", "original": "67.43.156.13 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -489,6 +573,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.13", "ip": "67.43.156.13" }, @@ -522,7 +618,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653111900Z", + "ingested": "2021-12-14T10:41:41.359888600Z", "original": "67.43.156.13 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -590,7 +686,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653118Z", + "ingested": "2021-12-14T10:41:41.359897600Z", "original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -658,7 +754,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653124800Z", + "ingested": "2021-12-14T10:41:41.359902500Z", "original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -726,7 +822,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653133100Z", + "ingested": "2021-12-14T10:41:41.359906200Z", "original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -794,7 +890,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:40.653141500Z", + "ingested": "2021-12-14T10:41:41.359912300Z", "original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json index 6c162949a2..dd2fdbded8 100644 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json @@ -44,7 +44,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:42.512992900Z", + "ingested": "2021-12-14T10:41:43.454109200Z", "original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -112,7 +112,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:42.513002400Z", + "ingested": "2021-12-14T10:41:43.454117800Z", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -149,6 +149,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.14", "ip": "67.43.156.14" }, @@ -182,7 +194,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:42.513008900Z", + "ingested": "2021-12-14T10:41:43.454124500Z", "original": "10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -217,6 +229,18 @@ } }, "source": { + "geo": { + "continent_name": "Asia", + "country_name": "Bhutan", + "location": { + "lon": 90.5, + "lat": 27.5 + }, + "country_iso_code": "BT" + }, + "as": { + "number": 35908 + }, "address": "67.43.156.14", "ip": "67.43.156.14" }, @@ -250,7 +274,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:42.513013400Z", + "ingested": "2021-12-14T10:41:43.454129500Z", "original": "67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\n\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"\n2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -306,7 +330,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-12-09T13:41:42.513018600Z", + "ingested": "2021-12-14T10:41:43.454135700Z", "original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nunix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"\nlocalhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nlocalhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\n", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json index 208427cae3..96ccb026f4 100644 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json @@ -48,7 +48,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:43.172715Z", + "ingested": "2021-12-14T10:41:44.167906500Z", "original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com 10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com:80 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\nexample.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -90,18 +90,12 @@ "source": { "geo": { "continent_name": "Europe", - "country_name": "Denmark", + "country_name": "Norway", "location": { "lon": 10.0, - "lat": 56.0 + "lat": 62.0 }, - "country_iso_code": "DK" - }, - "as": { - "number": 62121, - "organization": { - "name": "Christian Ebsen ApS" - } + "country_iso_code": "NO" }, "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" @@ -138,7 +132,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:43.172728700Z", + "ingested": "2021-12-14T10:41:44.167917900Z", "original": "67.43.156.15 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -194,7 +188,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-12-09T13:41:43.172732700Z", + "ingested": "2021-12-14T10:41:44.167925Z", "original": "67.43.156.15:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nexample.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -254,7 +248,7 @@ } }, "event": { - "ingested": "2021-12-09T13:41:43.172738700Z", + "ingested": "2021-12-14T10:41:44.167931700Z", "original": "67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json index 76feff53fe..032114eef0 100644 --- a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json +++ b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json @@ -20,7 +20,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-09T13:41:43.902411600Z", + "ingested": "2021-12-14T10:41:44.899046100Z", "original": "2016/10/25 14:49:34 [error] 54053#0: *1 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "category": [ "web" @@ -56,7 +56,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-09T13:41:43.902415500Z", + "ingested": "2021-12-14T10:41:44.899058700Z", "original": "2016/10/25 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "category": [ "web" @@ -92,7 +92,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-09T13:41:43.902420Z", + "ingested": "2021-12-14T10:41:44.899066900Z", "original": "2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "category": [ "web" @@ -128,7 +128,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-09T13:41:43.902426Z", + "ingested": "2021-12-14T10:41:44.899074900Z", "original": "2019/11/05 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "category": [ "web"