From 0e7a31dab88630e099801b770a0ea55b81730572 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 23 Mar 2021 23:53:28 +0000 Subject: [PATCH 01/16] Update Nginx pipelines --- CHANGELOG.next.asciidoc | 1 + .../module/apache/access/ingest/pipeline.yml | 24 +++++++-- .../module/nginx/access/ingest/pipeline.yml | 27 ++++++++-- filebeat/module/nginx/access/test/access.log | 1 + .../access/test/access.log-expected.json | 53 +++++++++++++++++++ .../test/test-with-host.log-expected.json | 10 ++++ .../nginx/access/test/test.log-expected.json | 10 ++++ .../module/nginx/error/ingest/pipeline.yml | 5 +- .../nginx/error/test/error.log-expected.json | 4 ++ 9 files changed, 125 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9d2388169a91..8368827f529f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -404,6 +404,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928] - Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038] - Change `checkpoint.source_object` from Long to Keyword. {issue}25124[25124] {pull}25145[25145] +- Fix Nginx module pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index a9f23eb4a103..f5bddc0ca5a2 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -8,21 +8,37 @@ processors: field: message patterns: - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "-" %{NUMBER:http.response.status_code:long} -' - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} - %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true +- urldecode: + field: url_original_tmp + target_: url.original + ignore_missing: true + ignore_failure: true +- urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true +- grok: + field: url_original_tmp + patterns: + - '%{URIPATH:url.path}(\?%{DATA:url.query})?' + ignore_missing: true - remove: - field: message + field: + - message + - url_original_tmp - set: field: event.kind value: event diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index 57fe9031b557..ca1bd8199765 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -4,8 +4,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" @@ -19,11 +22,27 @@ processors: - grok: field: nginx.access.info patterns: - - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' + - '%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}' - "" ignore_missing: true +- urldecode: + field: url_original_tmp + target_: url.original + ignore_missing: true + ignore_failure: true +- urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true +- grok: + field: url_original_tmp + patterns: + - '%{URIPATH:url.path}(\?%{DATA:url.query})?' + ignore_missing: true - remove: - field: nginx.access.info + field: + - nginx.access.info + - url_original_tmp - split: field: nginx.access.remote_ip_list separator: '"?,?\s+' @@ -93,8 +112,6 @@ processors: patterns: - ^%{IP:source.ip}$ ignore_failure: true -- remove: - field: message - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/nginx/access/test/access.log b/filebeat/module/nginx/access/test/access.log index 58991d26b3b9..fcf2271c7413 100644 --- a/filebeat/module/nginx/access/test/access.log +++ b/filebeat/module/nginx/access/test/access.log @@ -10,3 +10,4 @@ 127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index 7981a316c952..83ed78b01a83 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -55,6 +56,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -104,6 +106,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -152,6 +155,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -200,6 +204,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -249,6 +254,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -297,6 +303,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -345,6 +352,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -393,6 +401,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -431,6 +440,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -469,6 +479,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -507,6 +518,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -536,5 +548,46 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.12", "user_agent.version": "49.0." + }, + { + "@timestamp": "2020-06-09T19:10:39.000Z", + "destination.domain": "lessons.example.com", + "event.category": [ + "web" + ], + "event.dataset": "nginx.access", + "event.kind": "event", + "event.module": "nginx", + "event.original": "lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "http://lessons.example.com/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "http.response.body.bytes": 7648063, + "http.response.status_code": 206, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2346, + "nginx.access.remote_ip_list": [ + "192.168.0.1" + ], + "related.ip": [ + "192.168.0.1" + ], + "service.type": "nginx", + "source.address": "192.168.0.1", + "source.ip": "192.168.0.1", + "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user_agent.device.name": "Kindle", + "user_agent.name": "Amazon Silk", + "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", + "user_agent.os.full": "Android 5.1.1", + "user_agent.os.name": "Android", + "user_agent.os.version": "5.1.1", + "user_agent.version": "81.2.16" } ] \ No newline at end of file diff --git a/filebeat/module/nginx/access/test/test-with-host.log-expected.json b/filebeat/module/nginx/access/test/test-with-host.log-expected.json index e07836ce5208..36c1ebe44c16 100644 --- a/filebeat/module/nginx/access/test/test-with-host.log-expected.json +++ b/filebeat/module/nginx/access/test/test-with-host.log-expected.json @@ -8,6 +8,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -49,6 +50,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -88,6 +90,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -140,6 +143,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -190,6 +194,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -238,6 +243,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -283,6 +289,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -314,6 +321,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -336,6 +344,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -374,6 +383,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index b27c9ccf19bc..263cd2f52fba 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -47,6 +48,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -85,6 +87,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -135,6 +138,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -183,6 +187,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -230,6 +235,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -272,6 +278,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -300,6 +307,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -321,6 +329,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -355,6 +364,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 05691eeb7375..51170f6cbbbf 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -3,8 +3,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}' diff --git a/filebeat/module/nginx/error/test/error.log-expected.json b/filebeat/module/nginx/error/test/error.log-expected.json index 8896a4907059..ac5e5f83caf7 100644 --- a/filebeat/module/nginx/error/test/error.log-expected.json +++ b/filebeat/module/nginx/error/test/error.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2016/10/25 14:49:34 [error] 54053#0: *1 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "event.timezone": "-02:00", "event.type": [ "error" @@ -29,6 +30,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2016/10/25 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "event.timezone": "-02:00", "event.type": [ "error" @@ -51,6 +53,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "event.timezone": "-02:00", "event.type": [ "error" @@ -76,6 +79,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2019/11/05 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "event.timezone": "-02:00", "event.type": [ "error" From 49c2691de721e916b5d2e38207c927564d7edd8f Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 25 Mar 2021 19:44:35 +0000 Subject: [PATCH 02/16] Update Apache, Nginx, IIS, Traefik pipelines --- CHANGELOG.next.asciidoc | 1 + .../module/apache/access/ingest/pipeline.yml | 55 ++++++-- .../test/darwin-2.4.23.log-expected.json | 12 ++ .../access/test/ssl-request.log-expected.json | 12 +- .../access/test/test-vhost.log-expected.json | 3 + filebeat/module/apache/access/test/test.log | 1 + .../apache/access/test/test.log-expected.json | 42 ++++++ .../test/ubuntu-2.2.22.log-expected.json | 21 +++ .../module/apache/error/ingest/pipeline.yml | 54 +++++++- .../test/darwin-2.4.23.log-expected.json | 2 + .../apache/error/test/test.log-expected.json | 14 +- .../test/ubuntu-2.2.22.log-expected.json | 35 ++++- .../module/iis/access/ingest/pipeline.yml | 66 +++++++-- .../test/test-iis-7.2.log-expected.json | 19 ++- .../test/test-iis-7.5.log-expected.json | 8 ++ .../test/test-ipv6zone.log-expected.json | 3 + ...t-x-forward-for-extended.log-expected.json | 24 ++++ .../test/test-x-forward-for.log-expected.json | 22 ++- .../iis/access/test/test.log-expected.json | 14 +- filebeat/module/iis/error/ingest/pipeline.yml | 44 +++++- .../test/iis_error_url.log-expected.json | 35 +++-- .../error/test/ipv6_zone_id.log-expected.json | 1 + .../iis/error/test/test.log-expected.json | 15 ++- .../module/nginx/access/ingest/pipeline.yml | 44 ++++-- filebeat/module/nginx/access/test/access.log | 1 + .../access/test/access.log-expected.json | 60 +++++++++ .../test/test-with-host.log-expected.json | 19 ++- .../nginx/access/test/test.log-expected.json | 10 ++ .../nginx/ingress_controller/test/test.log | 1 + .../test/test.log-expected.json | 126 +++++++++++++++++- .../module/traefik/access/ingest/pipeline.yml | 51 +++++-- .../access/test/test.log-expected.json | 19 +++ 32 files changed, 754 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8368827f529f..0fe7883d70af 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -848,6 +848,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] +- Update URI decoding and parsing across multiple modules. {issue}19088[19088] {pull}24699[24699] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index f5bddc0ca5a2..5befc027d630 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -4,41 +4,45 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "-" %{NUMBER:http.response.status_code:long} -' - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} - %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url_original_tmp} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true - urldecode: - field: url_original_tmp - target_: url.original + field: url.original ignore_missing: true ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true ignore_failure: true -- grok: - field: url_original_tmp - patterns: - - '%{URIPATH:url.path}(\?%{DATA:url.query})?' - ignore_missing: true +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: field: - - message - - url_original_tmp + - _tmp.url_orig + ignore_missing: true - set: field: event.kind value: event @@ -113,6 +117,31 @@ processors: } ctx.tls.version_protocol = parts[0]; +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + on_failure: - set: field: error.message diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json index 1f3600f2e09c..468bfb9fdc9e 100644 --- a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -17,6 +18,7 @@ "source.address": "::1", "source.ip": "::1", "url.original": "/", + "url.path": "/", "user.name": "-" }, { @@ -25,6 +27,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -36,7 +39,9 @@ "service.type": "apache", "source.address": "::1", "source.ip": "::1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-" }, { @@ -45,6 +50,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, @@ -61,6 +67,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -83,6 +90,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user.name": "-" }, { @@ -91,6 +99,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -113,6 +122,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/notfound", + "url.path": "/notfound", "user.name": "-" }, { @@ -121,6 +131,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -143,6 +154,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/hmm", + "url.path": "/hmm", "user.name": "-" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index 3eb3e283b198..c815ede3c46c 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", "fileset.name": "access", "http.request.method": "GET", "http.response.body.bytes": 1375, @@ -19,7 +20,10 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", - "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" + "url.extension": "php", + "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "url.path": "/nagiosxi/ajaxhelper.php", + "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" }, { "@timestamp": "2019-10-16T09:53:47.000Z", @@ -29,6 +33,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "[16/Oct/2019:11:53:47 +0200] 11.19.0.217 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", "fileset.name": "access", "http.request.method": "GET", "http.version": "1.1", @@ -45,6 +50,9 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", - "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" + "url.extension": "php", + "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "url.path": "/appl/ajaxhelper.php", + "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json index b332788ad2b0..64f4926d5865 100644 --- a/filebeat/module/apache/access/test/test-vhost.log-expected.json +++ b/filebeat/module/apache/access/test/test-vhost.log-expected.json @@ -6,6 +6,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -17,7 +18,9 @@ "log.offset": 0, "service.type": "apache", "source.ip": "192.168.33.2", + "url.domain": "vhost1.domaine.fr", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/apache/access/test/test.log b/filebeat/module/apache/access/test/test.log index 7422d4415bd9..362543c91a8f 100644 --- a/filebeat/module/apache/access/test/test.log +++ b/filebeat/module/apache/access/test/test.log @@ -4,3 +4,4 @@ 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index ebe888475861..c0a03f36aa48 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -16,7 +17,9 @@ "service.type": "apache", "source.address": "::1", "source.ip": "::1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-" }, { @@ -25,6 +28,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -38,6 +42,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -53,6 +58,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, @@ -69,6 +75,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -82,6 +89,7 @@ "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", @@ -97,6 +105,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -110,6 +119,7 @@ "source.address": "monitoring-server", "source.domain": "monitoring-server", "url.original": "/status", + "url.path": "/status", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", @@ -125,6 +135,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "event.outcome": "failure", "fileset.name": "access", "http.request.referrer": "-", @@ -139,5 +150,36 @@ "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "-" + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": "web", + "event.dataset": "apache.access", + "event.kind": "event", + "event.module": "apache", + "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "event.outcome": "success", + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 612, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 666, + "service.type": "apache", + "source.address": "monitoring-server", + "source.domain": "monitoring-server", + "url.extension": "mp4", + "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox Alpha", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "user_agent.os.full": "Windows 7", + "user_agent.os.name": "Windows", + "user_agent.os.version": "7", + "user_agent.version": "15.0.a2" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json index e9680e5b7fbc..bcff56976927 100644 --- a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -18,6 +19,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Wget", @@ -31,6 +33,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -44,6 +47,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -59,6 +63,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -71,7 +76,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -87,6 +94,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -100,6 +108,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -115,6 +124,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -127,7 +137,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -143,6 +155,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -155,7 +168,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -171,6 +186,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -184,6 +200,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/test", + "url.path": "/test", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -199,6 +216,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -212,6 +230,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -227,6 +246,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -240,6 +260,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/crap", + "url.path": "/crap", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index aad4c3f4a5f5..f6e1abc05749 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -3,8 +3,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} @@ -14,6 +17,27 @@ processors: pattern_definitions: APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' ignore_missing: true +- grok: + field: message + patterns: + - "File does not exist: %{URIPATH:_tmp.url_orig}, referer: %{URI:http.request.referrer}" + - "File does not exist: %{URIPATH:_tmp.url_orig}" + ignore_missing: true + ignore_failure: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true + if: ctx._tmp?.url_orig != null +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + if: ctx._tmp?.url?.original != null +- urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true + if: ctx._tmp?.http?.request?.referrer != null - date: if: ctx.event.timezone == null field: apache.error.timestamp @@ -38,7 +62,9 @@ processors: field: error.message value: '{{ _ingest.on_failure_message }}' - remove: - field: apache.error.timestamp + field: + - apache.error.timestamp + - _tmp.url_orig ignore_failure: true - set: field: event.kind @@ -82,6 +108,30 @@ processors: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json index 4453f2bebec4..90990fc95a47 100644 --- a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json @@ -6,6 +6,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -23,6 +24,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json index d9f470db46a4..95155ce5f88d 100644 --- a/filebeat/module/apache/error/test/test.log-expected.json +++ b/filebeat/module/apache/error/test/test.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -14,7 +15,10 @@ "message": "File does not exist: /var/www/favicon.ico", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.extension": "ico", + "url.original": "/var/www/favicon.ico", + "url.path": "/var/www/favicon.ico" }, { "@timestamp": "2016-12-26T16:15:55.103-02:00", @@ -23,6 +27,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -40,6 +45,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -61,7 +67,10 @@ "source.geo.location.lon": -84.8154, "source.geo.region_iso_code": "US-GA", "source.geo.region_name": "Georgia", - "source.ip": "72.15.99.187" + "source.ip": "72.15.99.187", + "url.extension": "ico", + "url.original": "/usr/local/apache2/htdocs/favicon.ico", + "url.path": "/usr/local/apache2/htdocs/favicon.ico" }, { "@timestamp": "2019-06-27T06:58:09.169-02:00", @@ -70,6 +79,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", diff --git a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json index 41adf5d7ae58..7d2d731ecc6c 100644 --- a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -20,16 +21,21 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", + "http.request.referrer": "http://192.168.33.72/", "input.type": "log", "log.level": "error", "log.offset": 100, "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.extension": "ico", + "url.original": "/var/www/favicon.ico", + "url.path": "/var/www/favicon.ico" }, { "@timestamp": "2016-12-26T16:22:08.000-02:00", @@ -37,6 +43,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -46,7 +53,10 @@ "message": "File does not exist: /var/www/favicon.ico", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.extension": "ico", + "url.original": "/var/www/favicon.ico", + "url.path": "/var/www/favicon.ico" }, { "@timestamp": "2016-12-26T16:22:08.000-02:00", @@ -54,6 +64,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -63,7 +74,10 @@ "message": "File does not exist: /var/www/favicon.ico", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.extension": "ico", + "url.original": "/var/www/favicon.ico", + "url.path": "/var/www/favicon.ico" }, { "@timestamp": "2016-12-26T16:22:10.000-02:00", @@ -71,6 +85,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -80,7 +95,9 @@ "message": "File does not exist: /var/www/test", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.original": "/var/www/test", + "url.path": "/var/www/test" }, { "@timestamp": "2016-12-26T16:22:13.000-02:00", @@ -88,6 +105,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -97,7 +115,9 @@ "message": "File does not exist: /var/www/hello", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.original": "/var/www/hello", + "url.path": "/var/www/hello" }, { "@timestamp": "2016-12-26T16:22:17.000-02:00", @@ -105,6 +125,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", @@ -114,6 +135,8 @@ "message": "File does not exist: /var/www/crap", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1" + "source.ip": "192.168.33.1", + "url.original": "/var/www/crap", + "url.path": "/var/www/crap" } ] \ No newline at end of file diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 2e92efd48d56..2426360f1a17 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -4,42 +4,68 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{NOTSPACE:iis.access.site_name}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:iis.access.cookie}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NOTSPACE:destination.domain}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:http.response.body.bytes:long}) (?:-|%{NUMBER:http.request.body.bytes:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{NOTSPACE:iis.access.site_name}) (?:-|%{NOTSPACE:iis.access.server_name}) - (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:url.path}) - (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.url_path}) + (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|HTTP/%{NUMBER:http.version}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:iis.access.cookie}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NOTSPACE:destination.domain}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:http.response.body.bytes:long}) (?:-|%{NUMBER:http.request.body.bytes:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} \[%{IPORHOST:destination.address}\]\(http://%{IPORHOST:destination.address}\) - (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) + (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) \[%{IPORHOST:source.address}\]\(http://%{IPORHOST:source.address}\) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' ignore_missing: true +- uri_parts: + field: _tmp.url_path + ignore_failure: true +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true +- urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true +- urldecode: + field: _tmp.url_query + target_field: url.query + ignore_missing: true + ignore_failure: true + if: ctx.url?.query == null +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: message + field: + - _tmp + ignore_missing: true - rename: field: '@timestamp' target_field: event.created @@ -133,6 +159,30 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json index 64ad587bb8bc..5f37c7d4e43b 100644 --- a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 0", "event.outcome": "failure", "event.type": [ "connection" @@ -30,8 +31,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -53,6 +56,7 @@ "event.duration": 46000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 46", "event.outcome": "failure", "event.type": [ "connection" @@ -71,8 +75,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -94,6 +100,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /Director - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -112,6 +119,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/Director", "url.path": "/Director", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -134,6 +142,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET / - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -152,6 +161,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -174,6 +184,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 15", "event.outcome": "failure", "event.type": [ "connection" @@ -192,8 +203,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json index 952105369253..df3c9df6d04d 100644 --- a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 792000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-08-28 18:24:25 [10.100.220.70](http://10.100.220.70) GET / - 80 - [10.100.118.31](http://10.100.118.31) Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729) 404 4 2 792", "event.outcome": "failure", "event.type": [ "connection" @@ -30,6 +31,7 @@ "service.type": "iis", "source.address": "10.100.118.31", "source.ip": "10.100.118.31", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -52,6 +54,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -70,6 +73,7 @@ "service.type": "iis", "source.address": "10.0.140.2", "source.ip": "10.0.140.2", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" }, { @@ -85,6 +89,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -103,6 +108,7 @@ "service.type": "iis", "source.address": "10.0.140.2", "source.ip": "10.0.140.2", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" }, { @@ -118,6 +124,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 2001:cdba:0000:0000:0000:0000:3257:9652 GET /health-monitoring - 80 - 2001:cdba:0000:0000:0000:0000:3257:9652 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -136,6 +143,7 @@ "service.type": "iis", "source.address": "2001:cdba:0000:0000:0000:0000:3257:9652", "source.ip": "2001:cdba:0000:0000:0000:0000:3257:9652", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" } ] \ No newline at end of file diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json index c3f4a4932dac..35ce856496dc 100644 --- a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json +++ b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json @@ -13,6 +13,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME ::1%0 GET / - 80 - ::1%0 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "event.type": [ "connection" @@ -36,6 +37,8 @@ "service.type": "iis", "source.address": "::1%0", "source.ip": "::1", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", diff --git a/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json b/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json index 565bdfca17d6..edcefdf8e65c 100644 --- a/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json +++ b/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json @@ -13,6 +13,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-04 22:00:34 W3SVC2 freca1 10.24.129.162 GET /favicon.ico - 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64;+rv:81.0)+Gecko/20100101+Firefox/81.0 - https://images.hogeschoolrotterdam.nl/Blob/adeec119008c48758c1a6be53aeeb2ac/34ff475072d54117bcb46ea7f023bd87.jpg?width=1200&height=630&mode=crop images.hogeschoolrotterdam.nl 404 0 2 1437 534 0 116.189.86.89", "event.outcome": "failure", "event.type": [ "connection" @@ -38,6 +39,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "ico", + "url.original": "/favicon.ico", "url.path": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -61,6 +65,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:40:30 W3SVC2 freca1 10.24.129.162 GET /robots.txt - 80 - 10.24.136.240 HTTP/1.1 Twitterbot/1.0 - - images.hogeschoolrotterdam.nl 200 0 0 346 306 0 119.16.157.180", "event.outcome": "success", "event.type": [ "connection" @@ -85,6 +90,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "txt", + "url.original": "/robots.txt", "url.path": "/robots.txt", "user_agent.device.name": "Spider", "user_agent.name": "Twitterbot", @@ -105,6 +113,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:48:33 W3SVC2 freca1 10.24.129.162 GET /app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg width=35&height=38&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36+Edge/18.18362 BIGipServerYAkgvoMsadfHHYQAGkWEWnyAqAads=27246369425.20480.0000 https://www.rotterdamuas.com/ images.hogeschoolrotterdam.nl 200 0 0 2007 581 15 119.160.162.213", "event.outcome": "success", "event.type": [ "connection" @@ -131,6 +140,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg", "url.path": "/app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg", "url.query": "width=35&height=38&mode=crop", "user_agent.device.name": "Other", @@ -155,6 +167,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:48:33 W3SVC2 freca1 10.24.129.162 GET /app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg width=75&height=40&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36+Edge/18.18362 BIGipServerYAkgvoMHHYQAGkWEWadfsadfnyAqAere=27246369425.20480.0000 https://www.rotterdamuas.com/ images.hogeschoolrotterdam.nl 200 0 0 2926 581 0 119.160.162.213", "event.outcome": "success", "event.type": [ "connection" @@ -181,6 +194,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg", "url.path": "/app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg", "url.query": "width=75&height=40&mode=crop", "user_agent.device.name": "Other", @@ -205,6 +221,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-08 22:00:22 W3SVC2 freca1 10.24.129.162 GET /Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png width=60&height=20&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(iPhone;+CPU+iPhone+OS+13_7+like+Mac+OS+X)+AppleWebKit/605.1.15+(KHTML,+like+Gecko)+Version/13.1.2+Mobile/15E148+Safari/604.1 imagedpi=1;+imagequality=30;+_ga=GA1.2.14440223724.16021494379;+_gid=GA1.2.12504478161.16021944379;+_gat_UA-155746052-5=1;+BIGipServerYAkgvoMHHYQAGkWsadfsdfEWnyAqA=!vbipVwbt0UWJ4QSsPvjjVxViTXMqdXWYaferu1dRyQoUTVVBGLZPB39PoNX5Tw66+GT0ChJCfnp/xfhM8lI=;+_gcl_au=1.1.1297303538.1602194379 https://www.hogeschoolrotterdam.nl/opleidingen/bachelor/elektrotechniek/voltijd/ images.hogeschoolrotterdam.nl 304 0 0 388 979 15 156.189.143.218", "event.outcome": "success", "event.type": [ "connection" @@ -231,6 +248,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "png", + "url.original": "/Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png", "url.path": "/Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png", "url.query": "width=60&height=20&mode=crop", "user_agent.device.name": "iPhone", @@ -255,6 +275,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-08 22:00:22 W3SVC2 freca1 10.24.129.162 GET /Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg width=60&height=20&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(iPhone;+CPU+iPhone+OS+13_7+like+Mac+OS+X)+AppleWebKit/605.1.15+(KHTML,+like+Gecko)+Version/13.1.2+Mobile/15E148+Safari/604.1 imagedpi=1;+imagequality=30;+_ga=GA1.2.14440223724.16021494379;+_gid=GA1.2.12504748161.16021944379;+_gat_UA-155764052-5=1;+BIGipServerYAkgvoMHHYQAGkWEsadfsdfsWnyAqA=!vbipVwbt0UWJ4QSsPvjjVxViTXMqdXWYu1dRyQoUTVerwerVBGLZPB39PoNX5Tw66+GT0ChJCfnp/xfhM8lI=;+_gcl_au=1.1.1297303538.1602194379 https://www.hogeschoolrotterdam.nl/opleidingen/bachelor/elektrotechniek/voltijd/ images.hogeschoolrotterdam.nl 304 0 0 388 979 0 156.189.143.218", "event.outcome": "success", "event.type": [ "connection" @@ -281,6 +302,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg", "url.path": "/Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg", "url.query": "width=60&height=20&mode=crop", "user_agent.device.name": "iPhone", diff --git a/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json b/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json index 4d8ace5a7fb5..8977e920292e 100644 --- a/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json +++ b/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 26000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 26 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -32,6 +33,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -54,6 +56,7 @@ "event.duration": 32000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 32 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -74,6 +77,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -96,6 +100,7 @@ "event.duration": 46000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 46 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -116,6 +121,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -138,6 +144,7 @@ "event.duration": 32000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 GET /Production-UI/data/finance/legacy/GLAPAprvMaster $filter=BatchId%20eq%20%27FY21HSNG0820%27&$orderby=Subsys,Ref&$skip=0&$top=20 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 32 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -158,8 +165,9 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/data/finance/legacy/GLAPAprvMaster", "url.path": "/Production-UI/data/finance/legacy/GLAPAprvMaster", - "url.query": "$filter=BatchId%20eq%20%27FY21HSNG0820%27&$orderby=Subsys,Ref&$skip=0&$top=20", + "url.query": "$filter=BatchId eq 'FY21HSNG0820'&$orderby=Subsys,Ref&$skip=0&$top=20", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", @@ -181,6 +189,7 @@ "event.duration": 166000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 GET /Production-UI/data/finance/legacy/GLATrnsDetail $filter=Subsys%20eq%20%27JE%27%20and%20Ref%20eq%20%27HSNG08-MR%27%20and%20BatchId%20eq%20%27FY21HSNG0820%27&$orderby=RecNo&$skip=0&$top=20 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 166 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -201,8 +210,9 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/data/finance/legacy/GLATrnsDetail", "url.path": "/Production-UI/data/finance/legacy/GLATrnsDetail", - "url.query": "$filter=Subsys%20eq%20%27JE%27%20and%20Ref%20eq%20%27HSNG08-MR%27%20and%20BatchId%20eq%20%27FY21HSNG0820%27&$orderby=RecNo&$skip=0&$top=20", + "url.query": "$filter=Subsys eq 'JE' and Ref eq 'HSNG08-MR' and BatchId eq 'FY21HSNG0820'&$orderby=RecNo&$skip=0&$top=20", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", @@ -224,6 +234,7 @@ "event.duration": 60000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 GET /Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 60 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -244,6 +255,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB", "url.path": "/Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -266,6 +278,7 @@ "event.duration": 72000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/ - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 72 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -286,6 +299,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/", "url.path": "/Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -308,6 +322,7 @@ "event.duration": 88000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/ - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 88 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -328,6 +343,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/", "url.path": "/Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -350,6 +366,7 @@ "event.duration": 286000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:07:02 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/attachDoc - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 286 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -370,6 +387,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/attachDoc", "url.path": "/Production-UI/api/finance/legacy/documents/attachDoc", "user_agent.device.name": "Other", "user_agent.name": "Chrome", diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 786333c1379d..ca7809e8571d 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 123000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 08:09:10 127.0.0.1 GET / q=100 80 - 85.181.35.98 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - 200 0 0 123", "event.outcome": "success", "event.type": [ "connection" @@ -40,6 +41,7 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.original": "/", "url.path": "/", "url.query": "q=100", "user_agent.device.name": "Other", @@ -61,6 +63,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 09:10:11 W3SVC1 GET / - 80 - 127.0.0.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "fileset.name": "access", "http.request.body.bytes": 456, @@ -78,6 +81,8 @@ "service.type": "iis", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -101,6 +106,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "event.type": [ "connection" @@ -134,6 +140,8 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -156,6 +164,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:52:33 10.44.0.136 GET / redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()} 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -174,8 +183,9 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/", "url.path": "/", - "url.query": "redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()}", + "url.query": "redirect:${#req=#context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),#webroot=#req.getSession().getServletContext().getRealPath('/'),#resp.println(#webroot),#resp.flush(),#resp.close()}", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -197,6 +207,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:52:33 10.44.0.136 GET /${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 0", "event.outcome": "failure", "event.type": [ "connection" @@ -215,7 +226,6 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", - "url.path": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index a16fde841daf..f5ba07111d88 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -3,21 +3,33 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) - (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:url.original}) (?:%{NUMBER}|-) + (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:_tmp.url_orig}) (?:%{NUMBER}|-) (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:-|%{NOTSPACE:iis.error.reason_phrase}) (?:-|%{NOTSPACE:iis.error.queue_name})' - '%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) - (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:url.original}) (?:%{NUMBER:http.response.status_code:long}|-) + (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:_tmp.url_orig}) (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:-|%{NOTSPACE:iis.error.reason_phrase}) (?:-|%{NOTSPACE:iis.error.queue_name})' ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true - remove: - field: message + field: + - _tmp + ignore_missing: true - rename: field: '@timestamp' target_field: event.created @@ -90,6 +102,30 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/iis/error/test/iis_error_url.log-expected.json b/filebeat/module/iis/error/test/iis_error_url.log-expected.json index 8adb2e6d1d06..fddcdc433525 100644 --- a/filebeat/module/iis/error/test/iis_error_url.log-expected.json +++ b/filebeat/module/iis/error/test/iis_error_url.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/0.9 t3 12.2.1 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -35,7 +36,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "12.2.1" + "url.extension": "1", + "url.original": "12.2.1", + "url.path": "12.2.1" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -49,6 +52,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET ./././././../../../../../../../../ 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -73,7 +77,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "./././././../../../../../../../../" + "url.extension": "/", + "url.original": "./././././../../../../../../../../", + "url.path": "./././././../../../../../../../../" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -87,6 +93,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /..\\pixfir~1\\how_to_login.html 403 - Forbidden -", "event.outcome": "failure", "event.type": [ "connection" @@ -110,8 +117,7 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", - "source.port": 12345, - "url.original": "/..\\pixfir~1\\how_to_login.html" + "source.port": 12345 }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -125,6 +131,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET ..\\..\\..\\..\\..\\..\\winnt\\win.ini 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -148,8 +155,7 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", - "source.port": 12345, - "url.original": "..\\..\\..\\..\\..\\..\\winnt\\win.ini" + "source.port": 12345 }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -163,6 +169,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini 404 - NotFound -", "event.outcome": "failure", "event.type": [ "connection" @@ -187,7 +194,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini" + "url.extension": "ini", + "url.original": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini", + "url.path": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -201,6 +210,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /nessus\\..\\..\\..\\..\\..\\..\\winnt\\win.ini 403 - Forbidden -", "event.outcome": "failure", "event.type": [ "connection" @@ -224,8 +234,7 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", - "source.port": 12345, - "url.original": "/nessus\\..\\..\\..\\..\\..\\..\\winnt\\win.ini" + "source.port": 12345 }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -239,6 +248,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 OPTIONS * 404 - NotFound -", "event.outcome": "failure", "event.type": [ "connection" @@ -263,7 +273,8 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "*" + "url.original": "*", + "url.path": "*" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -277,6 +288,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /fee&fie=foe 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -301,6 +313,7 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/fee&fie=foe" + "url.original": "/fee&fie=foe", + "url.path": "/fee&fie=foe" } ] \ No newline at end of file diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json index ebcb2f36b842..e77f92bcc56a 100644 --- a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-30 14:22:07 ::1%0 49958 ::1%0 80 - - - - - - Timer_ConnectionIdle -", "event.type": [ "connection" ], diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json index 506ee6ba2edd..fdf8fa4bdf16 100644 --- a/filebeat/module/iis/error/test/test.log-expected.json +++ b/filebeat/module/iis/error/test/test.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 08:09:10 172.31.77.6 2094 172.31.77.6 80 HTTP/1.1 GET /qos/1kbfile.txt 503 - ConnLimit -", "event.outcome": "failure", "event.type": [ "connection" @@ -30,7 +31,9 @@ "source.address": "172.31.77.6", "source.ip": "172.31.77.6", "source.port": 2094, - "url.original": "/qos/1kbfile.txt" + "url.extension": "txt", + "url.original": "/qos/1kbfile.txt", + "url.path": "/qos/1kbfile.txt" }, { "@timestamp": "2018-01-01T09:10:11.000Z", @@ -44,6 +47,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 09:10:11 85.181.35.98 2780 127.0.0.1 80 HTTP/1.1 GET /ThisIsMyUrl.htm 400 - Hostname -", "event.outcome": "failure", "event.type": [ "connection" @@ -73,7 +77,9 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "source.port": 2780, - "url.original": "/ThisIsMyUrl.htm" + "url.extension": "htm", + "url.original": "/ThisIsMyUrl.htm", + "url.path": "/ThisIsMyUrl.htm" }, { "@timestamp": "2018-01-01T10:11:12.000Z", @@ -87,6 +93,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 85.181.35.98 2894 127.0.0.1 80 HTTP/2.0 GET / 505 - Version_N/S -", "event.outcome": "failure", "event.type": [ "connection" @@ -116,7 +123,8 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "source.port": 2894, - "url.original": "/" + "url.original": "/", + "url.path": "/" }, { "@timestamp": "2018-01-01T11:12:13.000Z", @@ -130,6 +138,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 11:12:13 85.181.35.98 64388 127.0.0.1 80 - - - - - Timer_MinBytesPerSecond -", "event.type": [ "connection" ], diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index ca1bd8199765..db8c54f8e28d 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -22,27 +22,29 @@ processors: - grok: field: nginx.access.info patterns: - - '%{WORD:http.request.method} %{DATA:url_original_tmp} HTTP/%{NUMBER:http.version}' + - '%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}' - "" ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true - urldecode: - field: url_original_tmp - target_: url.original + field: url.original ignore_missing: true ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true ignore_failure: true -- grok: - field: url_original_tmp - patterns: - - '%{URIPATH:url.path}(\?%{DATA:url.query})?' - ignore_missing: true +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: field: - nginx.access.info - - url_original_tmp + - _tmp.url_orig + ignore_missing: true - split: field: nginx.access.remote_ip_list separator: '"?,?\s+' @@ -178,6 +180,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/nginx/access/test/access.log b/filebeat/module/nginx/access/test/access.log index fcf2271c7413..7acb1428af89 100644 --- a/filebeat/module/nginx/access/test/access.log +++ b/filebeat/module/nginx/access/test/access.log @@ -11,3 +11,4 @@ 127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] "GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index 83ed78b01a83..f0f3078e8132 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -40,6 +40,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -89,7 +90,9 @@ "source.geo.region_iso_code": "DE-RP", "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -139,6 +142,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/adsasd", + "url.path": "/adsasd", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -188,6 +192,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -237,7 +242,9 @@ "source.geo.region_iso_code": "DE-RP", "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -287,6 +294,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", + "url.path": "/test", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -336,6 +344,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", + "url.path": "/test", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -385,6 +394,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test1", + "url.path": "/test1", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -424,6 +434,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/test1", + "url.path": "/test1", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -463,6 +474,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -502,6 +514,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -541,6 +554,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/taga", + "url.path": "/taga", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -581,7 +595,53 @@ "service.type": "nginx", "source.address": "192.168.0.1", "source.ip": "192.168.0.1", + "url.domain": "lessons.example.com", + "url.extension": "mp4", "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user_agent.device.name": "Kindle", + "user_agent.name": "Amazon Silk", + "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", + "user_agent.os.full": "Android 5.1.1", + "user_agent.os.name": "Android", + "user_agent.os.version": "5.1.1", + "user_agent.version": "81.2.16" + }, + { + "@timestamp": "2020-06-09T19:15:39.000Z", + "destination.domain": "lessons.example.com", + "event.category": [ + "web" + ], + "event.dataset": "nginx.access", + "event.kind": "event", + "event.module": "nginx", + "event.original": "lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "http://lessons.example.com/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "http.response.body.bytes": 7648063, + "http.response.status_code": 206, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2727, + "nginx.access.remote_ip_list": [ + "192.168.0.1" + ], + "related.ip": [ + "192.168.0.1" + ], + "service.type": "nginx", + "source.address": "192.168.0.1", + "source.ip": "192.168.0.1", + "url.domain": "lessons.example.com", + "url.original": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", + "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", "user_agent.device.name": "Kindle", "user_agent.name": "Amazon Silk", "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", diff --git a/filebeat/module/nginx/access/test/test-with-host.log-expected.json b/filebeat/module/nginx/access/test/test-with-host.log-expected.json index 36c1ebe44c16..85ba8c494f37 100644 --- a/filebeat/module/nginx/access/test/test-with-host.log-expected.json +++ b/filebeat/module/nginx/access/test/test-with-host.log-expected.json @@ -32,7 +32,9 @@ "service.type": "nginx", "source.address": "10.0.0.2", "source.ip": "10.0.0.2", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -72,7 +74,9 @@ "service.type": "nginx", "source.address": "172.17.0.1", "source.ip": "172.17.0.1", + "url.domain": "example.com", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -124,7 +128,9 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -175,7 +181,9 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -229,7 +237,10 @@ "source.geo.region_iso_code": "US-IL", "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", + "url.domain": "example.com", "url.original": "/assets/xxxx?q=100", + "url.path": "/assets/xxxx", + "url.query": "q=100", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -273,7 +284,9 @@ "source.geo.location.lat": 39.5, "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.extension": "html", "url.original": "/test.html", + "url.path": "/test.html", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -333,7 +346,8 @@ "input.type": "log", "log.offset": 1269, "service.type": "nginx", - "source.address": "unix:" + "source.address": "unix:", + "url.domain": "example.com" }, { "@timestamp": "2017-05-29T19:02:48.000Z", @@ -366,6 +380,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -402,7 +417,9 @@ ], "service.type": "nginx", "source.address": "localhost", + "url.domain": "example.com", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 263cd2f52fba..60731f0cc107 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -32,6 +32,7 @@ "source.address": "10.0.0.2", "source.ip": "10.0.0.2", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -71,6 +72,7 @@ "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -122,6 +124,7 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -171,6 +174,7 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -223,6 +227,8 @@ "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", "url.original": "/assets/xxxx?q=100", + "url.path": "/assets/xxxx", + "url.query": "q=100", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -264,7 +270,9 @@ "source.geo.location.lat": 39.5, "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.extension": "html", "url.original": "/test.html", + "url.path": "/test.html", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -348,6 +356,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -384,6 +393,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index c8ba580f64d0..ef502ebe86be 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -21,3 +21,4 @@ 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f 192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 +192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index c9cbfb36b7fd..a934be97f15f 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:48:51 +0000] \"POST /products HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 89 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 529a007902362a5f51385a5fa7049884", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -52,6 +53,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products", + "url.path": "/products", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -65,6 +67,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:15 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 91 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 68fa971ce4dfce685fdc01c877bfa645", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -110,6 +113,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -123,6 +127,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:30 +0000] \"DELETE /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 94 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 0be411044cb1cb67580e115413b2da60", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -168,6 +173,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -181,6 +187,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:43 +0000] \"PATCH /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 93 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 f479ab1d9cc8afcbac9e9f958ff8babc", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -226,6 +233,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -239,6 +247,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:50 +0000] \"PATCHp /products/42 HTTP/1.1\" 400 163 \"-\" \"-\" 0 0.000 [] [] - - - - 4c7d2079340e68353c7d0dfff00b904b", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -266,7 +275,8 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42" + "url.original": "/products/42", + "url.path": "/products/42" }, { "@timestamp": "2020-02-07T11:50:09.000Z", @@ -276,6 +286,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:50:09 +0000] \"geti /products/42 HTTP/1.1\" 400 163 \"-\" \"-\" 0 0.000 [] [] - - - - efb0c5aa8be6cdeb4a7e7bd090e3d893", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -303,7 +314,8 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42" + "url.original": "/products/42", + "url.path": "/products/42" }, { "@timestamp": "2020-02-07T11:55:05.000Z", @@ -313,6 +325,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:05 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Wget/1.20.3 (darwin18.6.0)\" 157 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 457b71c3e1ee1887bb809effd301a0ec", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -358,6 +371,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Wget", "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", @@ -371,6 +385,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 450 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 da29abf31e4d6324cebe5e7bca370709", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -416,6 +431,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -432,6 +448,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/products/42\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 381 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 e983c8cf3d713548baa50c9e2fffeb34", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -477,7 +494,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -494,6 +513,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 441 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 3d7ff18ff4181a7db5013a76f975d900", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -539,6 +559,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -555,6 +576,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/v2\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 372 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 d131fe4bcd359cf947f75efca4bfa553", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -600,7 +622,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -617,6 +641,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 ef6629fcaaa1ea0d1a843cf2bf40571d", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -662,6 +687,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -678,6 +704,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/products/42\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 325 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 2593a1126588922449c183c8d9ddbbeb", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -723,7 +750,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -740,6 +769,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.002 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 0f76ea730f282d5759018eb756b23b14", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -785,6 +815,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -801,6 +832,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET / HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 358 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 21efd18e3a7952fc78c0f2dcc1f05e69", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -846,6 +878,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -862,6 +895,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 314 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 e096809c2cb46f004c3b538b23916e5b", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -907,7 +941,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -924,6 +960,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 360 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.002 200 0a2a92d080e664dd4e95c85d097c9d3d", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -969,6 +1006,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -985,6 +1023,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/v2\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 316 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 4c024ebfc20acfb2d59e542e3ed60789", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1030,7 +1069,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -1047,6 +1088,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:00:28 +0000] \"GET /products/42?address=delhi+technological+university HTTP/1.1\" 200 59 \"-\" \"python-requests/2.22.0\" 197 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 9a7babf34ca4ee59d90ac48d452a9214", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1091,7 +1133,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42?address=delhi+technological+university", + "url.original": "/products/42?address=delhi technological university", + "url.path": "/products/42", + "url.query": "address=delhi+technological+university", "user_agent.device.name": "Other", "user_agent.name": "Python Requests", "user_agent.original": "python-requests/2.22.0", @@ -1105,6 +1149,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1150,6 +1195,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1166,6 +1212,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1210,7 +1257,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1227,6 +1276,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] \"GET /v2/some HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1272,6 +1322,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2/some", + "url.path": "/v2/some", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1288,6 +1339,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /v2/some HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1337,6 +1389,74 @@ "source.address": "192.168.64.14", "source.ip": "192.168.64.14", "url.original": "/v2/some", + "url.path": "/v2/some", + "user_agent.device.name": "Mac", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + }, + { + "@timestamp": "2020-02-07T12:02:42.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 6026, + "nginx.ingress_controller.http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "nginx.ingress_controller.http.request.length": 348, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.14" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.7", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, + "nginx.ingress_controller.upstream.response.length": 100, + "nginx.ingress_controller.upstream.response.length_list": [ + "61", + "100" + ], + "nginx.ingress_controller.upstream.response.status_code": 203, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200", + "203" + ], + "nginx.ingress_controller.upstream.response.time": 0.104, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.100", + "0.004" + ], + "nginx.ingress_controller.upstream_address_list": [ + "172.17.0.6:8080", + "172.17.0.7:8080" + ], + "related.ip": [ + "192.168.64.14" + ], + "service.type": "nginx", + "source.address": "192.168.64.14", + "source.ip": "192.168.64.14", + "url.extension": "mp4", + "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index dd5de1b0b0b6..482367c66438 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -4,10 +4,13 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- dissect: +- rename: field: message + target_field: event.original +- dissect: + field: event.original pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] - "%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code} + "%{http.request.method} %{temp.url_orig} HTTP/%{http.version}" %{http.response.status_code} %{traefik.access.message}' - grok: field: traefik.access.message @@ -17,12 +20,17 @@ processors: (?:"%{DATA:traefik.access.frontend_name}"|-)?)?( "%{DATA:traefik.access.backend_url}")?( %{NUMBER:temp.duration:long}ms)?)? ignore_missing: true -- remove: - field: message +- uri_parts: + field: temp.url_orig + ignore_failure: true +- urldecode: + field: url.original ignore_missing: true -- remove: - field: traefik.access.message + ignore_failure: true +- urldecode: + field: http.request.referrer ignore_missing: true + ignore_failure: true - rename: field: '@timestamp' target_field: event.created @@ -31,8 +39,6 @@ processors: target_field: '@timestamp' formats: - dd/MMM/yyyy:H:m:s Z -- remove: - field: traefik.access.time - convert: field: http.response.status_code type: long @@ -47,7 +53,10 @@ processors: scale: 1000000 if: ctx.temp?.duration != null - remove: - field: temp.duration + field: + - temp + - traefik.access.message + - traefik.access.time ignore_missing: true - user_agent: field: user_agent.original @@ -103,6 +112,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null && ctx.user.name != '-'" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json index 631673fe3515..ce695210bef1 100644 --- a/filebeat/module/traefik/access/test/test.log-expected.json +++ b/filebeat/module/traefik/access/test/test.log-expected.json @@ -8,6 +8,7 @@ "event.duration": 2000000, "event.kind": "event", "event.module": "traefik", + "event.original": "192.168.33.1 - - [02/Oct/2017:20:22:07 +0000] \"GET /ui/favicons/favicon-16x16.png HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 262 \"Host-host-1\" \"http://172.19.0.3:5601\" 2ms", "event.outcome": "success", "event.type": [ "access" @@ -30,7 +31,9 @@ "traefik.access.frontend_name": "Host-host-1", "traefik.access.request_count": 262, "traefik.access.user_identifier": "-", + "url.extension": "png", "url.original": "/ui/favicons/favicon-16x16.png", + "url.path": "/ui/favicons/favicon-16x16.png", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -47,6 +50,7 @@ "event.duration": 3000000, "event.kind": "event", "event.module": "traefik", + "event.original": "85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] \"GET /ui/favicons/favicon.ico HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 271 \"Host-host1\" \"http://172.19.0.3:5601\" 3ms", "event.outcome": "success", "event.type": [ "access" @@ -79,7 +83,9 @@ "traefik.access.frontend_name": "Host-host1", "traefik.access.request_count": 271, "traefik.access.user_identifier": "-", + "url.extension": "ico", "url.original": "/ui/favicons/favicon.ico", + "url.path": "/ui/favicons/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -96,6 +102,7 @@ "event.duration": 247000000, "event.kind": "event", "event.module": "traefik", + "event.original": "70.29.80.15 - - [28/Feb/2018:17:30:33 +0000] \"GET /en/ HTTP/2.0\" 200 2814 - \"Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1\" 13 \"Host-host1-com-0\" \"http://172.19.0.6:14008\" 247ms", "event.outcome": "success", "event.type": [ "access" @@ -128,6 +135,7 @@ "traefik.access.request_count": 13, "traefik.access.user_identifier": "-", "url.original": "/en/", + "url.path": "/en/", "user.name": "-", "user_agent.device.name": "iPhone", "user_agent.name": "Mobile Safari", @@ -146,6 +154,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "traefik", + "event.original": "::1 - - [29/Nov/2018:15:03:51 +0000] \"GET / HTTP/1.1\" 404 19 \"-\" \"curl/7.62.0\" 10 \"backend not found\" \"/\" 0ms", "event.outcome": "failure", "event.type": [ "access" @@ -169,6 +178,7 @@ "traefik.access.request_count": 10, "traefik.access.user_identifier": "-", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -184,6 +194,7 @@ "event.duration": 13000000, "event.kind": "event", "event.module": "traefik", + "event.original": "94.254.131.115 - - [19/Jan/2018:10:01:02 +0000] \"GET /assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo HTTP/1.1\" 200 85 - \"Android\" 623112 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.9:4140\" 13ms", "event.outcome": "success", "event.type": [ "access" @@ -216,6 +227,8 @@ "traefik.access.request_count": 623112, "traefik.access.user_identifier": "-", "url.original": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", + "url.path": "/assets/52f8f2e711d235d76044799e/owners", + "url.query": "oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", "user.name": "-", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Other", @@ -231,6 +244,7 @@ "event.duration": 8000000, "event.kind": "event", "event.module": "traefik", + "event.original": "89.64.35.193 - - [19/Jan/2018:10:01:02 +0000] \"GET /marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM HTTP/1.1\" 200 150 - \"Android\" 623114 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.6:4140\" 8ms", "event.outcome": "success", "event.type": [ "access" @@ -263,6 +277,8 @@ "traefik.access.request_count": 623114, "traefik.access.user_identifier": "-", "url.original": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", + "url.path": "/marketplace/tax", + "url.query": "oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", "user.name": "-", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Other", @@ -277,6 +293,7 @@ "event.dataset": "traefik.access", "event.kind": "event", "event.module": "traefik", + "event.original": "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326", "event.outcome": "success", "event.type": [ "access" @@ -298,7 +315,9 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "traefik.access.user_identifier": "-", + "url.extension": "gif", "url.original": "/apache_pb.gif", + "url.path": "/apache_pb.gif", "user.name": "frank" } ] \ No newline at end of file From d0310041e4fdfd5693a3cc4957690eac5255703a Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 26 Mar 2021 12:39:51 +0000 Subject: [PATCH 03/16] Update AWS S3 --- .../module/aws/s3access/ingest/pipeline.yml | 42 +++++++++++++++---- .../aws/s3access/test/s3_server_access.log | 2 +- .../test/s3_server_access.log-expected.json | 10 ++++- .../aws/s3access/test/test.log-expected.json | 2 + 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index 4dea7d027c6f..8cfaa7109c01 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -34,18 +34,19 @@ processors: field: aws.s3access.request_uri ignore_failure: true patterns: - - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' + - '%{NOTSPACE:http.request.method} %{NOTSPACE:_temp_.uri_orig} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' - # - # Best-effort parse of url.original in the form /path?query" - # - grok: - field: url.original + field: aws.s3access.host_header ignore_failure: true patterns: - - '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$' - pattern_definitions: - ABS_PATH: '/[^?]*' + - '(s3[a-z-]*).%{NOTSPACE:cloud.region}.%{WORD}.%{WORD}' + if: ctx.aws?.s3access?.host_header != null + + - uri_parts: + field: _temp_.uri_orig + ignore_failure: true + if: ctx?._temp_?.uri_orig != null - append: if: "ctx?.aws?.s3access?.bucket_owner != null" field: related.user @@ -188,6 +189,31 @@ processors: field: _temp_ ignore_missing: true + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + on_failure: - set: field: "error.message" diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log index f96091a76798..72b45f88ef24 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log @@ -3,4 +3,4 @@ 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json index aa9d1bf6938a..37345ff30fc9 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json @@ -22,6 +22,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -96,6 +97,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -171,6 +173,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.BUCKET", "event.category": "web", "event.dataset": "aws.s3access", @@ -245,6 +248,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -317,6 +321,7 @@ "client.ip": "77.227.156.41", "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", "cloud.provider": "aws", + "cloud.region": "eu-central-1", "event.action": "BATCH.DELETE.OBJECT", "event.category": "web", "event.dataset": "aws.s3access", @@ -360,7 +365,7 @@ "aws.s3access.bucket": "test-s3-ks", "aws.s3access.bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", "aws.s3access.cipher_suite": "ECDHE-RSA-AES128-SHA", - "aws.s3access.host_header": "s3-ap-southeast-1.amazonaws.com", + "aws.s3access.host_header": "s3.ap-southeast-1.amazonaws.com", "aws.s3access.host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=", "aws.s3access.http_status": 204, "aws.s3access.key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png", @@ -375,13 +380,14 @@ "client.ip": "174.29.206.152", "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "BATCH.DELETE.OBJECT", "event.category": "web", "event.dataset": "aws.s3access", "event.id": "6CE38F1312D32BDD", "event.kind": "event", "event.module": "aws", - "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2", + "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "event.outcome": "success", "event.type": [ "access" diff --git a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json index f6ca4d4edf36..94866cfe505c 100644 --- a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json @@ -305,6 +305,7 @@ "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.1", "tls.version_protocol": "tls", + "url.extension": "pdf", "url.original": "/awsexamplebucket/s3-dg.pdf", "url.path": "/awsexamplebucket/s3-dg.pdf", "user_agent.device.name": "Other", @@ -368,6 +369,7 @@ "tls.version": "1.1", "tls.version_protocol": "tls", "url.original": "*", + "url.path": "*", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "S3Console/0.4" From 1ffb8a542ef9351d5ff2c6d202d3b8a593154f89 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 26 Mar 2021 15:09:19 +0000 Subject: [PATCH 04/16] Update Cisco --- .../additional_messages.log-expected.json | 10 +- .../cisco/asa/test/sample.log-expected.json | 14 +- .../cisco/ftd/test/sample.log-expected.json | 14 +- .../security-connection.log-expected.json | 15 +- .../security-file-malware.log-expected.json | 45 +++++ .../security-malware-site.log-expected.json | 8 +- .../module/cisco/meraki/ingest/pipeline.yml | 27 +++ .../meraki/test/generated.log-expected.json | 176 +++++++++++++++--- .../cisco/shared/ingest/asa-ftd-pipeline.yml | 49 +++++ 9 files changed, 320 insertions(+), 38 deletions(-) diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 35c5882513c8..652da70d3512 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -2499,7 +2499,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/", + "url.path": "/", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2631,7 +2634,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/", + "url.path": "/", + "url.scheme": "http" }, { "cisco.asa.connection_id": "2751765169", diff --git a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json index 6a04d9e08e43..7a8c5d428483 100644 --- a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json @@ -3516,7 +3516,8 @@ "cisco-asa", "forwarded" ], - "url.original": "/app" + "url.original": "/app", + "url.path": "/app" }, { "@timestamp": "2009-11-16T14:12:36.000-02:00", @@ -3558,7 +3559,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://example.com" + "url.domain": "example.com", + "url.original": "http://example.com", + "url.path": "", + "url.scheme": "http" }, { "@timestamp": "2009-11-16T14:12:37.000-02:00", @@ -3602,7 +3606,11 @@ "cisco-asa", "forwarded" ], - "url.original": "http://www.example.net/images/favicon.ico" + "url.domain": "www.example.net", + "url.extension": "ico", + "url.original": "http://www.example.net/images/favicon.ico", + "url.path": "/images/favicon.ico", + "url.scheme": "http" }, { "@timestamp": "2021-01-13T19:12:37.000-02:00", diff --git a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json index 865c5a2764e5..a287b33f2525 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json @@ -3502,7 +3502,8 @@ "cisco-ftd", "forwarded" ], - "url.original": "/app" + "url.original": "/app", + "url.path": "/app" }, { "@timestamp": "2009-11-16T14:12:36.000-02:00", @@ -3543,7 +3544,10 @@ "cisco-ftd", "forwarded" ], - "url.original": "http://example.com" + "url.domain": "example.com", + "url.original": "http://example.com", + "url.path": "", + "url.scheme": "http" }, { "@timestamp": "2009-11-16T14:12:37.000-02:00", @@ -3586,6 +3590,10 @@ "cisco-ftd", "forwarded" ], - "url.original": "http://www.example.net/images/favicon.ico" + "url.domain": "www.example.net", + "url.extension": "ico", + "url.original": "http://www.example.net/images/favicon.ico", + "url.path": "/images/favicon.ico", + "url.scheme": "http" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json index be1d11ad0afc..f65ccf20d71b 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json @@ -609,7 +609,10 @@ "forwarded" ], "url.domain": "eu-central-1.ec2.archive.ubuntu.com", + "url.extension": "deb", "url.original": "http://eu-central-1.ec2.archive.ubuntu.com/ubuntu/pool/main/m/manpages/manpages-dev_4.15-1_all.deb", + "url.path": "/ubuntu/pool/main/m/manpages/manpages-dev_4.15-1_all.deb", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "Debian APT-HTTP/1.3 (1.6.11)" @@ -822,7 +825,10 @@ "forwarded" ], "url.domain": "www.eicar.org", + "url.extension": "zip", "url.original": "http://www.eicar.org/download/eicar_com.zip", + "url.path": "/download/eicar_com.zip", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "curl/7.58.0" @@ -1014,8 +1020,15 @@ "cisco-ftd", "forwarded" ], - "url.domain": "10.0.100.30:8000", + "url.domain": [ + "10.0.100.30", + "10.0.100.30:8000" + ], + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "curl/7.58.0" diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json index 135a29792105..367c559c2e67 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json @@ -69,7 +69,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "exe", "url.original": "http://10.0.100.30:8000/exploit.exe", + "url.path": "/exploit.exe", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -143,7 +148,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "exe", "url.original": "http://10.0.100.30:8000/exploit.exe", + "url.path": "/exploit.exe", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -217,7 +227,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "com", "url.original": "http://10.0.100.30:8000/eicar.com", + "url.path": "/eicar.com", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -291,7 +306,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "txt", "url.original": "http://10.0.100.30:8000/eicar.com.txt", + "url.path": "/eicar.com.txt", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -374,7 +394,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -457,7 +482,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -544,7 +574,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -640,7 +675,11 @@ "cisco-ftd", "forwarded" ], + "url.domain": "www.eicar.org", + "url.extension": "zip", "url.original": "http://www.eicar.org/download/eicar_com.zip", + "url.path": "/download/eicar_com.zip", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -726,7 +765,10 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", "url.original": "http://10.0.100.30/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.path": "/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -823,7 +865,10 @@ "cisco-ftd", "forwarded" ], + "url.domain": "18.197.225.123", "url.original": "http://18.197.225.123/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.path": "/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" } diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json index b23b07b6ac23..e98e7fc90cdf 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json @@ -117,8 +117,14 @@ "cisco-ftd", "forwarded" ], - "url.domain": "eyedropper-color-pick.info", + "url.domain": [ + "bad-malwaresite-grr.info", + "eyedropper-color-pick.info" + ], + "url.extension": "ico", "url.original": "http://bad-malwaresite-grr.info/favicon.ico", + "url.path": "/favicon.ico", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36" diff --git a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml index a18507659aa5..0065fe86f8e5 100644 --- a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml @@ -10,6 +10,33 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json index 32182b7f2576..041ff5b2a743 100644 --- a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json @@ -40,7 +40,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.org/odoco/ria.jpg?ritin=uredolor#tatemac" + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "tatemac", + "url.original": "https://example.org/odoco/ria.jpg?ritin=uredolor#tatemac", + "url.path": "/odoco/ria.jpg", + "url.query": "ritin=uredolor", + "url.scheme": "https" }, { "@timestamp": "2016-02-12T13:12:33.000Z", @@ -214,7 +220,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://api.example.org/liqu/lorem.gif?ueipsaqu=uidolore#niamqu" + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "niamqu", + "url.original": "https://api.example.org/liqu/lorem.gif?ueipsaqu=uidolore#niamqu", + "url.path": "/liqu/lorem.gif", + "url.query": "ueipsaqu=uidolore", + "url.scheme": "https" }, { "@timestamp": "2016-04-24T00:25:25.000Z", @@ -377,7 +389,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.net/sun/essecill.html?saute=vel#quu" + "url.domain": "www5.example.net", + "url.extension": "html", + "url.fragment": "quu", + "url.original": "https://www5.example.net/sun/essecill.html?saute=vel#quu", + "url.path": "/sun/essecill.html", + "url.query": "saute=vel", + "url.scheme": "https" }, { "@timestamp": "2016-07-04T11:38:16.000Z", @@ -618,7 +636,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.com/quae/maccusa.htm?rQuisau=idex#xerci" + "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "xerci", + "url.original": "https://internal.example.com/quae/maccusa.htm?rQuisau=idex#xerci", + "url.path": "/quae/maccusa.htm", + "url.query": "rQuisau=idex", + "url.scheme": "https" }, { "@timestamp": "2016-10-12T12:56:16.000Z", @@ -638,8 +662,8 @@ "appliance" ], "related.ip": [ - "10.219.84.37", - "10.205.47.51" + "10.205.47.51", + "10.219.84.37" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -730,7 +754,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.net/ris/uamqu.txt?liqui=quioffi#uptate" + "url.domain": "internal.example.net", + "url.extension": "txt", + "url.fragment": "uptate", + "url.original": "https://internal.example.net/ris/uamqu.txt?liqui=quioffi#uptate", + "url.path": "/ris/uamqu.txt", + "url.query": "liqui=quioffi", + "url.scheme": "https" }, { "@timestamp": "2016-11-24T10:03:59.000Z", @@ -773,7 +803,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.com/icabo/itatio.jpg?eleum=sintoc#volupt" + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "volupt", + "url.original": "https://mail.example.com/icabo/itatio.jpg?eleum=sintoc#volupt", + "url.path": "/icabo/itatio.jpg", + "url.query": "eleum=sintoc", + "url.scheme": "https" }, { "@timestamp": "2016-12-08T17:06:33.000Z", @@ -926,8 +962,8 @@ "appliance" ], "related.ip": [ - "10.93.68.231", - "10.135.217.12" + "10.135.217.12", + "10.93.68.231" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1257,8 +1293,8 @@ "appliance" ], "related.ip": [ - "10.204.230.166", - "10.98.194.212" + "10.98.194.212", + "10.204.230.166" ], "rsa.counters.dclass_r1": "enimadmi", "rsa.internal.messageid": "events", @@ -1277,7 +1313,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.net/ites/isetq.gif?nisiut=tur#avolupt" + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "avolupt", + "url.original": "https://example.net/ites/isetq.gif?nisiut=tur#avolupt", + "url.path": "/ites/isetq.gif", + "url.query": "nisiut=tur", + "url.scheme": "https" }, { "@timestamp": "2017-06-26T19:42:33.000Z", @@ -1309,7 +1351,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.org/ueipsa/tae.html?eriti=atcupi#corpori" + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "corpori", + "url.original": "https://www.example.org/ueipsa/tae.html?eriti=atcupi#corpori", + "url.path": "/ueipsa/tae.html", + "url.query": "eriti=atcupi", + "url.scheme": "https" }, { "@timestamp": "2017-07-11T02:45:07.000Z", @@ -1453,8 +1501,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.94.6.140", - "10.147.15.213" + "10.147.15.213", + "10.94.6.140" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1554,7 +1602,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/irured/illumqui.txt?tionula=ritqu#ecatcupi" + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "ecatcupi", + "url.original": "https://www.example.net/irured/illumqui.txt?tionula=ritqu#ecatcupi", + "url.path": "/irured/illumqui.txt", + "url.query": "tionula=ritqu", + "url.scheme": "https" }, { "@timestamp": "2017-10-19T04:03:07.000Z", @@ -1735,7 +1789,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.net/uat/lupta.html?uptassit=ncidi#tlabori" + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "tlabori", + "url.original": "https://mail.example.net/uat/lupta.html?uptassit=ncidi#tlabori", + "url.path": "/uat/lupta.html", + "url.query": "uptassit=ncidi", + "url.scheme": "https" }, { "@timestamp": "2018-01-12T22:18:32.000Z", @@ -2019,7 +2079,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.com/prehend/eufug.htm?eufug=est#civelits" + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "civelits", + "url.original": "https://mail.example.com/prehend/eufug.htm?eufug=est#civelits", + "url.path": "/prehend/eufug.htm", + "url.query": "eufug=est", + "url.scheme": "https" }, { "@timestamp": "2018-05-07T06:39:06.000Z", @@ -2337,7 +2403,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/doconse/eni.html?mSec=smoditem#tatisetq" + "url.domain": "www.example.net", + "url.extension": "html", + "url.fragment": "tatisetq", + "url.original": "https://www.example.net/doconse/eni.html?mSec=smoditem#tatisetq", + "url.path": "/doconse/eni.html", + "url.query": "mSec=smoditem", + "url.scheme": "https" }, { "@timestamp": "2018-09-12T22:02:15.000Z", @@ -2380,7 +2452,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.com/seq/moll.htm?sunt=dquianon#urExc" + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "urExc", + "url.original": "https://www.example.com/seq/moll.htm?sunt=dquianon#urExc", + "url.path": "/seq/moll.htm", + "url.query": "sunt=dquianon", + "url.scheme": "https" }, { "@timestamp": "2018-09-27T05:04:49.000Z", @@ -2535,7 +2613,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/sectetur/edquian.html?turQuis=taevi#uames" + "url.domain": "www.example.net", + "url.extension": "html", + "url.fragment": "uames", + "url.original": "https://www.example.net/sectetur/edquian.html?turQuis=taevi#uames", + "url.path": "/sectetur/edquian.html", + "url.query": "turQuis=taevi", + "url.scheme": "https" }, { "@timestamp": "2018-11-23T09:15:06.000Z", @@ -2719,7 +2803,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.com/totamrem/aliqu.htm?sBonorum=moenimi#lor" + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "lor", + "url.original": "https://www.example.com/totamrem/aliqu.htm?sBonorum=moenimi#lor", + "url.path": "/totamrem/aliqu.htm", + "url.query": "sBonorum=moenimi", + "url.scheme": "https" }, { "@timestamp": "2019-02-17T03:30:32.000Z", @@ -3137,7 +3227,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.org/eturadi/umS.txt?mSecti=henderi#taevitae" + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "taevitae", + "url.original": "https://www5.example.org/eturadi/umS.txt?mSecti=henderi#taevitae", + "url.path": "/eturadi/umS.txt", + "url.query": "mSecti=henderi", + "url.scheme": "https" }, { "@timestamp": "2019-08-07T16:01:23.000Z", @@ -3180,7 +3276,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.org/fdeFi/ratv.htm?sequatu=tiumtot#tate" + "url.domain": "internal.example.org", + "url.extension": "htm", + "url.fragment": "tate", + "url.original": "https://internal.example.org/fdeFi/ratv.htm?sequatu=tiumtot#tate", + "url.path": "/fdeFi/ratv.htm", + "url.query": "sequatu=tiumtot", + "url.scheme": "https" }, { "@timestamp": "2019-08-21T23:03:57.000Z", @@ -3213,7 +3315,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.net/uov/pariat.htm?litsed=lumd#tiaec" + "url.domain": "www5.example.net", + "url.extension": "htm", + "url.fragment": "tiaec", + "url.original": "https://www5.example.net/uov/pariat.htm?litsed=lumd#tiaec", + "url.path": "/uov/pariat.htm", + "url.query": "litsed=lumd", + "url.scheme": "https" }, { "@timestamp": "2019-09-05T06:06:31.000Z", @@ -3347,7 +3455,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.com/illoinve/etcon.htm?nevolup=erspici#itinvolu" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "itinvolu", + "url.original": "https://www5.example.com/illoinve/etcon.htm?nevolup=erspici#itinvolu", + "url.path": "/illoinve/etcon.htm", + "url.query": "nevolup=erspici", + "url.scheme": "https" }, { "@timestamp": "2019-11-01T10:16:48.000Z", @@ -3446,7 +3560,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.com/qui/itse.gif?orsitame=tasn#exeaco" + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "exeaco", + "url.original": "https://example.com/qui/itse.gif?orsitame=tasn#exeaco", + "url.path": "/qui/itse.gif", + "url.query": "orsitame=tasn", + "url.scheme": "https" }, { "@timestamp": "2019-12-14T07:24:31.000Z", diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index b016a5c3fd06..86dfb21c7289 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1646,6 +1646,31 @@ processors: ) ) + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true + # # Populate ECS event.code # @@ -1882,6 +1907,30 @@ processors: value: "{{source.domain}}" if: ctx.source?.domain != null && ctx.source?.domain != '' allow_duplicates: false + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: # Copy any fields under _temp_.cisco to its final destination. Those can help # with diagnosing the failure. From b339e8fc69cec983f06789c17748281af3201990 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 26 Mar 2021 15:11:39 +0000 Subject: [PATCH 05/16] Update F5 --- .../bigipafm/test/generated.log-expected.json | 354 +++++++++--------- .../module/f5/bigipapm/ingest/pipeline.yml | 27 ++ .../bigipapm/test/generated.log-expected.json | 18 +- 3 files changed, 229 insertions(+), 170 deletions(-) diff --git a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json index 0a259d4811c6..f499a74458c7 100644 --- a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json @@ -24,9 +24,9 @@ "tatemac3541.api.corp" ], "related.ip": [ - "10.208.121.85", - "10.11.196.142", "10.228.193.207", + "10.11.196.142", + "10.208.121.85", "10.165.201.71" ], "related.user": [ @@ -94,8 +94,8 @@ "related.ip": [ "10.51.132.10", "10.162.9.235", - "10.92.202.200", - "10.94.67.230" + "10.94.67.230", + "10.92.202.200" ], "related.user": [ "byC" @@ -160,10 +160,10 @@ "gelit6728.api.invalid" ], "related.ip": [ - "10.82.56.117", "10.209.155.149", - "10.191.68.244", - "10.122.116.161" + "10.82.56.117", + "10.122.116.161", + "10.191.68.244" ], "related.user": [ "seq" @@ -227,9 +227,9 @@ "uid545.www5.localhost" ], "related.ip": [ + "10.12.44.169", "10.50.112.141", "10.131.233.27", - "10.12.44.169", "10.202.66.28" ], "related.user": [ @@ -295,10 +295,10 @@ "emquiavo452.internal.localhost" ], "related.ip": [ - "10.159.182.171", "10.96.35.212", "10.206.197.113", - "10.151.111.38" + "10.151.111.38", + "10.159.182.171" ], "related.user": [ "mol" @@ -363,9 +363,9 @@ "sun1403.www.invalid" ], "related.ip": [ + "10.126.177.162", "10.213.113.28", "10.169.144.147", - "10.126.177.162", "10.89.163.114" ], "related.user": [ @@ -498,10 +498,10 @@ "ume465.corp" ], "related.ip": [ - "10.150.220.75", - "10.110.99.17", "10.189.109.245", - "10.69.57.206" + "10.69.57.206", + "10.110.99.17", + "10.150.220.75" ], "related.user": [ "onse" @@ -632,10 +632,10 @@ "aliqu6801.api.localdomain" ], "related.ip": [ - "10.57.103.192", - "10.64.141.105", "10.182.199.231", - "10.46.27.57" + "10.57.103.192", + "10.46.27.57", + "10.64.141.105" ], "related.user": [ "ice" @@ -699,10 +699,10 @@ "itame189.domain" ], "related.ip": [ - "10.160.210.31", - "10.164.6.207", "10.3.134.237", - "10.32.67.231" + "10.164.6.207", + "10.32.67.231", + "10.160.210.31" ], "related.user": [ "pic" @@ -767,10 +767,10 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ + "10.182.178.217", "10.235.101.253", "10.201.6.10", - "10.42.138.192", - "10.182.178.217" + "10.42.138.192" ], "related.user": [ "giatnu" @@ -835,6 +835,8 @@ "stlabo1228.mail.host" ], "related.ip": [ + "10.22.102.198", + "10.86.101.235", "10.151.161.70", "10.194.247.171", "10.86.101.235", @@ -970,10 +972,10 @@ "smo7167.www.test" ], "related.ip": [ - "10.182.191.174", - "10.214.249.164", "10.99.249.210", - "10.81.26.208" + "10.182.191.174", + "10.81.26.208", + "10.214.249.164" ], "related.user": [ "upta" @@ -1038,9 +1040,9 @@ ], "related.ip": [ "10.101.226.128", - "10.220.5.143", "10.88.101.53", - "10.201.238.90" + "10.201.238.90", + "10.220.5.143" ], "related.user": [ "porro" @@ -1171,10 +1173,10 @@ "quid3147.mail.home" ], "related.ip": [ + "10.167.227.44", "10.181.133.187", "10.148.161.250", - "10.66.181.6", - "10.167.227.44" + "10.66.181.6" ], "related.user": [ "adipisc" @@ -1239,10 +1241,10 @@ "umdolo1029.mail.localhost" ], "related.ip": [ - "10.74.11.43", "10.84.163.178", - "10.107.9.163", - "10.54.17.32" + "10.74.11.43", + "10.54.17.32", + "10.107.9.163" ], "related.user": [ "mquisno" @@ -1306,10 +1308,10 @@ "lorsita2019.internal.home" ], "related.ip": [ - "10.192.229.221", - "10.112.32.213", "10.230.129.252", - "10.184.73.211" + "10.184.73.211", + "10.112.32.213", + "10.192.229.221" ], "related.user": [ "odi" @@ -1374,10 +1376,10 @@ "paquioff624.mail.invalid" ], "related.ip": [ + "10.7.200.140", "10.198.213.189", - "10.161.148.64", "10.199.216.143", - "10.7.200.140" + "10.161.148.64" ], "related.user": [ "ccaeca" @@ -1508,9 +1510,9 @@ "avolupt7576.api.corp" ], "related.ip": [ - "10.71.114.14", "10.194.210.62", "10.68.253.120", + "10.71.114.14", "10.183.130.225" ], "related.user": [ @@ -1644,10 +1646,10 @@ "nsequat1971.internal.invalid" ], "related.ip": [ - "10.55.105.113", + "10.213.94.135", "10.225.212.189", "10.44.58.106", - "10.213.94.135" + "10.55.105.113" ], "related.user": [ "dquia" @@ -1711,9 +1713,9 @@ "ectiono2241.lan" ], "related.ip": [ - "10.163.209.70", "10.255.74.136", "10.69.161.78", + "10.163.209.70", "10.2.114.9" ], "related.user": [ @@ -1778,10 +1780,10 @@ "umetMal1664.mail.lan" ], "related.ip": [ - "10.184.59.148", "10.12.129.137", + "10.252.102.110", "10.46.115.216", - "10.252.102.110" + "10.184.59.148" ], "related.user": [ "perspici" @@ -1846,10 +1848,10 @@ "derit5270.mail.local" ], "related.ip": [ - "10.81.184.7", "10.155.204.243", "10.105.52.140", - "10.199.194.79" + "10.199.194.79", + "10.105.52.140" ], "related.user": [ "eetd" @@ -1914,10 +1916,10 @@ "orisni5238.mail.lan" ], "related.ip": [ - "10.18.226.72", + "10.251.231.142", "10.177.238.45", - "10.110.2.166", - "10.251.231.142" + "10.18.226.72", + "10.110.2.166" ], "related.user": [ "taliqui" @@ -1982,10 +1984,10 @@ "iutali7297.www.domain" ], "related.ip": [ - "10.190.122.27", "10.100.199.226", - "10.99.202.229", - "10.192.98.247" + "10.190.122.27", + "10.192.98.247", + "10.99.202.229" ], "related.user": [ "lloinven" @@ -2051,9 +2053,9 @@ ], "related.ip": [ "10.172.154.97", - "10.248.111.207", "10.37.193.70", - "10.162.97.197" + "10.162.97.197", + "10.248.111.207" ], "related.user": [ "culpaq" @@ -2184,9 +2186,9 @@ "tnonproi195.api.home" ], "related.ip": [ - "10.199.127.211", "10.83.238.145", "10.238.4.219", + "10.199.127.211", "10.1.171.61" ], "related.user": [ @@ -2318,6 +2320,8 @@ "uido492.www5.home" ], "related.ip": [ + "10.183.223.149", + "10.225.141.172", "10.225.255.211", "10.225.141.172", "10.183.223.149", @@ -2387,8 +2391,8 @@ "related.ip": [ "10.176.64.28", "10.97.138.181", - "10.169.123.103", - "10.205.174.181" + "10.205.174.181", + "10.169.123.103" ], "related.user": [ "eseruntm" @@ -2453,9 +2457,9 @@ "dolorem1698.www.domain" ], "related.ip": [ + "10.75.120.11", "10.53.101.131", "10.204.4.40", - "10.75.120.11", "10.169.101.161" ], "related.user": [ @@ -2521,10 +2525,10 @@ "evitae7333.www.lan" ], "related.ip": [ - "10.28.51.219", - "10.6.222.112", "10.87.120.87", - "10.156.117.169" + "10.156.117.169", + "10.28.51.219", + "10.6.222.112" ], "related.user": [ "onsequu" @@ -2588,10 +2592,10 @@ "arc2412.mail.lan" ], "related.ip": [ - "10.57.89.155", - "10.4.126.103", "10.253.167.17", - "10.247.44.59" + "10.247.44.59", + "10.57.89.155", + "10.4.126.103" ], "related.user": [ "ntorever" @@ -2657,8 +2661,8 @@ "related.ip": [ "10.143.183.208", "10.36.69.125", - "10.15.240.220", - "10.248.206.210" + "10.248.206.210", + "10.15.240.220" ], "related.user": [ "met" @@ -2724,9 +2728,9 @@ ], "related.ip": [ "10.69.170.107", - "10.6.32.7", "10.142.186.43", - "10.34.133.2" + "10.34.133.2", + "10.6.32.7" ], "related.user": [ "ipitlabo" @@ -2791,10 +2795,11 @@ "ender5647.www5.example" ], "related.ip": [ + "10.121.153.197", + "10.170.165.164", "10.142.22.24", "10.59.103.10", - "10.170.165.164", - "10.121.153.197" + "10.170.165.164" ], "related.user": [ "borumSec" @@ -2927,10 +2932,10 @@ "uatu2894.api.lan" ], "related.ip": [ + "10.64.139.17", "10.8.29.219", "10.70.7.23", - "10.40.177.138", - "10.64.139.17" + "10.40.177.138" ], "related.user": [ "rep" @@ -2994,6 +2999,9 @@ "rmagnido5483.local" ], "related.ip": [ + "10.180.62.222", + "10.67.173.228", + "10.67.221.220", "10.67.173.228", "10.2.189.20", "10.67.221.220", @@ -3062,10 +3070,11 @@ "uian521.www.example" ], "related.ip": [ + "10.56.134.118", "10.147.127.181", - "10.209.52.47", "10.56.134.118", - "10.196.176.243" + "10.196.176.243", + "10.209.52.47" ], "related.user": [ "tasu" @@ -3130,9 +3139,9 @@ ], "related.ip": [ "10.85.13.237", + "10.226.24.84", "10.231.18.90", - "10.248.140.59", - "10.226.24.84" + "10.248.140.59" ], "related.user": [ "Nem" @@ -3197,10 +3206,10 @@ "ntsunt4894.mail.domain" ], "related.ip": [ - "10.207.183.204", - "10.8.224.72", "10.59.215.207", - "10.203.46.215" + "10.207.183.204", + "10.203.46.215", + "10.8.224.72" ], "related.user": [ "eruntmo" @@ -3265,10 +3274,10 @@ "mexer3864.api.corp" ], "related.ip": [ - "10.255.145.22", "10.73.84.95", "10.230.38.148", - "10.98.154.146" + "10.98.154.146", + "10.255.145.22" ], "related.user": [ "sitam" @@ -3400,10 +3409,10 @@ "fugiatnu2498.www.localhost" ], "related.ip": [ + "10.220.202.102", "10.182.213.195", - "10.122.133.162", "10.195.139.25", - "10.220.202.102" + "10.122.133.162" ], "related.user": [ "aquae" @@ -3536,7 +3545,9 @@ ], "related.ip": [ "10.113.65.192", + "10.21.58.162", "10.241.143.145", + "10.21.58.162", "10.35.190.164", "10.21.58.162" ], @@ -3603,10 +3614,10 @@ "itanimi1934.home" ], "related.ip": [ + "10.75.113.240", "10.53.27.253", "10.129.16.166", - "10.19.154.103", - "10.75.113.240" + "10.19.154.103" ], "related.user": [ "luptat" @@ -3673,8 +3684,8 @@ "related.ip": [ "10.150.153.61", "10.125.150.220", - "10.22.213.196", - "10.120.50.13" + "10.120.50.13", + "10.22.213.196" ], "related.user": [ "inculpa" @@ -3739,10 +3750,10 @@ "edquiaco6562.api.lan" ], "related.ip": [ + "10.238.171.184", "10.85.52.249", "10.229.155.171", - "10.113.2.13", - "10.238.171.184" + "10.113.2.13" ], "related.user": [ "tatiset" @@ -3809,6 +3820,7 @@ "related.ip": [ "10.249.174.35", "10.198.150.185", + "10.249.174.35", "10.51.245.225", "10.220.1.249" ], @@ -3875,9 +3887,9 @@ "eosqui3723.api.localdomain" ], "related.ip": [ + "10.152.157.32", "10.190.96.181", "10.38.185.31", - "10.152.157.32", "10.251.82.195" ], "related.user": [ @@ -3944,8 +3956,8 @@ "related.ip": [ "10.230.112.179", "10.103.102.242", - "10.211.198.50", - "10.190.247.194" + "10.190.247.194", + "10.211.198.50" ], "related.user": [ "tDuisaut" @@ -4010,9 +4022,9 @@ ], "related.ip": [ "10.47.223.155", - "10.251.101.61", "10.219.83.199", - "10.101.13.122" + "10.101.13.122", + "10.251.101.61" ], "related.user": [ "ectetur" @@ -4077,10 +4089,10 @@ "saute7421.www.invalid" ], "related.ip": [ - "10.21.30.43", - "10.83.136.233", + "10.21.80.157", "10.31.86.83", - "10.21.80.157" + "10.21.30.43", + "10.83.136.233" ], "related.user": [ "litsed" @@ -4145,10 +4157,11 @@ "oluptas1637.home" ], "related.ip": [ + "10.45.152.205", + "10.27.181.27", "10.195.90.73", - "10.194.197.107", "10.45.152.205", - "10.27.181.27" + "10.194.197.107" ], "related.user": [ "datatn" @@ -4213,10 +4226,10 @@ "ididu5505.api.localdomain" ], "related.ip": [ - "10.43.239.97", - "10.129.161.18", "10.222.2.132", - "10.183.90.25" + "10.129.161.18", + "10.183.90.25", + "10.43.239.97" ], "related.user": [ "aedicta" @@ -4280,10 +4293,10 @@ "mqui1099.api.corp" ], "related.ip": [ - "10.231.167.171", - "10.248.156.138", + "10.189.162.131", "10.67.129.100", - "10.189.162.131" + "10.248.156.138", + "10.231.167.171" ], "related.user": [ "sedquia" @@ -4350,8 +4363,9 @@ "related.ip": [ "10.63.103.30", "10.185.107.27", - "10.142.106.66", - "10.6.146.184" + "10.63.103.30", + "10.6.146.184", + "10.142.106.66" ], "related.user": [ "sequu" @@ -4415,10 +4429,10 @@ "tatiset4191.localdomain" ], "related.ip": [ - "10.119.179.182", - "10.0.202.9", "10.93.39.237", - "10.214.93.200" + "10.0.202.9", + "10.214.93.200", + "10.119.179.182" ], "related.user": [ "tionofd" @@ -4483,10 +4497,10 @@ "aute2433.mail.lan" ], "related.ip": [ - "10.28.145.163", - "10.252.204.162", "10.30.189.166", - "10.123.154.140" + "10.252.204.162", + "10.123.154.140", + "10.28.145.163" ], "related.user": [ "imadmin" @@ -4550,9 +4564,9 @@ "idolo6535.internal.example" ], "related.ip": [ - "10.46.162.198", "10.79.49.3", "10.29.122.183", + "10.46.162.198", "10.145.128.250" ], "related.user": [ @@ -4618,10 +4632,10 @@ "one7728.api.localdomain" ], "related.ip": [ + "10.142.235.217", "10.166.169.167", "10.65.174.196", - "10.177.232.136", - "10.142.235.217" + "10.177.232.136" ], "related.user": [ "olors" @@ -4754,9 +4768,9 @@ "emq2514.api.localhost" ], "related.ip": [ - "10.74.74.129", "10.76.148.147", "10.135.77.156", + "10.74.74.129", "10.46.222.149" ], "related.user": [ @@ -4824,7 +4838,7 @@ "10.130.203.37", "10.145.49.29", "10.96.200.223", - "10.11.146.253" + "10.130.203.37" ], "related.user": [ "mvele" @@ -4889,9 +4903,8 @@ ], "related.ip": [ "10.162.78.48", - "10.48.75.140", - "10.24.23.209", - "10.162.2.180" + "10.162.2.180", + "10.48.75.140" ], "related.user": [ "rumwr" @@ -4955,8 +4968,9 @@ "sequatD163.internal.example" ], "related.ip": [ + "10.97.105.115", + "10.119.12.186", "10.151.206.38", - "10.66.92.83", "10.119.12.186", "10.97.105.115" ], @@ -5022,9 +5036,9 @@ "itamet1303.invalid" ], "related.ip": [ + "10.64.76.142", "10.201.132.114", "10.12.148.73", - "10.64.76.142", "10.169.139.250" ], "related.user": [ @@ -5090,10 +5104,10 @@ "epr3512.internal.domain" ], "related.ip": [ - "10.35.38.185", "10.111.128.11", + "10.9.236.18", "10.200.116.191", - "10.9.236.18" + "10.35.38.185" ], "related.user": [ "umfug" @@ -5157,10 +5171,10 @@ "uredol2174.home" ], "related.ip": [ + "10.236.67.227", "10.191.27.182", - "10.240.62.238", "10.134.238.8", - "10.236.67.227" + "10.240.62.238" ], "related.user": [ "tlabo" @@ -5224,10 +5238,10 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.22.231.91", + "10.65.35.64", "10.109.14.142", - "10.165.66.92", - "10.65.35.64" + "10.22.231.91", + "10.165.66.92" ], "related.user": [ "perna" @@ -5291,10 +5305,10 @@ "inimav5557.www5.test" ], "related.ip": [ + "10.29.230.203", "10.71.112.86", "10.64.161.215", - "10.89.221.90", - "10.29.230.203" + "10.89.221.90" ], "related.user": [ "rnatur" @@ -5358,6 +5372,10 @@ "nonn1650.www.test" ], "related.ip": [ + "10.79.208.135", + "10.221.199.137", + "10.140.118.182", + "10.79.208.135", "10.88.226.76", "10.221.199.137", "10.140.118.182", @@ -5426,10 +5444,10 @@ "acons3940.api.lan" ], "related.ip": [ - "10.35.73.208", "10.126.61.230", "10.189.244.22", - "10.133.48.55" + "10.133.48.55", + "10.35.73.208" ], "related.user": [ "tia" @@ -5496,7 +5514,8 @@ "10.35.65.72", "10.240.94.109", "10.239.194.105", - "10.81.154.115" + "10.81.154.115", + "10.240.94.109" ], "related.user": [ "reseo" @@ -5561,10 +5580,10 @@ "mnisiut6146.internal.local" ], "related.ip": [ - "10.248.72.104", - "10.38.253.213", "10.150.56.227", - "10.52.70.192" + "10.52.70.192", + "10.38.253.213", + "10.248.72.104" ], "related.user": [ "ionem" @@ -5629,10 +5648,10 @@ "borios1067.www5.home" ], "related.ip": [ - "10.62.218.239", - "10.73.172.186", "10.203.193.134", - "10.218.15.164" + "10.218.15.164", + "10.62.218.239", + "10.73.172.186" ], "related.user": [ "reh" @@ -5764,8 +5783,8 @@ "tdolorem813.internal.host" ], "related.ip": [ - "10.233.181.250", "10.50.177.151", + "10.233.181.250", "10.187.237.220", "10.248.0.74" ], @@ -5832,10 +5851,10 @@ "volupt4626.internal.test" ], "related.ip": [ - "10.189.43.11", - "10.80.129.81", "10.96.223.46", - "10.248.248.120" + "10.80.129.81", + "10.248.248.120", + "10.189.43.11" ], "related.user": [ "iatn" @@ -5900,10 +5919,11 @@ "ntium5103.www5.localhost" ], "related.ip": [ - "10.91.115.139", - "10.102.109.199", + "10.66.106.186", "10.173.114.63", - "10.66.106.186" + "10.102.109.199", + "10.66.106.186", + "10.102.109.199" ], "related.user": [ "tNequ" @@ -5968,10 +5988,10 @@ "orpori3334.www.local" ], "related.ip": [ - "10.0.175.17", - "10.221.223.127", "10.198.157.122", - "10.159.155.88" + "10.159.155.88", + "10.221.223.127", + "10.0.175.17" ], "related.user": [ "iquipex" @@ -6035,10 +6055,10 @@ "equu7361.www5.localdomain" ], "related.ip": [ - "10.189.70.237", + "10.30.20.187", "10.7.212.201", "10.252.136.130", - "10.30.20.187" + "10.189.70.237" ], "related.user": [ "ugiat" @@ -6171,10 +6191,10 @@ "uisnostr2390.mail.domain" ], "related.ip": [ - "10.219.174.45", - "10.17.20.93", "10.251.167.219", - "10.181.134.69" + "10.17.20.93", + "10.181.134.69", + "10.219.174.45" ], "related.user": [ "Uteni" @@ -6239,10 +6259,10 @@ "luptate4811.mail.example" ], "related.ip": [ - "10.28.233.253", "10.37.14.20", + "10.223.99.90", "10.30.117.82", - "10.223.99.90" + "10.28.233.253" ], "related.user": [ "numqua" @@ -6307,10 +6327,10 @@ "lites1614.www.corp" ], "related.ip": [ - "10.50.61.114", "10.125.20.22", - "10.8.32.17", - "10.57.85.113" + "10.57.85.113", + "10.50.61.114", + "10.8.32.17" ], "related.user": [ "qua" @@ -6375,10 +6395,10 @@ "lorinrep7686.mail.corp" ], "related.ip": [ - "10.215.224.27", "10.113.78.101", + "10.200.28.55", "10.181.63.82", - "10.200.28.55" + "10.215.224.27" ], "related.user": [ "ficiade" @@ -6511,10 +6531,10 @@ "ntu1279.mail.lan" ], "related.ip": [ - "10.92.168.198", - "10.18.176.44", "10.90.93.4", - "10.39.100.88" + "10.92.168.198", + "10.39.100.88", + "10.18.176.44" ], "related.user": [ "adminima" @@ -6646,8 +6666,8 @@ "cipitl2184.localdomain" ], "related.ip": [ - "10.240.47.113", "10.84.64.28", + "10.240.47.113", "10.31.147.51", "10.209.226.7" ], @@ -6757,4 +6777,4 @@ ], "user.name": "odtemp" } -] \ No newline at end of file +] diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 895c0723f10d..c61cbf6261c3 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -10,6 +10,33 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json index 879cbc128a61..181f7b8e9f12 100644 --- a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json @@ -771,7 +771,13 @@ "f5.bigipapm", "forwarded" ], - "url.original": "https://www5.example.com/olli/rever.html?rsp=oluptat#metco" + "url.domain": "www5.example.com", + "url.extension": "html", + "url.fragment": "metco", + "url.original": "https://www5.example.com/olli/rever.html?rsp=oluptat#metco", + "url.path": "/olli/rever.html", + "url.query": "rsp=oluptat", + "url.scheme": "https" }, { "@timestamp": "2017-03-18T20:24:33.000Z", @@ -2131,7 +2137,13 @@ "f5.bigipapm", "forwarded" ], - "url.original": "https://example.net/equamn/scipi.txt?eiu=maliquam#gnama" + "url.domain": "example.net", + "url.extension": "txt", + "url.fragment": "gnama", + "url.original": "https://example.net/equamn/scipi.txt?eiu=maliquam#gnama", + "url.path": "/equamn/scipi.txt", + "url.query": "eiu=maliquam", + "url.scheme": "https" }, { "@timestamp": "2019-04-01T02:38:14.000Z", @@ -2640,4 +2652,4 @@ ], "user.name": "oditem" } -] \ No newline at end of file +] From 3b213b6e351aaf2f1470b103ce33e8ec68af80e3 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 26 Mar 2021 15:23:20 +0000 Subject: [PATCH 06/16] Update Fortinet --- .../test/generated.log-expected.json | 16 +- .../fortinet/fortimanager/ingest/pipeline.yml | 26 + .../test/generated.log-expected.json | 473 ++++++++++++++---- 3 files changed, 398 insertions(+), 117 deletions(-) diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json index 160c14c7981b..695e1a33648d 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json @@ -25,8 +25,8 @@ "boNemoe4402.www.invalid" ], "related.ip": [ - "10.150.92.220", - "10.102.123.34" + "10.102.123.34", + "10.150.92.220" ], "related.user": [ "sumdo" @@ -1385,8 +1385,8 @@ "adol485.example" ], "related.ip": [ - "10.6.167.7", - "10.153.111.103" + "10.153.111.103", + "10.6.167.7" ], "related.user": [ "eumfug" @@ -3112,8 +3112,8 @@ "giatquov1918.internal.example" ], "related.ip": [ - "10.180.195.43", - "10.85.185.13" + "10.85.185.13", + "10.180.195.43" ], "related.user": [ "voluptas" @@ -5834,8 +5834,8 @@ "rumSecti111.www5.domain" ], "related.ip": [ - "10.87.90.49", - "10.222.245.80" + "10.222.245.80", + "10.87.90.49" ], "related.user": [ "ptatemse" diff --git a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml index eda458f01ded..87270afa2112 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml @@ -10,6 +10,32 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json index bd66027098dc..252b96fc403a 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json @@ -155,8 +155,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "roinBCS", "url.original": "https://www.example.net/orisn/cca.htm?ofdeF=metcons#roinBCS", - "url.query": "taspe", + "url.path": "/orisn/cca.htm", + "url.query": "ofdeF=metcons", + "url.scheme": "https", "user.name": "oluptas" }, { @@ -188,9 +193,9 @@ "ccaecat" ], "related.ip": [ + "10.94.103.117", "10.15.159.80", - "10.200.188.142", - "10.94.103.117" + "10.200.188.142" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -383,8 +388,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "saute", "url.original": "https://api.example.com/iumto/aboreetd.gif?dun=enim#saute", - "url.query": "nsect", + "url.path": "/iumto/aboreetd.gif", + "url.query": "dun=enim", + "url.scheme": "https", "user.name": "rveli" }, { @@ -476,8 +486,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "picia", "url.original": "https://www.example.net/tasuntex/sunt.txt?ume=incidi#picia", - "url.query": "nsec", + "url.path": "/tasuntex/sunt.txt", + "url.query": "ume=incidi", + "url.scheme": "https", "user.name": "aperia" }, { @@ -569,8 +584,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "tlabore", "url.original": "https://www.example.org/inesci/serror.html?mqu=apariat#tlabore", - "url.query": "nsectet", + "url.path": "/inesci/serror.html", + "url.query": "mqu=apariat", + "url.scheme": "https", "user.name": "nnum" }, { @@ -673,8 +693,8 @@ "eturadi6608.mail.host" ], "related.ip": [ - "10.23.62.94", - "10.61.163.4" + "10.61.163.4", + "10.23.62.94" ], "related.user": [ "non" @@ -729,8 +749,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "issus", "url.original": "https://api.example.com/nimid/itatione.htm?umwr=oluptate#issus", - "url.query": "atem", + "url.path": "/nimid/itatione.htm", + "url.query": "umwr=oluptate", + "url.scheme": "https", "user.name": "non" }, { @@ -761,8 +786,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2580", "related.hosts": [ - "ecte", "tani", + "ecte", "ipsumdol4488.api.localdomain" ], "related.ip": [ @@ -822,8 +847,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "htm", + "url.fragment": "pora", "url.original": "https://example.com/tetu/stru.htm?tlabore=Exc#pora", - "url.query": "atno", + "url.path": "/tetu/stru.htm", + "url.query": "tlabore=Exc", + "url.scheme": "https", "user.name": "cons" }, { @@ -856,8 +886,8 @@ ], "related.ip": [ "10.106.162.153", - "10.58.214.16", - "10.238.164.74" + "10.238.164.74", + "10.58.214.16" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -922,9 +952,9 @@ "tenimad" ], "related.ip": [ - "10.225.141.20", + "10.217.150.196", "10.110.31.190", - "10.217.150.196" + "10.225.141.20" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1049,8 +1079,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "isnisi", "url.original": "https://api.example.com/isnostr/umqu.htm?emquia=inesci#isnisi", - "url.query": "uptate", + "url.path": "/isnostr/umqu.htm", + "url.query": "emquia=inesci", + "url.scheme": "https", "user.name": "proide" }, { @@ -1209,8 +1244,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "urerepr", "url.original": "https://example.org/tiaec/rumwrit.txt?oconsequ=edquiac#urerepr", - "url.query": "iaeco", + "url.path": "/tiaec/rumwrit.txt", + "url.query": "oconsequ=edquiac", + "url.scheme": "https", "user.name": "tetur" }, { @@ -1242,8 +1282,8 @@ "xea" ], "related.ip": [ - "10.233.120.207", "10.51.213.42", + "10.233.120.207", "10.98.194.212" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -1380,8 +1420,8 @@ "tore7088.www.invalid" ], "related.ip": [ - "10.212.214.4", - "10.199.47.220" + "10.199.47.220", + "10.212.214.4" ], "related.user": [ "atv" @@ -1436,8 +1476,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "itlab", "url.original": "https://mail.example.org/ptassita/its.gif?risnis=uov#itlab", - "url.query": "hil", + "url.path": "/ptassita/its.gif", + "url.query": "risnis=uov", + "url.scheme": "https", "user.name": "atv" }, { @@ -1529,8 +1574,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "toditaut", "url.original": "https://www.example.org/redol/gnaa.htm?aliquamq=dtempori#toditaut", - "url.query": "quames", + "url.path": "/redol/gnaa.htm", + "url.query": "aliquamq=dtempori", + "url.scheme": "https", "user.name": "orese" }, { @@ -1622,8 +1672,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "offici", "url.original": "https://example.net/aparia/tatnon.jpg?rever=ore#offici", - "url.query": "essecill", + "url.path": "/aparia/tatnon.jpg", + "url.query": "rever=ore", + "url.scheme": "https", "user.name": "enim" }, { @@ -1715,8 +1770,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "equamnih", "url.original": "https://www.example.com/tali/BCS.txt?iqu=niamqu#equamnih", - "url.query": "quiineav", + "url.path": "/tali/BCS.txt", + "url.query": "iqu=niamqu", + "url.scheme": "https", "user.name": "inculp" }, { @@ -1808,8 +1868,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "mexer", "url.original": "https://www.example.com/iame/orroquis.htm?tiumd=ntmoll#mexer", - "url.query": "tec", + "url.path": "/iame/orroquis.htm", + "url.query": "tiumd=ntmoll", + "url.scheme": "https", "user.name": "olorem" }, { @@ -1901,8 +1966,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "mnisi", "url.original": "https://www.example.net/iame/laudanti.htm?stquido=rsitvolu#mnisi", - "url.query": "serror", + "url.path": "/iame/laudanti.htm", + "url.query": "stquido=rsitvolu", + "url.scheme": "https", "user.name": "lmo" }, { @@ -1935,9 +2005,9 @@ "oluptat" ], "related.ip": [ + "10.168.20.20", "10.117.63.181", - "10.247.53.179", - "10.168.20.20" + "10.247.53.179" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2063,8 +2133,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "rQuisaut", "url.original": "https://www.example.org/ess/quiad.jpg?ten=litanim#rQuisaut", - "url.query": "emp", + "url.path": "/ess/quiad.jpg", + "url.query": "ten=litanim", + "url.scheme": "https", "user.name": "amqu" }, { @@ -2157,8 +2232,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "txt", + "url.fragment": "Finibus", "url.original": "https://example.com/xeac/llitanim.txt?oreverit=scip#Finibus", - "url.query": "atura", + "url.path": "/xeac/llitanim.txt", + "url.query": "oreverit=scip", + "url.scheme": "https", "user.name": "iconsequ" }, { @@ -2250,8 +2330,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "iameaque", "url.original": "https://api.example.org/orio/gna.gif?aaliquaU=olu#iameaque", - "url.query": "tatio", + "url.path": "/orio/gna.gif", + "url.query": "aaliquaU=olu", + "url.scheme": "https", "user.name": "iosamni" }, { @@ -2287,8 +2372,8 @@ "uido2046.mail.lan" ], "related.ip": [ - "10.70.7.23", - "10.130.240.11" + "10.130.240.11", + "10.70.7.23" ], "related.user": [ "eavolup" @@ -2343,8 +2428,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "umtota", "url.original": "https://mail.example.org/nimadmin/lumqui.txt?iquip=tinculpa#umtota", - "url.query": "iuta", + "url.path": "/nimadmin/lumqui.txt", + "url.query": "iquip=tinculpa", + "url.scheme": "https", "user.name": "eavolup" }, { @@ -2570,8 +2660,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "quisn", "url.original": "https://internal.example.com/temse/caecat.jpg?emeu=tatemac#quisn", - "url.query": "mveniam", + "url.path": "/temse/caecat.jpg", + "url.query": "emeu=tatemac", + "url.scheme": "https", "user.name": "usanti" }, { @@ -2603,9 +2698,9 @@ "iam" ], "related.ip": [ - "10.38.168.190", "10.77.105.81", - "10.174.17.46" + "10.174.17.46", + "10.38.168.190" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2931,8 +3026,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "isni", "url.original": "https://www5.example.com/elit/sam.htm?nevolu=unt#isni", - "url.query": "onoru", + "url.path": "/elit/sam.htm", + "url.query": "nevolu=unt", + "url.scheme": "https", "user.name": "estiaec" }, { @@ -2968,8 +3068,8 @@ "itaspe3216.localdomain" ], "related.ip": [ - "10.94.177.125", - "10.116.82.108" + "10.116.82.108", + "10.94.177.125" ], "related.user": [ "ecatc" @@ -3024,8 +3124,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "iadeseru", "url.original": "https://example.org/porro/issu.htm?inculpa=ruredol#iadeseru", - "url.query": "olorsit", + "url.path": "/porro/issu.htm", + "url.query": "inculpa=ruredol", + "url.scheme": "https", "user.name": "ecatc" }, { @@ -3118,8 +3223,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "suntexpl", "url.original": "https://mail.example.org/uamquaer/texplica.gif?sequa=lorum#suntexpl", - "url.query": "udexerci", + "url.path": "/uamquaer/texplica.gif", + "url.query": "sequa=lorum", + "url.scheme": "https", "user.name": "norumetM" }, { @@ -3211,8 +3321,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "rsit", "url.original": "https://www5.example.org/aqu/utemvele.gif?serrorsi=tsedquia#rsit", - "url.query": "nihilmol", + "url.path": "/aqu/utemvele.gif", + "url.query": "serrorsi=tsedquia", + "url.scheme": "https", "user.name": "dictasun" }, { @@ -3244,9 +3359,9 @@ "lors" ], "related.ip": [ + "10.205.83.138", "10.99.55.115", - "10.98.52.184", - "10.205.83.138" + "10.98.52.184" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3485,8 +3600,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "adeserun", "url.original": "https://www5.example.com/etcon/chit.txt?erspici=itinvolu#adeserun", - "url.query": "tinvolu", + "url.path": "/etcon/chit.txt", + "url.query": "erspici=itinvolu", + "url.scheme": "https", "user.name": "tcons" }, { @@ -3578,8 +3698,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "nseq", "url.original": "https://api.example.org/fug/liquid.txt?ptate=lloi#nseq", - "url.query": "undeo", + "url.path": "/fug/liquid.txt", + "url.query": "ptate=lloi", + "url.scheme": "https", "user.name": "eumiure" }, { @@ -3745,9 +3870,9 @@ "usantiu" ], "related.ip": [ - "10.5.67.140", "10.140.59.161", - "10.118.111.183" + "10.118.111.183", + "10.5.67.140" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3872,8 +3997,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.net", + "url.extension": "html", + "url.fragment": "litess", "url.original": "https://example.net/aturQui/tquii.html?uiac=squ#litess", - "url.query": "Loremip", + "url.path": "/aturQui/tquii.html", + "url.query": "uiac=squ", + "url.scheme": "https", "user.name": "itsed" }, { @@ -3965,8 +4095,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "aturvel", "url.original": "https://mail.example.org/mvele/teveli.htm?Nequepor=luptate#aturvel", - "url.query": "lab", + "url.path": "/mvele/teveli.htm", + "url.query": "Nequepor=luptate", + "url.scheme": "https", "user.name": "rporis" }, { @@ -3998,8 +4133,8 @@ "sitv" ], "related.ip": [ - "10.251.183.113", "10.217.145.137", + "10.251.183.113", "10.22.149.132" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4065,9 +4200,9 @@ "nisi" ], "related.ip": [ - "10.183.16.252", "10.51.60.203", - "10.203.66.175" + "10.203.66.175", + "10.183.16.252" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4147,8 +4282,8 @@ "rsa.investigations.event_vcat": "nisiuta", "rsa.misc.OS": "olu", "rsa.misc.action": [ - "tquov", - "accept" + "accept", + "tquov" ], "rsa.misc.category": "quatD", "rsa.misc.client": "acomm", @@ -4192,8 +4327,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "etc", "url.original": "https://example.org/tem/iadeseru.jpg?olorsita=odoco#etc", - "url.query": "natu", + "url.path": "/tem/iadeseru.jpg", + "url.query": "olorsita=odoco", + "url.scheme": "https", "user.name": "nimadmi" }, { @@ -4285,8 +4425,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "tobe", "url.original": "https://mail.example.net/epteurs/usmodtem.gif?itvo=asi#tobe", - "url.query": "equepor", + "url.path": "/epteurs/usmodtem.gif", + "url.query": "itvo=asi", + "url.scheme": "https", "user.name": "nimadmin" }, { @@ -4378,8 +4523,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "eav", "url.original": "https://www.example.com/iadolo/cidu.txt?aliquide=redolori#eav", - "url.query": "tiu", + "url.path": "/iadolo/cidu.txt", + "url.query": "aliquide=redolori", + "url.scheme": "https", "user.name": "msequin" }, { @@ -4426,8 +4576,8 @@ "rsa.investigations.event_vcat": "uia", "rsa.misc.OS": "mquae", "rsa.misc.action": [ - "deny", - "tenatus" + "tenatus", + "deny" ], "rsa.misc.category": "abo", "rsa.misc.client": "umtota", @@ -4471,8 +4621,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "boreetd", "url.original": "https://internal.example.com/isc/umdol.jpg?atn=sectet#boreetd", - "url.query": "odic", + "url.path": "/isc/umdol.jpg", + "url.query": "atn=sectet", + "url.scheme": "https", "user.name": "tat" }, { @@ -4505,9 +4660,9 @@ "nonnumq" ], "related.ip": [ - "10.249.16.201", + "10.34.41.75", "10.107.168.208", - "10.34.41.75" + "10.249.16.201" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4632,8 +4787,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "ptat", "url.original": "https://www.example.com/imadm/ugiat.txt?Nequepor=nisiu#ptat", - "url.query": "stiaec", + "url.path": "/imadm/ugiat.txt", + "url.query": "Nequepor=nisiu", + "url.scheme": "https", "user.name": "ommodoc" }, { @@ -4725,8 +4885,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "sed", "url.original": "https://mail.example.net/ionemu/nul.jpg?volupt=ori#sed", - "url.query": "radipi", + "url.path": "/ionemu/nul.jpg", + "url.query": "volupt=ori", + "url.scheme": "https", "user.name": "aed" }, { @@ -4825,9 +4990,9 @@ "litsedq" ], "related.ip": [ - "10.251.212.166", + "10.77.105.160", "10.90.229.92", - "10.77.105.160" + "10.251.212.166" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4952,8 +5117,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "henderi", "url.original": "https://mail.example.org/asuntex/uovolup.html?amali=uiav#henderi", - "url.query": "eriti", + "url.path": "/asuntex/uovolup.html", + "url.query": "amali=uiav", + "url.scheme": "https", "user.name": "urmag" }, { @@ -5045,8 +5215,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "dminimve", "url.original": "https://example.org/mnisiut/porinci.htm?norum=emUten#dminimve", - "url.query": "texplica", + "url.path": "/mnisiut/porinci.htm", + "url.query": "norum=emUten", + "url.scheme": "https", "user.name": "lpaquiof" }, { @@ -5145,9 +5320,9 @@ "llumdo" ], "related.ip": [ - "10.242.119.111", "10.188.131.18", - "10.239.231.168" + "10.239.231.168", + "10.242.119.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5272,8 +5447,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "gif", + "url.fragment": "rai", "url.original": "https://api.example.net/nsec/smo.gif?etq=trumexe#rai", - "url.query": "ipsu", + "url.path": "/nsec/smo.gif", + "url.query": "etq=trumexe", + "url.scheme": "https", "user.name": "ainci" }, { @@ -5412,8 +5592,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "upta", "url.original": "https://internal.example.com/dolo/exeacom.txt?tlab=eufugiat#upta", - "url.query": "Lor", + "url.path": "/dolo/exeacom.txt", + "url.query": "tlab=eufugiat", + "url.scheme": "https", "user.name": "erspici" }, { @@ -5505,8 +5690,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "utod", "url.original": "https://www5.example.org/orumet/aliqu.txt?tion=sun#utod", - "url.query": "con", + "url.path": "/orumet/aliqu.txt", + "url.query": "tion=sun", + "url.scheme": "https", "user.name": "Sedutp" }, { @@ -5732,8 +5922,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "gif", + "url.fragment": "iscin", "url.original": "https://internal.example.com/nonnumqu/isciveli.gif?wri=aute#iscin", - "url.query": "eca", + "url.path": "/nonnumqu/isciveli.gif", + "url.query": "wri=aute", + "url.scheme": "https", "user.name": "ciduntut" }, { @@ -5825,8 +6020,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "html", + "url.fragment": "upta", "url.original": "https://www5.example.org/utlabo/scip.html?voluptas=inv#upta", - "url.query": "atnu", + "url.path": "/utlabo/scip.html", + "url.query": "voluptas=inv", + "url.scheme": "https", "user.name": "nulapar" }, { @@ -5918,8 +6118,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "obeata", "url.original": "https://internal.example.com/ssusci/snostrud.txt?dolo=siutaliq#obeata", - "url.query": "ptat", + "url.path": "/ssusci/snostrud.txt", + "url.query": "dolo=siutaliq", + "url.scheme": "https", "user.name": "moenimi" }, { @@ -6011,8 +6216,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ptasn", "url.original": "https://api.example.com/temporai/umw.jpg?mveniamq=litsed#ptasn", - "url.query": "lore", + "url.path": "/temporai/umw.jpg", + "url.query": "mveniamq=litsed", + "url.scheme": "https", "user.name": "idolo" }, { @@ -6044,9 +6254,9 @@ "eleumiu" ], "related.ip": [ - "10.221.100.157", "10.120.212.78", - "10.236.211.111" + "10.236.211.111", + "10.221.100.157" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6171,8 +6381,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "umiurere", "url.original": "https://mail.example.org/ngelit/dipiscin.gif?serro=ctet#umiurere", - "url.query": "diconseq", + "url.path": "/ngelit/dipiscin.gif", + "url.query": "serro=ctet", + "url.scheme": "https", "user.name": "riatur" }, { @@ -6398,8 +6613,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "lestia", "url.original": "https://internal.example.net/inc/riaturEx.htm?mnihilm=itinvo#lestia", - "url.query": "icaboN", + "url.path": "/inc/riaturEx.htm", + "url.query": "mnihilm=itinvo", + "url.scheme": "https", "user.name": "udantiu" }, { @@ -6498,9 +6718,9 @@ "datatno" ], "related.ip": [ - "10.238.49.73", "10.74.88.209", - "10.92.3.166" + "10.92.3.166", + "10.238.49.73" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6633,8 +6853,8 @@ ], "related.ip": [ "10.135.213.17", - "10.167.128.229", - "10.30.239.222" + "10.30.239.222", + "10.167.128.229" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6759,8 +6979,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "rExce", "url.original": "https://api.example.net/ita/esse.txt?amquis=iatquovo#rExce", - "url.query": "ididun", + "url.path": "/ita/esse.txt", + "url.query": "amquis=iatquovo", + "url.scheme": "https", "user.name": "emq" }, { @@ -6852,8 +7077,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "rat", "url.original": "https://www5.example.net/culpa/isun.txt?cola=tura#rat", - "url.query": "orumwrit", + "url.path": "/culpa/isun.txt", + "url.query": "cola=tura", + "url.scheme": "https", "user.name": "caecatcu" }, { @@ -6886,8 +7116,8 @@ ], "related.ip": [ "10.224.212.88", - "10.53.82.96", - "10.35.240.70" + "10.35.240.70", + "10.53.82.96" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6952,9 +7182,9 @@ "oeius" ], "related.ip": [ - "10.66.149.234", + "10.186.253.240", "10.233.128.7", - "10.186.253.240" + "10.66.149.234" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7087,8 +7317,8 @@ ], "related.ip": [ "10.69.130.207", - "10.170.236.123", - "10.205.18.11" + "10.205.18.11", + "10.170.236.123" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7213,8 +7443,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "aborios", "url.original": "https://example.com/loremag/mqu.gif?bore=lapari#aborios", - "url.query": "sequa", + "url.path": "/loremag/mqu.gif", + "url.query": "bore=lapari", + "url.scheme": "https", "user.name": "epor" }, { @@ -7246,9 +7481,9 @@ "ineavol" ], "related.ip": [ - "10.9.41.221", "10.204.98.238", - "10.81.58.91" + "10.81.58.91", + "10.9.41.221" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7427,9 +7662,9 @@ "aturvel" ], "related.ip": [ + "10.207.207.106", "10.143.65.84", - "10.199.201.26", - "10.207.207.106" + "10.199.201.26" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7495,8 +7730,8 @@ ], "related.ip": [ "10.163.236.253", - "10.41.61.88", - "10.204.27.48" + "10.204.27.48", + "10.41.61.88" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7689,8 +7924,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ido", "url.original": "https://api.example.com/aco/empo.jpg?iumdol=iusm#ido", - "url.query": "ipsamv", + "url.path": "/aco/empo.jpg", + "url.query": "iumdol=iusm", + "url.scheme": "https", "user.name": "aquaeab" }, { @@ -7782,8 +8022,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "ect", "url.original": "https://example.org/uidolor/turve.htm?temporai=uasiarch#ect", - "url.query": "quatDui", + "url.path": "/uidolor/turve.htm", + "url.query": "temporai=uasiarch", + "url.scheme": "https", "user.name": "maperia" }, { @@ -7875,8 +8120,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "odtempo", "url.original": "https://www.example.net/Maloru/lapariat.htm?tlabori=rehender#odtempo", - "url.query": "isiutali", + "url.path": "/Maloru/lapariat.htm", + "url.query": "tlabori=rehender", + "url.scheme": "https", "user.name": "modo" }, { @@ -7923,8 +8173,8 @@ "rsa.investigations.event_vcat": "olupt", "rsa.misc.OS": "rumw", "rsa.misc.action": [ - "tali", - "block" + "block", + "tali" ], "rsa.misc.category": "itsedq", "rsa.misc.client": "esciu", @@ -7968,8 +8218,13 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "html", + "url.fragment": "one", "url.original": "https://www5.example.com/xea/ssecill.html?quianonn=quun#one", - "url.query": "tasper", + "url.path": "/xea/ssecill.html", + "url.query": "quianonn=quun", + "url.scheme": "https", "user.name": "dolo" } -] \ No newline at end of file +] From 0cc1bf49f1e261ab217dc75e85e6029808683f63 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Sat, 27 Mar 2021 21:29:10 +0000 Subject: [PATCH 07/16] Update Imperva, Netscout, O365, Sophos, Squid, Suricata, Zscaler --- .../module/f5/bigipafm/ingest/pipeline.yml | 31 + .../bigipafm/test/generated.log-expected.json | 398 ++++---- .../module/f5/bigipapm/ingest/pipeline.yml | 28 +- .../bigipapm/test/generated.log-expected.json | 6 +- .../test/generated.log-expected.json | 72 +- .../test/generated.log-expected.json | 16 +- .../fortinet/fortimanager/ingest/pipeline.yml | 27 +- .../test/generated.log-expected.json | 446 +++++--- .../imperva/securesphere/ingest/pipeline.yml | 31 + .../test/generated.log-expected.json | 401 ++++++-- .../netscout/sightline/ingest/pipeline.yml | 31 + .../test/generated.log-expected.json | 144 ++- .../module/o365/audit/ingest/pipeline.yml | 31 + .../06-sharepointfileop.log-expected.json | 32 + .../test/11-dlp-sharepoint.log-expected.json | 42 +- .../test/13-dlp-exchange.log-expected.json | 4 + .../module/sophos/utm/ingest/pipeline.yml | 31 + .../utm/test/generated.log-expected.json | 92 +- .../module/sophos/xg/ingest/antivirus.yml | 27 + .../filebeat/module/sophos/xg/ingest/atp.yml | 27 + .../xg/test/anti-virus.log-expected.json | 6 + .../sophos/xg/test/atp.log-expected.json | 12 +- .../module/squid/log/ingest/pipeline.yml | 35 + .../squid/log/test/access1.log-expected.json | 307 +++++- .../log/test/generated.log-expected.json | 952 +++++++++++++++--- .../module/suricata/eve/ingest/http.yml | 90 ++ .../module/suricata/eve/ingest/pipeline.yml | 56 +- .../filebeat/module/suricata/eve/manifest.yml | 1 + .../eve/test/eve-6.0.log-expected.json | 1 + .../module/suricata/eve/test/eve-small.log | 2 +- .../eve/test/eve-small.log-expected.json | 14 +- .../module/zscaler/zia/ingest/pipeline.yml | 33 +- .../zia/test/generated.log-expected.json | 664 +++++++++++- 33 files changed, 3279 insertions(+), 811 deletions(-) create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/http.yml diff --git a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml index 1c939d65a22f..43486b2e0570 100644 --- a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml @@ -10,6 +10,37 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json index f499a74458c7..0b756af87b85 100644 --- a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json @@ -24,10 +24,10 @@ "tatemac3541.api.corp" ], "related.ip": [ - "10.228.193.207", - "10.11.196.142", + "10.165.201.71", "10.208.121.85", - "10.165.201.71" + "10.228.193.207", + "10.11.196.142" ], "related.user": [ "billoi" @@ -92,10 +92,10 @@ "enatus2114.mail.home" ], "related.ip": [ + "10.92.202.200", "10.51.132.10", - "10.162.9.235", "10.94.67.230", - "10.92.202.200" + "10.162.9.235" ], "related.user": [ "byC" @@ -160,9 +160,9 @@ "gelit6728.api.invalid" ], "related.ip": [ + "10.122.116.161", "10.209.155.149", "10.82.56.117", - "10.122.116.161", "10.191.68.244" ], "related.user": [ @@ -295,10 +295,10 @@ "emquiavo452.internal.localhost" ], "related.ip": [ - "10.96.35.212", "10.206.197.113", "10.151.111.38", - "10.159.182.171" + "10.159.182.171", + "10.96.35.212" ], "related.user": [ "mol" @@ -364,9 +364,9 @@ ], "related.ip": [ "10.126.177.162", + "10.89.163.114", "10.213.113.28", - "10.169.144.147", - "10.89.163.114" + "10.169.144.147" ], "related.user": [ "ist" @@ -430,10 +430,10 @@ "ittenbyC7838.api.localdomain" ], "related.ip": [ - "10.18.124.28", + "10.146.88.52", "10.101.223.43", - "10.103.107.47", - "10.146.88.52" + "10.18.124.28", + "10.103.107.47" ], "related.user": [ "rudexerc" @@ -498,10 +498,10 @@ "ume465.corp" ], "related.ip": [ - "10.189.109.245", "10.69.57.206", + "10.150.220.75", "10.110.99.17", - "10.150.220.75" + "10.189.109.245" ], "related.user": [ "onse" @@ -567,8 +567,9 @@ "related.ip": [ "10.153.136.222", "10.19.194.101", - "10.199.34.241", - "10.121.219.204" + "10.121.219.204", + "10.153.136.222", + "10.199.34.241" ], "related.user": [ "temveleu" @@ -632,10 +633,10 @@ "aliqu6801.api.localdomain" ], "related.ip": [ - "10.182.199.231", - "10.57.103.192", "10.46.27.57", - "10.64.141.105" + "10.182.199.231", + "10.64.141.105", + "10.57.103.192" ], "related.user": [ "ice" @@ -701,8 +702,8 @@ "related.ip": [ "10.3.134.237", "10.164.6.207", - "10.32.67.231", - "10.160.210.31" + "10.160.210.31", + "10.32.67.231" ], "related.user": [ "pic" @@ -767,8 +768,8 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ - "10.182.178.217", "10.235.101.253", + "10.182.178.217", "10.201.6.10", "10.42.138.192" ], @@ -835,11 +836,9 @@ "stlabo1228.mail.host" ], "related.ip": [ - "10.22.102.198", "10.86.101.235", - "10.151.161.70", "10.194.247.171", - "10.86.101.235", + "10.151.161.70", "10.22.102.198" ], "related.user": [ @@ -905,9 +904,9 @@ "ecte4762.local" ], "related.ip": [ + "10.204.35.15", "10.107.168.60", "10.167.172.155", - "10.204.35.15", "10.174.252.105" ], "related.user": [ @@ -972,10 +971,10 @@ "smo7167.www.test" ], "related.ip": [ - "10.99.249.210", "10.182.191.174", + "10.214.249.164", "10.81.26.208", - "10.214.249.164" + "10.99.249.210" ], "related.user": [ "upta" @@ -1039,10 +1038,10 @@ "sauteiru4554.api.domain" ], "related.ip": [ - "10.101.226.128", + "10.220.5.143", "10.88.101.53", - "10.201.238.90", - "10.220.5.143" + "10.101.226.128", + "10.201.238.90" ], "related.user": [ "porro" @@ -1107,9 +1106,9 @@ ], "related.ip": [ "10.30.133.66", + "10.243.218.215", "10.157.18.252", - "10.217.150.196", - "10.243.218.215" + "10.217.150.196" ], "related.user": [ "evit" @@ -1173,10 +1172,10 @@ "quid3147.mail.home" ], "related.ip": [ - "10.167.227.44", "10.181.133.187", - "10.148.161.250", - "10.66.181.6" + "10.66.181.6", + "10.167.227.44", + "10.148.161.250" ], "related.user": [ "adipisc" @@ -1241,10 +1240,10 @@ "umdolo1029.mail.localhost" ], "related.ip": [ - "10.84.163.178", - "10.74.11.43", + "10.107.9.163", "10.54.17.32", - "10.107.9.163" + "10.84.163.178", + "10.74.11.43" ], "related.user": [ "mquisno" @@ -1309,9 +1308,9 @@ ], "related.ip": [ "10.230.129.252", - "10.184.73.211", + "10.192.229.221", "10.112.32.213", - "10.192.229.221" + "10.184.73.211" ], "related.user": [ "odi" @@ -1377,9 +1376,9 @@ ], "related.ip": [ "10.7.200.140", - "10.198.213.189", "10.199.216.143", - "10.161.148.64" + "10.161.148.64", + "10.198.213.189" ], "related.user": [ "ccaeca" @@ -1443,10 +1442,10 @@ "mex2054.mail.corp" ], "related.ip": [ - "10.206.96.56", - "10.128.157.27", + "10.65.232.27", "10.22.187.69", - "10.65.232.27" + "10.206.96.56", + "10.128.157.27" ], "related.user": [ "uaeab" @@ -1511,9 +1510,9 @@ ], "related.ip": [ "10.194.210.62", - "10.68.253.120", "10.71.114.14", - "10.183.130.225" + "10.183.130.225", + "10.68.253.120" ], "related.user": [ "admin" @@ -1580,8 +1579,9 @@ "related.ip": [ "10.107.45.175", "10.31.177.226", - "10.47.255.237", - "10.45.253.103" + "10.107.45.175", + "10.45.253.103", + "10.47.255.237" ], "related.user": [ "remagn" @@ -1713,10 +1713,10 @@ "ectiono2241.lan" ], "related.ip": [ - "10.255.74.136", - "10.69.161.78", + "10.2.114.9", "10.163.209.70", - "10.2.114.9" + "10.255.74.136", + "10.69.161.78" ], "related.user": [ "olabor" @@ -1780,10 +1780,10 @@ "umetMal1664.mail.lan" ], "related.ip": [ - "10.12.129.137", - "10.252.102.110", "10.46.115.216", - "10.184.59.148" + "10.184.59.148", + "10.252.102.110", + "10.12.129.137" ], "related.user": [ "perspici" @@ -1848,10 +1848,10 @@ "derit5270.mail.local" ], "related.ip": [ - "10.155.204.243", "10.105.52.140", "10.199.194.79", - "10.105.52.140" + "10.155.204.243", + "10.81.184.7" ], "related.user": [ "eetd" @@ -2052,10 +2052,10 @@ "orumw5960.www5.home" ], "related.ip": [ - "10.172.154.97", + "10.248.111.207", "10.37.193.70", "10.162.97.197", - "10.248.111.207" + "10.172.154.97" ], "related.user": [ "culpaq" @@ -2119,10 +2119,10 @@ "oinv5493.internal.domain" ], "related.ip": [ - "10.171.221.230", - "10.45.35.180", "10.36.63.31", - "10.222.165.250" + "10.45.35.180", + "10.222.165.250", + "10.171.221.230" ], "related.user": [ "otamr" @@ -2186,10 +2186,10 @@ "tnonproi195.api.home" ], "related.ip": [ - "10.83.238.145", "10.238.4.219", "10.199.127.211", - "10.1.171.61" + "10.1.171.61", + "10.83.238.145" ], "related.user": [ "reetdolo" @@ -2253,10 +2253,10 @@ "edictasu5362.internal.localhost" ], "related.ip": [ - "10.44.226.104", "10.170.252.219", + "10.74.213.42", "10.65.141.244", - "10.74.213.42" + "10.44.226.104" ], "related.user": [ "Nequepo" @@ -2320,11 +2320,9 @@ "uido492.www5.home" ], "related.ip": [ - "10.183.223.149", - "10.225.141.172", "10.225.255.211", - "10.225.141.172", "10.183.223.149", + "10.225.141.172", "10.180.48.221" ], "related.user": [ @@ -2389,10 +2387,10 @@ "redo6311.api.invalid" ], "related.ip": [ + "10.169.123.103", "10.176.64.28", - "10.97.138.181", "10.205.174.181", - "10.169.123.103" + "10.97.138.181" ], "related.user": [ "eseruntm" @@ -2457,10 +2455,10 @@ "dolorem1698.www.domain" ], "related.ip": [ - "10.75.120.11", "10.53.101.131", + "10.169.101.161", "10.204.4.40", - "10.169.101.161" + "10.75.120.11" ], "related.user": [ "tquo" @@ -2525,10 +2523,10 @@ "evitae7333.www.lan" ], "related.ip": [ + "10.6.222.112", "10.87.120.87", "10.156.117.169", - "10.28.51.219", - "10.6.222.112" + "10.28.51.219" ], "related.user": [ "onsequu" @@ -2592,10 +2590,10 @@ "arc2412.mail.lan" ], "related.ip": [ - "10.253.167.17", "10.247.44.59", + "10.4.126.103", "10.57.89.155", - "10.4.126.103" + "10.253.167.17" ], "related.user": [ "ntorever" @@ -2660,9 +2658,9 @@ ], "related.ip": [ "10.143.183.208", - "10.36.69.125", + "10.15.240.220", "10.248.206.210", - "10.15.240.220" + "10.36.69.125" ], "related.user": [ "met" @@ -2727,9 +2725,9 @@ "edqu2208.www.localhost" ], "related.ip": [ + "10.34.133.2", "10.69.170.107", "10.142.186.43", - "10.34.133.2", "10.6.32.7" ], "related.user": [ @@ -2795,11 +2793,10 @@ "ender5647.www5.example" ], "related.ip": [ - "10.121.153.197", - "10.170.165.164", "10.142.22.24", - "10.59.103.10", - "10.170.165.164" + "10.170.165.164", + "10.121.153.197", + "10.59.103.10" ], "related.user": [ "borumSec" @@ -2932,9 +2929,9 @@ "uatu2894.api.lan" ], "related.ip": [ - "10.64.139.17", - "10.8.29.219", "10.70.7.23", + "10.8.29.219", + "10.64.139.17", "10.40.177.138" ], "related.user": [ @@ -3000,12 +2997,9 @@ ], "related.ip": [ "10.180.62.222", - "10.67.173.228", - "10.67.221.220", - "10.67.173.228", "10.2.189.20", - "10.67.221.220", - "10.180.62.222" + "10.67.173.228", + "10.67.221.220" ], "related.user": [ "uptasnul" @@ -3070,11 +3064,10 @@ "uian521.www.example" ], "related.ip": [ + "10.209.52.47", "10.56.134.118", "10.147.127.181", - "10.56.134.118", - "10.196.176.243", - "10.209.52.47" + "10.196.176.243" ], "related.user": [ "tasu" @@ -3139,8 +3132,8 @@ ], "related.ip": [ "10.85.13.237", - "10.226.24.84", "10.231.18.90", + "10.226.24.84", "10.248.140.59" ], "related.user": [ @@ -3206,10 +3199,10 @@ "ntsunt4894.mail.domain" ], "related.ip": [ - "10.59.215.207", - "10.207.183.204", "10.203.46.215", - "10.8.224.72" + "10.59.215.207", + "10.8.224.72", + "10.207.183.204" ], "related.user": [ "eruntmo" @@ -3341,10 +3334,10 @@ "oluptat6960.www5.test" ], "related.ip": [ - "10.166.142.198", - "10.105.120.162", + "10.211.29.187", "10.175.181.138", - "10.211.29.187" + "10.105.120.162", + "10.166.142.198" ], "related.user": [ "tium" @@ -3409,10 +3402,10 @@ "fugiatnu2498.www.localhost" ], "related.ip": [ - "10.220.202.102", - "10.182.213.195", "10.195.139.25", - "10.122.133.162" + "10.122.133.162", + "10.220.202.102", + "10.182.213.195" ], "related.user": [ "aquae" @@ -3477,10 +3470,11 @@ "ptat3230.domain" ], "related.ip": [ + "10.247.144.9", "10.156.208.5", - "10.53.72.161", "10.33.143.163", - "10.247.144.9" + "10.53.72.161", + "10.33.143.163" ], "related.user": [ "scip" @@ -3544,12 +3538,10 @@ "exer447.internal.localhost" ], "related.ip": [ - "10.113.65.192", "10.21.58.162", "10.241.143.145", - "10.21.58.162", "10.35.190.164", - "10.21.58.162" + "10.113.65.192" ], "related.user": [ "porin" @@ -3614,10 +3606,10 @@ "itanimi1934.home" ], "related.ip": [ - "10.75.113.240", + "10.19.154.103", "10.53.27.253", - "10.129.16.166", - "10.19.154.103" + "10.75.113.240", + "10.129.16.166" ], "related.user": [ "luptat" @@ -3682,10 +3674,10 @@ "pteurs1031.mail.corp" ], "related.ip": [ - "10.150.153.61", - "10.125.150.220", + "10.22.213.196", "10.120.50.13", - "10.22.213.196" + "10.125.150.220", + "10.150.153.61" ], "related.user": [ "inculpa" @@ -3751,9 +3743,9 @@ ], "related.ip": [ "10.238.171.184", - "10.85.52.249", + "10.113.2.13", "10.229.155.171", - "10.113.2.13" + "10.85.52.249" ], "related.user": [ "tatiset" @@ -3818,11 +3810,10 @@ "tatis7315.mail.home" ], "related.ip": [ - "10.249.174.35", - "10.198.150.185", "10.249.174.35", "10.51.245.225", - "10.220.1.249" + "10.220.1.249", + "10.198.150.185" ], "related.user": [ "quela" @@ -3887,10 +3878,10 @@ "eosqui3723.api.localdomain" ], "related.ip": [ + "10.251.82.195", "10.152.157.32", "10.190.96.181", - "10.38.185.31", - "10.251.82.195" + "10.38.185.31" ], "related.user": [ "olorese" @@ -3954,9 +3945,9 @@ "itaedict199.mail.corp" ], "related.ip": [ - "10.230.112.179", "10.103.102.242", "10.190.247.194", + "10.230.112.179", "10.211.198.50" ], "related.user": [ @@ -4022,7 +4013,6 @@ ], "related.ip": [ "10.47.223.155", - "10.219.83.199", "10.101.13.122", "10.251.101.61" ], @@ -4089,9 +4079,9 @@ "saute7421.www.invalid" ], "related.ip": [ + "10.21.30.43", "10.21.80.157", "10.31.86.83", - "10.21.30.43", "10.83.136.233" ], "related.user": [ @@ -4157,11 +4147,10 @@ "oluptas1637.home" ], "related.ip": [ - "10.45.152.205", + "10.194.197.107", "10.27.181.27", - "10.195.90.73", "10.45.152.205", - "10.194.197.107" + "10.195.90.73" ], "related.user": [ "datatn" @@ -4226,10 +4215,10 @@ "ididu5505.api.localdomain" ], "related.ip": [ - "10.222.2.132", "10.129.161.18", - "10.183.90.25", - "10.43.239.97" + "10.222.2.132", + "10.43.239.97", + "10.183.90.25" ], "related.user": [ "aedicta" @@ -4293,10 +4282,10 @@ "mqui1099.api.corp" ], "related.ip": [ - "10.189.162.131", - "10.67.129.100", "10.248.156.138", - "10.231.167.171" + "10.231.167.171", + "10.189.162.131", + "10.67.129.100" ], "related.user": [ "sedquia" @@ -4361,11 +4350,10 @@ "siuta2155.lan" ], "related.ip": [ - "10.63.103.30", - "10.185.107.27", - "10.63.103.30", "10.6.146.184", - "10.142.106.66" + "10.142.106.66", + "10.185.107.27", + "10.63.103.30" ], "related.user": [ "sequu" @@ -4430,8 +4418,8 @@ ], "related.ip": [ "10.93.39.237", - "10.0.202.9", "10.214.93.200", + "10.0.202.9", "10.119.179.182" ], "related.user": [ @@ -4497,10 +4485,10 @@ "aute2433.mail.lan" ], "related.ip": [ - "10.30.189.166", "10.252.204.162", - "10.123.154.140", - "10.28.145.163" + "10.28.145.163", + "10.30.189.166", + "10.123.154.140" ], "related.user": [ "imadmin" @@ -4564,10 +4552,10 @@ "idolo6535.internal.example" ], "related.ip": [ - "10.79.49.3", "10.29.122.183", + "10.145.128.250", "10.46.162.198", - "10.145.128.250" + "10.79.49.3" ], "related.user": [ "eni" @@ -4632,10 +4620,10 @@ "one7728.api.localdomain" ], "related.ip": [ - "10.142.235.217", - "10.166.169.167", "10.65.174.196", - "10.177.232.136" + "10.177.232.136", + "10.142.235.217", + "10.166.169.167" ], "related.user": [ "olors" @@ -4701,9 +4689,9 @@ ], "related.ip": [ "10.53.188.140", + "10.29.217.44", "10.215.184.154", - "10.191.78.86", - "10.29.217.44" + "10.191.78.86" ], "related.user": [ "iarc" @@ -4768,10 +4756,10 @@ "emq2514.api.localhost" ], "related.ip": [ - "10.76.148.147", - "10.135.77.156", "10.74.74.129", - "10.46.222.149" + "10.135.77.156", + "10.46.222.149", + "10.76.148.147" ], "related.user": [ "urve" @@ -4835,10 +4823,10 @@ "agna5654.www.corp" ], "related.ip": [ - "10.130.203.37", - "10.145.49.29", "10.96.200.223", - "10.130.203.37" + "10.130.203.37", + "10.11.146.253", + "10.145.49.29" ], "related.user": [ "mvele" @@ -4904,6 +4892,7 @@ "related.ip": [ "10.162.78.48", "10.162.2.180", + "10.24.23.209", "10.48.75.140" ], "related.user": [ @@ -4968,8 +4957,7 @@ "sequatD163.internal.example" ], "related.ip": [ - "10.97.105.115", - "10.119.12.186", + "10.66.92.83", "10.151.206.38", "10.119.12.186", "10.97.105.115" @@ -5036,8 +5024,8 @@ "itamet1303.invalid" ], "related.ip": [ - "10.64.76.142", "10.201.132.114", + "10.64.76.142", "10.12.148.73", "10.169.139.250" ], @@ -5171,10 +5159,10 @@ "uredol2174.home" ], "related.ip": [ + "10.240.62.238", "10.236.67.227", "10.191.27.182", - "10.134.238.8", - "10.240.62.238" + "10.134.238.8" ], "related.user": [ "tlabo" @@ -5238,10 +5226,10 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.65.35.64", - "10.109.14.142", "10.22.231.91", - "10.165.66.92" + "10.165.66.92", + "10.109.14.142", + "10.65.35.64" ], "related.user": [ "perna" @@ -5372,13 +5360,9 @@ "nonn1650.www.test" ], "related.ip": [ - "10.79.208.135", "10.221.199.137", "10.140.118.182", - "10.79.208.135", "10.88.226.76", - "10.221.199.137", - "10.140.118.182", "10.79.208.135" ], "related.user": [ @@ -5445,9 +5429,9 @@ ], "related.ip": [ "10.126.61.230", + "10.35.73.208", "10.189.244.22", - "10.133.48.55", - "10.35.73.208" + "10.133.48.55" ], "related.user": [ "tia" @@ -5511,11 +5495,10 @@ "suscipit587.www.localhost" ], "related.ip": [ + "10.81.154.115", "10.35.65.72", "10.240.94.109", - "10.239.194.105", - "10.81.154.115", - "10.240.94.109" + "10.239.194.105" ], "related.user": [ "reseo" @@ -5580,10 +5563,10 @@ "mnisiut6146.internal.local" ], "related.ip": [ - "10.150.56.227", - "10.52.70.192", "10.38.253.213", - "10.248.72.104" + "10.248.72.104", + "10.52.70.192", + "10.150.56.227" ], "related.user": [ "ionem" @@ -5648,10 +5631,10 @@ "borios1067.www5.home" ], "related.ip": [ - "10.203.193.134", "10.218.15.164", - "10.62.218.239", - "10.73.172.186" + "10.73.172.186", + "10.203.193.134", + "10.62.218.239" ], "related.user": [ "reh" @@ -5715,10 +5698,10 @@ "msequ323.www.example" ], "related.ip": [ - "10.10.46.43", "10.131.127.113", - "10.136.211.234", - "10.60.20.76" + "10.60.20.76", + "10.10.46.43", + "10.136.211.234" ], "related.user": [ "nev" @@ -5783,10 +5766,10 @@ "tdolorem813.internal.host" ], "related.ip": [ - "10.50.177.151", "10.233.181.250", "10.187.237.220", - "10.248.0.74" + "10.248.0.74", + "10.50.177.151" ], "related.user": [ "ugiatq" @@ -5852,8 +5835,8 @@ ], "related.ip": [ "10.96.223.46", - "10.80.129.81", "10.248.248.120", + "10.80.129.81", "10.189.43.11" ], "related.user": [ @@ -5920,10 +5903,9 @@ ], "related.ip": [ "10.66.106.186", - "10.173.114.63", + "10.91.115.139", "10.102.109.199", - "10.66.106.186", - "10.102.109.199" + "10.173.114.63" ], "related.user": [ "tNequ" @@ -5988,10 +5970,10 @@ "orpori3334.www.local" ], "related.ip": [ - "10.198.157.122", - "10.159.155.88", "10.221.223.127", - "10.0.175.17" + "10.198.157.122", + "10.0.175.17", + "10.159.155.88" ], "related.user": [ "iquipex" @@ -6055,10 +6037,10 @@ "equu7361.www5.localdomain" ], "related.ip": [ - "10.30.20.187", "10.7.212.201", - "10.252.136.130", - "10.189.70.237" + "10.189.70.237", + "10.30.20.187", + "10.252.136.130" ], "related.user": [ "ugiat" @@ -6191,8 +6173,8 @@ "uisnostr2390.mail.domain" ], "related.ip": [ - "10.251.167.219", "10.17.20.93", + "10.251.167.219", "10.181.134.69", "10.219.174.45" ], @@ -6259,10 +6241,10 @@ "luptate4811.mail.example" ], "related.ip": [ + "10.28.233.253", "10.37.14.20", - "10.223.99.90", "10.30.117.82", - "10.28.233.253" + "10.223.99.90" ], "related.user": [ "numqua" @@ -6329,8 +6311,8 @@ "related.ip": [ "10.125.20.22", "10.57.85.113", - "10.50.61.114", - "10.8.32.17" + "10.8.32.17", + "10.50.61.114" ], "related.user": [ "qua" @@ -6395,9 +6377,9 @@ "lorinrep7686.mail.corp" ], "related.ip": [ - "10.113.78.101", - "10.200.28.55", "10.181.63.82", + "10.200.28.55", + "10.113.78.101", "10.215.224.27" ], "related.user": [ @@ -6532,8 +6514,8 @@ ], "related.ip": [ "10.90.93.4", - "10.92.168.198", "10.39.100.88", + "10.92.168.198", "10.18.176.44" ], "related.user": [ @@ -6666,10 +6648,10 @@ "cipitl2184.localdomain" ], "related.ip": [ - "10.84.64.28", "10.240.47.113", - "10.31.147.51", - "10.209.226.7" + "10.84.64.28", + "10.209.226.7", + "10.31.147.51" ], "related.user": [ "ull" @@ -6736,8 +6718,8 @@ "related.ip": [ "10.32.20.4", "10.52.13.192", - "10.86.1.244", - "10.225.189.229" + "10.225.189.229", + "10.86.1.244" ], "related.user": [ "odtemp" diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index c61cbf6261c3..2c878a87f973 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -11,21 +11,26 @@ processors: field: user_agent.original ignore_missing: true # URL - - set: - field: _temp_.url_domain - value: "{{url.domain}}" - ignore_failure: true - if: ctx?.url?.domain != null - uri_parts: field: url.original + target_field: _temp_.url ignore_failure: true if: ctx?.url?.original != null - - append: - field: url.domain - value: "{{_temp_.url_domain}}" - ignore_failure: true - allow_duplicates: false - if: ctx?._temp_?.url_domain != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } - remove: field: _temp_ ignore_missing: true @@ -36,7 +41,6 @@ processors: - urldecode: field: http.request.referrer ignore_missing: true - ignore_failure: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json index 181f7b8e9f12..a9d1a6036046 100644 --- a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json @@ -367,8 +367,8 @@ "observer.vendor": "F5", "process.pid": 2289, "related.ip": [ - "10.225.160.182", - "10.204.123.107" + "10.204.123.107", + "10.225.160.182" ], "rsa.internal.messageid": "01490500", "rsa.misc.log_session_id": "eFinib", @@ -2652,4 +2652,4 @@ ], "user.name": "oditem" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json index 695e1a33648d..5e403ecc4e6c 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json @@ -25,8 +25,8 @@ "boNemoe4402.www.invalid" ], "related.ip": [ - "10.102.123.34", - "10.150.92.220" + "10.150.92.220", + "10.102.123.34" ], "related.user": [ "sumdo" @@ -211,8 +211,8 @@ "tinculp2940.internal.local" ], "related.ip": [ - "10.202.204.154", - "10.134.137.177" + "10.134.137.177", + "10.202.204.154" ], "related.user": [ "orsitame" @@ -582,8 +582,8 @@ "tatno4987.www5.localhost" ], "related.ip": [ - "10.203.5.162", - "10.54.231.100" + "10.54.231.100", + "10.203.5.162" ], "related.user": [ "umdolore" @@ -706,8 +706,8 @@ "essecill2595.mail.local" ], "related.ip": [ - "10.57.40.29", - "10.210.213.18" + "10.210.213.18", + "10.57.40.29" ], "related.user": [ "onse" @@ -890,8 +890,8 @@ "dolore6103.www5.example" ], "related.ip": [ - "10.38.22.45", - "10.72.29.73" + "10.72.29.73", + "10.38.22.45" ], "related.user": [ "onproide" @@ -1199,8 +1199,8 @@ "eme6710.mail.invalid" ], "related.ip": [ - "10.121.219.204", - "10.104.134.200" + "10.104.134.200", + "10.121.219.204" ], "related.user": [ "uptat" @@ -1507,8 +1507,8 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ - "10.178.77.231", - "10.163.5.243" + "10.163.5.243", + "10.178.77.231" ], "related.user": [ "liquide" @@ -2185,8 +2185,8 @@ "eumiu765.api.lan" ], "related.ip": [ - "10.4.157.1", - "10.184.18.202" + "10.184.18.202", + "10.4.157.1" ], "related.user": [ "oditem" @@ -2432,8 +2432,8 @@ "ccaeca7077.internal.corp" ], "related.ip": [ - "10.215.205.216", - "10.216.54.184" + "10.216.54.184", + "10.215.205.216" ], "related.user": [ "ameiusm" @@ -3236,8 +3236,8 @@ "Sedut1775.www.domain" ], "related.ip": [ - "10.248.165.185", - "10.86.11.48" + "10.86.11.48", + "10.248.165.185" ], "related.user": [ "dquiac" @@ -3298,8 +3298,8 @@ "mac7484.www5.test" ], "related.ip": [ - "10.118.6.177", - "10.47.125.38" + "10.47.125.38", + "10.118.6.177" ], "related.user": [ "quunt" @@ -3732,8 +3732,8 @@ "minim459.mail.local" ], "related.ip": [ - "10.123.199.198", - "10.17.87.79" + "10.17.87.79", + "10.123.199.198" ], "related.user": [ "ratvolu" @@ -4042,8 +4042,8 @@ "mag1506.internal.domain" ], "related.ip": [ - "10.131.126.109", - "10.182.152.242" + "10.182.152.242", + "10.131.126.109" ], "related.user": [ "dolor" @@ -4412,8 +4412,8 @@ "emullamc5418.mail.test" ], "related.ip": [ - "10.45.54.107", - "10.82.133.66" + "10.82.133.66", + "10.45.54.107" ], "related.user": [ "olorem" @@ -4844,8 +4844,8 @@ "laborum5749.www.example" ], "related.ip": [ - "10.189.42.62", - "10.36.110.69" + "10.36.110.69", + "10.189.42.62" ], "related.user": [ "eque" @@ -5216,8 +5216,8 @@ "amc3059.local" ], "related.ip": [ - "10.181.41.154", - "10.29.109.126" + "10.29.109.126", + "10.181.41.154" ], "related.user": [ "labo" @@ -5834,8 +5834,8 @@ "rumSecti111.www5.domain" ], "related.ip": [ - "10.222.245.80", - "10.87.90.49" + "10.87.90.49", + "10.222.245.80" ], "related.user": [ "ptatemse" @@ -5896,8 +5896,8 @@ "olores7881.local" ], "related.ip": [ - "10.143.53.214", - "10.87.144.208" + "10.87.144.208", + "10.143.53.214" ], "related.user": [ "psumq" diff --git a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json index 8e79962d5bc2..920acad2a309 100644 --- a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json @@ -596,8 +596,8 @@ "lamcolab3252.www.invalid" ], "related.ip": [ - "10.179.124.125", - "10.177.36.38" + "10.177.36.38", + "10.179.124.125" ], "rsa.email.email_dst": "ectio", "rsa.email.email_src": "sequine", @@ -1011,8 +1011,8 @@ "lapariat7287.internal.host" ], "related.ip": [ - "10.140.7.83", - "10.68.246.187" + "10.68.246.187", + "10.140.7.83" ], "rsa.email.email_dst": "gna", "rsa.email.email_src": "icabo", @@ -3131,8 +3131,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "modi6930.internal.test", - "taevitae6868.www.corp" + "taevitae6868.www.corp", + "modi6930.internal.test" ], "related.ip": [ "10.60.164.100", @@ -3845,8 +3845,8 @@ "ncu3839.www.localhost" ], "related.ip": [ - "10.201.105.58", - "10.251.183.113" + "10.251.183.113", + "10.201.105.58" ], "rsa.email.email_dst": "ionemu", "rsa.email.email_src": "ent", diff --git a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml index 87270afa2112..5ee19350ce83 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml @@ -11,21 +11,26 @@ processors: field: user_agent.original ignore_missing: true # URL - - set: - field: _temp_.url_domain - value: "{{url.domain}}" - ignore_failure: true - if: ctx?.url?.domain != null - uri_parts: field: url.original + target_field: _temp_.url ignore_failure: true if: ctx?.url?.original != null - - append: - field: url.domain - value: "{{_temp_.url_domain}}" - ignore_failure: true - allow_duplicates: false - if: ctx?._temp_?.url_domain != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } - remove: field: _temp_ ignore_missing: true diff --git a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json index 252b96fc403a..dfec5c0d23a0 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json @@ -160,7 +160,10 @@ "url.fragment": "roinBCS", "url.original": "https://www.example.net/orisn/cca.htm?ofdeF=metcons#roinBCS", "url.path": "/orisn/cca.htm", - "url.query": "ofdeF=metcons", + "url.query": [ + "taspe", + "ofdeF=metcons" + ], "url.scheme": "https", "user.name": "oluptas" }, @@ -393,7 +396,10 @@ "url.fragment": "saute", "url.original": "https://api.example.com/iumto/aboreetd.gif?dun=enim#saute", "url.path": "/iumto/aboreetd.gif", - "url.query": "dun=enim", + "url.query": [ + "nsect", + "dun=enim" + ], "url.scheme": "https", "user.name": "rveli" }, @@ -425,8 +431,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.152", "related.hosts": [ - "orain", "onse", + "orain", "agna7678.internal.host" ], "related.ip": [ @@ -491,7 +497,10 @@ "url.fragment": "picia", "url.original": "https://www.example.net/tasuntex/sunt.txt?ume=incidi#picia", "url.path": "/tasuntex/sunt.txt", - "url.query": "ume=incidi", + "url.query": [ + "nsec", + "ume=incidi" + ], "url.scheme": "https", "user.name": "aperia" }, @@ -589,7 +598,10 @@ "url.fragment": "tlabore", "url.original": "https://www.example.org/inesci/serror.html?mqu=apariat#tlabore", "url.path": "/inesci/serror.html", - "url.query": "mqu=apariat", + "url.query": [ + "nsectet", + "mqu=apariat" + ], "url.scheme": "https", "user.name": "nnum" }, @@ -622,9 +634,9 @@ "labore" ], "related.ip": [ + "10.176.216.90", "10.186.85.3", - "10.114.16.155", - "10.176.216.90" + "10.114.16.155" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -693,8 +705,8 @@ "eturadi6608.mail.host" ], "related.ip": [ - "10.61.163.4", - "10.23.62.94" + "10.23.62.94", + "10.61.163.4" ], "related.user": [ "non" @@ -754,7 +766,10 @@ "url.fragment": "issus", "url.original": "https://api.example.com/nimid/itatione.htm?umwr=oluptate#issus", "url.path": "/nimid/itatione.htm", - "url.query": "umwr=oluptate", + "url.query": [ + "atem", + "umwr=oluptate" + ], "url.scheme": "https", "user.name": "non" }, @@ -786,8 +801,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2580", "related.hosts": [ - "tani", "ecte", + "tani", "ipsumdol4488.api.localdomain" ], "related.ip": [ @@ -852,7 +867,10 @@ "url.fragment": "pora", "url.original": "https://example.com/tetu/stru.htm?tlabore=Exc#pora", "url.path": "/tetu/stru.htm", - "url.query": "tlabore=Exc", + "url.query": [ + "atno", + "tlabore=Exc" + ], "url.scheme": "https", "user.name": "cons" }, @@ -885,8 +903,8 @@ "diconseq" ], "related.ip": [ - "10.106.162.153", "10.238.164.74", + "10.106.162.153", "10.58.214.16" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -952,8 +970,8 @@ "tenimad" ], "related.ip": [ - "10.217.150.196", "10.110.31.190", + "10.217.150.196", "10.225.141.20" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -1034,8 +1052,8 @@ "rsa.investigations.event_vcat": "xer", "rsa.misc.OS": "fugi", "rsa.misc.action": [ - "deny", - "umdolo" + "umdolo", + "deny" ], "rsa.misc.category": "conseq", "rsa.misc.client": "cusant", @@ -1084,7 +1102,10 @@ "url.fragment": "isnisi", "url.original": "https://api.example.com/isnostr/umqu.htm?emquia=inesci#isnisi", "url.path": "/isnostr/umqu.htm", - "url.query": "emquia=inesci", + "url.query": [ + "uptate", + "emquia=inesci" + ], "url.scheme": "https", "user.name": "proide" }, @@ -1249,7 +1270,10 @@ "url.fragment": "urerepr", "url.original": "https://example.org/tiaec/rumwrit.txt?oconsequ=edquiac#urerepr", "url.path": "/tiaec/rumwrit.txt", - "url.query": "oconsequ=edquiac", + "url.query": [ + "iaeco", + "oconsequ=edquiac" + ], "url.scheme": "https", "user.name": "tetur" }, @@ -1282,9 +1306,9 @@ "xea" ], "related.ip": [ - "10.51.213.42", "10.233.120.207", - "10.98.194.212" + "10.98.194.212", + "10.51.213.42" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1349,9 +1373,9 @@ "tla" ], "related.ip": [ - "10.67.132.242", "10.241.132.176", - "10.245.187.229" + "10.245.187.229", + "10.67.132.242" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1420,8 +1444,8 @@ "tore7088.www.invalid" ], "related.ip": [ - "10.199.47.220", - "10.212.214.4" + "10.212.214.4", + "10.199.47.220" ], "related.user": [ "atv" @@ -1481,7 +1505,10 @@ "url.fragment": "itlab", "url.original": "https://mail.example.org/ptassita/its.gif?risnis=uov#itlab", "url.path": "/ptassita/its.gif", - "url.query": "risnis=uov", + "url.query": [ + "hil", + "risnis=uov" + ], "url.scheme": "https", "user.name": "atv" }, @@ -1579,7 +1606,10 @@ "url.fragment": "toditaut", "url.original": "https://www.example.org/redol/gnaa.htm?aliquamq=dtempori#toditaut", "url.path": "/redol/gnaa.htm", - "url.query": "aliquamq=dtempori", + "url.query": [ + "quames", + "aliquamq=dtempori" + ], "url.scheme": "https", "user.name": "orese" }, @@ -1616,8 +1646,8 @@ "eturad6143.www.home" ], "related.ip": [ - "10.128.46.70", - "10.95.117.134" + "10.95.117.134", + "10.128.46.70" ], "related.user": [ "enim" @@ -1677,7 +1707,10 @@ "url.fragment": "offici", "url.original": "https://example.net/aparia/tatnon.jpg?rever=ore#offici", "url.path": "/aparia/tatnon.jpg", - "url.query": "rever=ore", + "url.query": [ + "essecill", + "rever=ore" + ], "url.scheme": "https", "user.name": "enim" }, @@ -1775,7 +1808,10 @@ "url.fragment": "equamnih", "url.original": "https://www.example.com/tali/BCS.txt?iqu=niamqu#equamnih", "url.path": "/tali/BCS.txt", - "url.query": "iqu=niamqu", + "url.query": [ + "quiineav", + "iqu=niamqu" + ], "url.scheme": "https", "user.name": "inculp" }, @@ -1873,7 +1909,10 @@ "url.fragment": "mexer", "url.original": "https://www.example.com/iame/orroquis.htm?tiumd=ntmoll#mexer", "url.path": "/iame/orroquis.htm", - "url.query": "tiumd=ntmoll", + "url.query": [ + "tec", + "tiumd=ntmoll" + ], "url.scheme": "https", "user.name": "olorem" }, @@ -1971,7 +2010,10 @@ "url.fragment": "mnisi", "url.original": "https://www.example.net/iame/laudanti.htm?stquido=rsitvolu#mnisi", "url.path": "/iame/laudanti.htm", - "url.query": "stquido=rsitvolu", + "url.query": [ + "serror", + "stquido=rsitvolu" + ], "url.scheme": "https", "user.name": "lmo" }, @@ -2138,7 +2180,10 @@ "url.fragment": "rQuisaut", "url.original": "https://www.example.org/ess/quiad.jpg?ten=litanim#rQuisaut", "url.path": "/ess/quiad.jpg", - "url.query": "ten=litanim", + "url.query": [ + "emp", + "ten=litanim" + ], "url.scheme": "https", "user.name": "amqu" }, @@ -2176,8 +2221,8 @@ "bore5546.www.local" ], "related.ip": [ - "10.44.198.184", - "10.189.82.19" + "10.189.82.19", + "10.44.198.184" ], "related.user": [ "iconsequ" @@ -2237,7 +2282,10 @@ "url.fragment": "Finibus", "url.original": "https://example.com/xeac/llitanim.txt?oreverit=scip#Finibus", "url.path": "/xeac/llitanim.txt", - "url.query": "oreverit=scip", + "url.query": [ + "atura", + "oreverit=scip" + ], "url.scheme": "https", "user.name": "iconsequ" }, @@ -2335,7 +2383,10 @@ "url.fragment": "iameaque", "url.original": "https://api.example.org/orio/gna.gif?aaliquaU=olu#iameaque", "url.path": "/orio/gna.gif", - "url.query": "aaliquaU=olu", + "url.query": [ + "tatio", + "aaliquaU=olu" + ], "url.scheme": "https", "user.name": "iosamni" }, @@ -2383,8 +2434,8 @@ "rsa.investigations.event_vcat": "uatu", "rsa.misc.OS": "tnulapar", "rsa.misc.action": [ - "deny", - "odic" + "odic", + "deny" ], "rsa.misc.category": "deri", "rsa.misc.client": "scivelit", @@ -2433,7 +2484,10 @@ "url.fragment": "umtota", "url.original": "https://mail.example.org/nimadmin/lumqui.txt?iquip=tinculpa#umtota", "url.path": "/nimadmin/lumqui.txt", - "url.query": "iquip=tinculpa", + "url.query": [ + "iuta", + "iquip=tinculpa" + ], "url.scheme": "https", "user.name": "eavolup" }, @@ -2466,9 +2520,9 @@ "uio" ], "related.ip": [ + "10.111.182.212", "10.37.161.101", - "10.17.209.252", - "10.111.182.212" + "10.17.209.252" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2533,9 +2587,9 @@ "itautfu" ], "related.ip": [ + "10.170.196.181", "10.158.175.98", - "10.153.166.133", - "10.170.196.181" + "10.153.166.133" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2665,7 +2719,10 @@ "url.fragment": "quisn", "url.original": "https://internal.example.com/temse/caecat.jpg?emeu=tatemac#quisn", "url.path": "/temse/caecat.jpg", - "url.query": "emeu=tatemac", + "url.query": [ + "mveniam", + "emeu=tatemac" + ], "url.scheme": "https", "user.name": "usanti" }, @@ -2698,8 +2755,8 @@ "iam" ], "related.ip": [ - "10.77.105.81", "10.174.17.46", + "10.77.105.81", "10.38.168.190" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -2765,8 +2822,8 @@ "ons" ], "related.ip": [ - "10.225.37.73", "10.166.142.198", + "10.225.37.73", "10.36.99.207" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -2832,9 +2889,9 @@ "eturadip" ], "related.ip": [ - "10.145.194.12", "10.66.90.225", - "10.214.156.161" + "10.214.156.161", + "10.145.194.12" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3031,7 +3088,10 @@ "url.fragment": "isni", "url.original": "https://www5.example.com/elit/sam.htm?nevolu=unt#isni", "url.path": "/elit/sam.htm", - "url.query": "nevolu=unt", + "url.query": [ + "onoru", + "nevolu=unt" + ], "url.scheme": "https", "user.name": "estiaec" }, @@ -3068,8 +3128,8 @@ "itaspe3216.localdomain" ], "related.ip": [ - "10.116.82.108", - "10.94.177.125" + "10.94.177.125", + "10.116.82.108" ], "related.user": [ "ecatc" @@ -3129,7 +3189,10 @@ "url.fragment": "iadeseru", "url.original": "https://example.org/porro/issu.htm?inculpa=ruredol#iadeseru", "url.path": "/porro/issu.htm", - "url.query": "inculpa=ruredol", + "url.query": [ + "olorsit", + "inculpa=ruredol" + ], "url.scheme": "https", "user.name": "ecatc" }, @@ -3228,7 +3291,10 @@ "url.fragment": "suntexpl", "url.original": "https://mail.example.org/uamquaer/texplica.gif?sequa=lorum#suntexpl", "url.path": "/uamquaer/texplica.gif", - "url.query": "sequa=lorum", + "url.query": [ + "udexerci", + "sequa=lorum" + ], "url.scheme": "https", "user.name": "norumetM" }, @@ -3326,7 +3392,10 @@ "url.fragment": "rsit", "url.original": "https://www5.example.org/aqu/utemvele.gif?serrorsi=tsedquia#rsit", "url.path": "/aqu/utemvele.gif", - "url.query": "serrorsi=tsedquia", + "url.query": [ + "nihilmol", + "serrorsi=tsedquia" + ], "url.scheme": "https", "user.name": "dictasun" }, @@ -3359,9 +3428,9 @@ "lors" ], "related.ip": [ - "10.205.83.138", "10.99.55.115", - "10.98.52.184" + "10.98.52.184", + "10.205.83.138" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3426,8 +3495,8 @@ "reprehe" ], "related.ip": [ - "10.197.128.162", "10.90.189.248", + "10.197.128.162", "10.228.11.50" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -3605,7 +3674,10 @@ "url.fragment": "adeserun", "url.original": "https://www5.example.com/etcon/chit.txt?erspici=itinvolu#adeserun", "url.path": "/etcon/chit.txt", - "url.query": "erspici=itinvolu", + "url.query": [ + "tinvolu", + "erspici=itinvolu" + ], "url.scheme": "https", "user.name": "tcons" }, @@ -3653,8 +3725,8 @@ "rsa.investigations.event_vcat": "iae", "rsa.misc.OS": "evelite", "rsa.misc.action": [ - "block", - "essequam" + "essequam", + "block" ], "rsa.misc.category": "tmollita", "rsa.misc.client": "uiinea", @@ -3703,7 +3775,10 @@ "url.fragment": "nseq", "url.original": "https://api.example.org/fug/liquid.txt?ptate=lloi#nseq", "url.path": "/fug/liquid.txt", - "url.query": "ptate=lloi", + "url.query": [ + "undeo", + "ptate=lloi" + ], "url.scheme": "https", "user.name": "eumiure" }, @@ -3736,8 +3811,8 @@ "teni" ], "related.ip": [ - "10.200.12.126", "10.250.231.196", + "10.200.12.126", "10.14.145.107" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -3803,9 +3878,9 @@ "atuse" ], "related.ip": [ + "10.103.36.192", "10.21.203.112", - "10.225.34.176", - "10.103.36.192" + "10.225.34.176" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3871,8 +3946,8 @@ ], "related.ip": [ "10.140.59.161", - "10.118.111.183", - "10.5.67.140" + "10.5.67.140", + "10.118.111.183" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4002,7 +4077,10 @@ "url.fragment": "litess", "url.original": "https://example.net/aturQui/tquii.html?uiac=squ#litess", "url.path": "/aturQui/tquii.html", - "url.query": "uiac=squ", + "url.query": [ + "Loremip", + "uiac=squ" + ], "url.scheme": "https", "user.name": "itsed" }, @@ -4034,8 +4112,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.6506", "related.hosts": [ - "ecillum", "sedqui", + "ecillum", "reprehe3525.www5.example" ], "related.ip": [ @@ -4100,7 +4178,10 @@ "url.fragment": "aturvel", "url.original": "https://mail.example.org/mvele/teveli.htm?Nequepor=luptate#aturvel", "url.path": "/mvele/teveli.htm", - "url.query": "Nequepor=luptate", + "url.query": [ + "lab", + "Nequepor=luptate" + ], "url.scheme": "https", "user.name": "rporis" }, @@ -4133,9 +4214,9 @@ "sitv" ], "related.ip": [ - "10.217.145.137", + "10.22.149.132", "10.251.183.113", - "10.22.149.132" + "10.217.145.137" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4201,8 +4282,8 @@ ], "related.ip": [ "10.51.60.203", - "10.203.66.175", - "10.183.16.252" + "10.183.16.252", + "10.203.66.175" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4332,7 +4413,10 @@ "url.fragment": "etc", "url.original": "https://example.org/tem/iadeseru.jpg?olorsita=odoco#etc", "url.path": "/tem/iadeseru.jpg", - "url.query": "olorsita=odoco", + "url.query": [ + "natu", + "olorsita=odoco" + ], "url.scheme": "https", "user.name": "nimadmi" }, @@ -4430,7 +4514,10 @@ "url.fragment": "tobe", "url.original": "https://mail.example.net/epteurs/usmodtem.gif?itvo=asi#tobe", "url.path": "/epteurs/usmodtem.gif", - "url.query": "itvo=asi", + "url.query": [ + "equepor", + "itvo=asi" + ], "url.scheme": "https", "user.name": "nimadmin" }, @@ -4528,7 +4615,10 @@ "url.fragment": "eav", "url.original": "https://www.example.com/iadolo/cidu.txt?aliquide=redolori#eav", "url.path": "/iadolo/cidu.txt", - "url.query": "aliquide=redolori", + "url.query": [ + "tiu", + "aliquide=redolori" + ], "url.scheme": "https", "user.name": "msequin" }, @@ -4565,8 +4655,8 @@ "ntex5135.corp" ], "related.ip": [ - "10.234.171.117", - "10.239.194.105" + "10.239.194.105", + "10.234.171.117" ], "related.user": [ "tat" @@ -4576,8 +4666,8 @@ "rsa.investigations.event_vcat": "uia", "rsa.misc.OS": "mquae", "rsa.misc.action": [ - "tenatus", - "deny" + "deny", + "tenatus" ], "rsa.misc.category": "abo", "rsa.misc.client": "umtota", @@ -4626,7 +4716,10 @@ "url.fragment": "boreetd", "url.original": "https://internal.example.com/isc/umdol.jpg?atn=sectet#boreetd", "url.path": "/isc/umdol.jpg", - "url.query": "atn=sectet", + "url.query": [ + "odic", + "atn=sectet" + ], "url.scheme": "https", "user.name": "tat" }, @@ -4660,8 +4753,8 @@ "nonnumq" ], "related.ip": [ - "10.34.41.75", "10.107.168.208", + "10.34.41.75", "10.249.16.201" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4731,8 +4824,8 @@ "tat1845.internal.invalid" ], "related.ip": [ - "10.96.168.24", - "10.109.106.194" + "10.109.106.194", + "10.96.168.24" ], "related.user": [ "ommodoc" @@ -4792,7 +4885,10 @@ "url.fragment": "ptat", "url.original": "https://www.example.com/imadm/ugiat.txt?Nequepor=nisiu#ptat", "url.path": "/imadm/ugiat.txt", - "url.query": "Nequepor=nisiu", + "url.query": [ + "stiaec", + "Nequepor=nisiu" + ], "url.scheme": "https", "user.name": "ommodoc" }, @@ -4890,7 +4986,10 @@ "url.fragment": "sed", "url.original": "https://mail.example.net/ionemu/nul.jpg?volupt=ori#sed", "url.path": "/ionemu/nul.jpg", - "url.query": "volupt=ori", + "url.query": [ + "radipi", + "volupt=ori" + ], "url.scheme": "https", "user.name": "aed" }, @@ -4991,8 +5090,8 @@ ], "related.ip": [ "10.77.105.160", - "10.90.229.92", - "10.251.212.166" + "10.251.212.166", + "10.90.229.92" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5122,7 +5221,10 @@ "url.fragment": "henderi", "url.original": "https://mail.example.org/asuntex/uovolup.html?amali=uiav#henderi", "url.path": "/asuntex/uovolup.html", - "url.query": "amali=uiav", + "url.query": [ + "eriti", + "amali=uiav" + ], "url.scheme": "https", "user.name": "urmag" }, @@ -5170,8 +5272,8 @@ "rsa.investigations.event_vcat": "uatDuisa", "rsa.misc.OS": "citation", "rsa.misc.action": [ - "utlabore", - "accept" + "accept", + "utlabore" ], "rsa.misc.category": "reeu", "rsa.misc.client": "ntut", @@ -5220,7 +5322,10 @@ "url.fragment": "dminimve", "url.original": "https://example.org/mnisiut/porinci.htm?norum=emUten#dminimve", "url.path": "/mnisiut/porinci.htm", - "url.query": "norum=emUten", + "url.query": [ + "texplica", + "norum=emUten" + ], "url.scheme": "https", "user.name": "lpaquiof" }, @@ -5253,8 +5358,8 @@ "cons" ], "related.ip": [ - "10.246.41.77", "10.228.61.5", + "10.246.41.77", "10.157.22.21" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5391,8 +5496,8 @@ "tru3812.mail.lan" ], "related.ip": [ - "10.106.101.87", - "10.247.124.74" + "10.247.124.74", + "10.106.101.87" ], "related.user": [ "ainci" @@ -5452,7 +5557,10 @@ "url.fragment": "rai", "url.original": "https://api.example.net/nsec/smo.gif?etq=trumexe#rai", "url.path": "/nsec/smo.gif", - "url.query": "etq=trumexe", + "url.query": [ + "ipsu", + "etq=trumexe" + ], "url.scheme": "https", "user.name": "ainci" }, @@ -5597,7 +5705,10 @@ "url.fragment": "upta", "url.original": "https://internal.example.com/dolo/exeacom.txt?tlab=eufugiat#upta", "url.path": "/dolo/exeacom.txt", - "url.query": "tlab=eufugiat", + "url.query": [ + "Lor", + "tlab=eufugiat" + ], "url.scheme": "https", "user.name": "erspici" }, @@ -5695,7 +5806,10 @@ "url.fragment": "utod", "url.original": "https://www5.example.org/orumet/aliqu.txt?tion=sun#utod", "url.path": "/orumet/aliqu.txt", - "url.query": "tion=sun", + "url.query": [ + "con", + "tion=sun" + ], "url.scheme": "https", "user.name": "Sedutp" }, @@ -5796,8 +5910,9 @@ ], "related.ip": [ "10.7.230.206", - "10.249.93.150", - "10.154.151.111" + "10.154.151.111", + "10.7.230.206", + "10.249.93.150" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5927,7 +6042,10 @@ "url.fragment": "iscin", "url.original": "https://internal.example.com/nonnumqu/isciveli.gif?wri=aute#iscin", "url.path": "/nonnumqu/isciveli.gif", - "url.query": "wri=aute", + "url.query": [ + "eca", + "wri=aute" + ], "url.scheme": "https", "user.name": "ciduntut" }, @@ -5959,8 +6077,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.6603", "related.hosts": [ - "ssuscipi", "eac", + "ssuscipi", "dquiac6194.api.lan" ], "related.ip": [ @@ -6025,7 +6143,10 @@ "url.fragment": "upta", "url.original": "https://www5.example.org/utlabo/scip.html?voluptas=inv#upta", "url.path": "/utlabo/scip.html", - "url.query": "voluptas=inv", + "url.query": [ + "atnu", + "voluptas=inv" + ], "url.scheme": "https", "user.name": "nulapar" }, @@ -6123,7 +6244,10 @@ "url.fragment": "obeata", "url.original": "https://internal.example.com/ssusci/snostrud.txt?dolo=siutaliq#obeata", "url.path": "/ssusci/snostrud.txt", - "url.query": "dolo=siutaliq", + "url.query": [ + "ptat", + "dolo=siutaliq" + ], "url.scheme": "https", "user.name": "moenimi" }, @@ -6221,7 +6345,10 @@ "url.fragment": "ptasn", "url.original": "https://api.example.com/temporai/umw.jpg?mveniamq=litsed#ptasn", "url.path": "/temporai/umw.jpg", - "url.query": "mveniamq=litsed", + "url.query": [ + "lore", + "mveniamq=litsed" + ], "url.scheme": "https", "user.name": "idolo" }, @@ -6254,9 +6381,9 @@ "eleumiu" ], "related.ip": [ - "10.120.212.78", "10.236.211.111", - "10.221.100.157" + "10.221.100.157", + "10.120.212.78" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6386,7 +6513,10 @@ "url.fragment": "umiurere", "url.original": "https://mail.example.org/ngelit/dipiscin.gif?serro=ctet#umiurere", "url.path": "/ngelit/dipiscin.gif", - "url.query": "serro=ctet", + "url.query": [ + "diconseq", + "serro=ctet" + ], "url.scheme": "https", "user.name": "riatur" }, @@ -6552,8 +6682,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.95", "related.hosts": [ - "inculp", "Utenimad", + "inculp", "emveleu4029.api.local" ], "related.ip": [ @@ -6618,7 +6748,10 @@ "url.fragment": "lestia", "url.original": "https://internal.example.net/inc/riaturEx.htm?mnihilm=itinvo#lestia", "url.path": "/inc/riaturEx.htm", - "url.query": "mnihilm=itinvo", + "url.query": [ + "icaboN", + "mnihilm=itinvo" + ], "url.scheme": "https", "user.name": "udantiu" }, @@ -6719,8 +6852,8 @@ ], "related.ip": [ "10.74.88.209", - "10.92.3.166", - "10.238.49.73" + "10.238.49.73", + "10.92.3.166" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6785,9 +6918,9 @@ "ptate" ], "related.ip": [ + "10.84.200.121", "10.119.248.36", - "10.187.107.47", - "10.84.200.121" + "10.187.107.47" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6852,9 +6985,9 @@ "tasu" ], "related.ip": [ - "10.135.213.17", "10.30.239.222", - "10.167.128.229" + "10.167.128.229", + "10.135.213.17" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6934,8 +7067,8 @@ "rsa.investigations.event_vcat": "iumdol", "rsa.misc.OS": "min", "rsa.misc.action": [ - "block", - "eleumiur" + "eleumiur", + "block" ], "rsa.misc.category": "ero", "rsa.misc.client": "gia", @@ -6984,7 +7117,10 @@ "url.fragment": "rExce", "url.original": "https://api.example.net/ita/esse.txt?amquis=iatquovo#rExce", "url.path": "/ita/esse.txt", - "url.query": "amquis=iatquovo", + "url.query": [ + "ididun", + "amquis=iatquovo" + ], "url.scheme": "https", "user.name": "emq" }, @@ -7016,13 +7152,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.4195", "related.hosts": [ - "aconse", "Except", + "aconse", "rror3870.www5.local" ], "related.ip": [ - "10.146.255.40", - "10.226.39.82" + "10.226.39.82", + "10.146.255.40" ], "related.user": [ "caecatcu" @@ -7082,7 +7218,10 @@ "url.fragment": "rat", "url.original": "https://www5.example.net/culpa/isun.txt?cola=tura#rat", "url.path": "/culpa/isun.txt", - "url.query": "cola=tura", + "url.query": [ + "orumwrit", + "cola=tura" + ], "url.scheme": "https", "user.name": "caecatcu" }, @@ -7115,9 +7254,9 @@ "emaperi" ], "related.ip": [ - "10.224.212.88", "10.35.240.70", - "10.53.82.96" + "10.53.82.96", + "10.224.212.88" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7183,8 +7322,8 @@ ], "related.ip": [ "10.186.253.240", - "10.233.128.7", - "10.66.149.234" + "10.66.149.234", + "10.233.128.7" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7316,8 +7455,8 @@ "emp" ], "related.ip": [ - "10.69.130.207", "10.205.18.11", + "10.69.130.207", "10.170.236.123" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7382,8 +7521,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2682", "related.hosts": [ - "rehend", "ine", + "rehend", "velill3821.mail.invalid" ], "related.ip": [ @@ -7448,7 +7587,10 @@ "url.fragment": "aborios", "url.original": "https://example.com/loremag/mqu.gif?bore=lapari#aborios", "url.path": "/loremag/mqu.gif", - "url.query": "bore=lapari", + "url.query": [ + "sequa", + "bore=lapari" + ], "url.scheme": "https", "user.name": "epor" }, @@ -7481,9 +7623,9 @@ "ineavol" ], "related.ip": [ - "10.204.98.238", + "10.9.41.221", "10.81.58.91", - "10.9.41.221" + "10.204.98.238" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7662,9 +7804,9 @@ "aturvel" ], "related.ip": [ - "10.207.207.106", + "10.199.201.26", "10.143.65.84", - "10.199.201.26" + "10.207.207.106" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7729,8 +7871,8 @@ "gni" ], "related.ip": [ - "10.163.236.253", "10.204.27.48", + "10.163.236.253", "10.41.61.88" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7796,9 +7938,9 @@ "dents" ], "related.ip": [ - "10.185.44.26", "10.246.81.164", - "10.53.110.111" + "10.53.110.111", + "10.185.44.26" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7929,7 +8071,10 @@ "url.fragment": "ido", "url.original": "https://api.example.com/aco/empo.jpg?iumdol=iusm#ido", "url.path": "/aco/empo.jpg", - "url.query": "iumdol=iusm", + "url.query": [ + "ipsamv", + "iumdol=iusm" + ], "url.scheme": "https", "user.name": "aquaeab" }, @@ -8027,7 +8172,10 @@ "url.fragment": "ect", "url.original": "https://example.org/uidolor/turve.htm?temporai=uasiarch#ect", "url.path": "/uidolor/turve.htm", - "url.query": "temporai=uasiarch", + "url.query": [ + "quatDui", + "temporai=uasiarch" + ], "url.scheme": "https", "user.name": "maperia" }, @@ -8125,7 +8273,10 @@ "url.fragment": "odtempo", "url.original": "https://www.example.net/Maloru/lapariat.htm?tlabori=rehender#odtempo", "url.path": "/Maloru/lapariat.htm", - "url.query": "tlabori=rehender", + "url.query": [ + "isiutali", + "tlabori=rehender" + ], "url.scheme": "https", "user.name": "modo" }, @@ -8162,8 +8313,8 @@ "neavo4796.internal.domain" ], "related.ip": [ - "10.188.124.185", - "10.35.10.19" + "10.35.10.19", + "10.188.124.185" ], "related.user": [ "dolo" @@ -8173,8 +8324,8 @@ "rsa.investigations.event_vcat": "olupt", "rsa.misc.OS": "rumw", "rsa.misc.action": [ - "block", - "tali" + "tali", + "block" ], "rsa.misc.category": "itsedq", "rsa.misc.client": "esciu", @@ -8223,7 +8374,10 @@ "url.fragment": "one", "url.original": "https://www5.example.com/xea/ssecill.html?quianonn=quun#one", "url.path": "/xea/ssecill.html", - "url.query": "quianonn=quun", + "url.query": [ + "tasper", + "quianonn=quun" + ], "url.scheme": "https", "user.name": "dolo" } diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index aab537751715..18e4409c4245 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -10,6 +10,37 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json index 749ef4301f2f..ad6ac605be41 100644 --- a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json +++ b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json @@ -247,8 +247,8 @@ "rsa.internal.event_desc": "snostrud", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "cancel", - "dol" + "dol", + "cancel" ], "rsa.misc.category": "nama", "rsa.misc.disposition": "quisnos", @@ -273,8 +273,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "eumfug", "url.original": "https://internal.example.net/namali/taevit.html?nsecte=itame#eumfug", - "url.query": "lit", + "url.path": "/namali/taevit.html", + "url.query": [ + "lit", + "nsecte=itame" + ], + "url.scheme": "https", "user.name": "occae" }, { @@ -342,8 +350,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "giatnu", "url.original": "https://example.net/tlabo/uames.gif?mpo=offi#giatnu", - "url.query": "ulapa", + "url.path": "/tlabo/uames.gif", + "url.query": [ + "ulapa", + "mpo=offi" + ], + "url.scheme": "https", "user.name": "ncidid" }, { @@ -375,8 +391,8 @@ ], "related.user": [ "labor", - "eveli", - "orema" + "orema", + "eveli" ], "rsa.counters.dclass_c1": 6855, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -432,8 +448,8 @@ "pora6854.www5.home" ], "related.ip": [ - "10.214.191.180", - "10.112.250.193" + "10.112.250.193", + "10.214.191.180" ], "related.user": [ "ipsumdol", @@ -498,8 +514,8 @@ ], "related.user": [ "ovol", - "iquipe", - "tnonpro" + "tnonpro", + "iquipe" ], "rsa.counters.dclass_c1": 3645, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -556,8 +572,8 @@ ], "related.user": [ "idunt", - "archite", - "boree" + "boree", + "archite" ], "rsa.counters.dclass_c1": 248, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -679,9 +695,9 @@ "10.49.167.57" ], "related.user": [ - "sau", + "ccaeca", "tali", - "ccaeca" + "sau" ], "rsa.counters.dclass_c1": 6818, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -743,8 +759,8 @@ "10.216.125.252" ], "related.user": [ - "llamco", "dolore", + "llamco", "lorsita" ], "rsa.counters.event_counter": 4603, @@ -779,8 +795,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "elit", "url.original": "https://example.org/umwrit/uptate.html?ctetura=aveni#elit", - "url.query": "seosqui", + "url.path": "/umwrit/uptate.html", + "url.query": [ + "seosqui", + "ctetura=aveni" + ], + "url.scheme": "https", "user.name": "llamco" }, { @@ -848,8 +872,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "mvele", "url.original": "https://api.example.org/roidents/tem.txt?tametcon=liqua#mvele", - "url.query": "isis", + "url.path": "/roidents/tem.txt", + "url.query": [ + "isis", + "tametcon=liqua" + ], + "url.scheme": "https", "user.name": "rum" }, { @@ -879,8 +911,8 @@ "10.34.148.166" ], "related.user": [ - "icabo", "untutlab", + "icabo", "miu" ], "rsa.counters.dclass_c1": 5427, @@ -937,9 +969,9 @@ "10.134.5.40" ], "related.user": [ - "licabo", + "siu", "conse", - "siu" + "licabo" ], "rsa.counters.dclass_c1": 6356, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1153,7 +1185,8 @@ "related.user": [ "proident", "boru", - "ritati" + "ritati", + "proident" ], "rsa.counters.dclass_c1": 5923, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1275,8 +1308,8 @@ "10.167.252.183" ], "related.user": [ - "redol", "asnu", + "redol", "ationul" ], "rsa.counters.dclass_c1": 6606, @@ -1339,8 +1372,8 @@ "10.81.184.7" ], "related.user": [ - "lmole", "undeomni", + "lmole", "iameaque" ], "rsa.counters.event_counter": 6344, @@ -1375,8 +1408,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "tetur", "url.original": "https://mail.example.com/lorsi/repreh.gif?sitamet=utlabo#tetur", - "url.query": "tionula", + "url.path": "/lorsi/repreh.gif", + "url.query": [ + "tionula", + "sitamet=utlabo" + ], + "url.scheme": "https", "user.name": "undeomni" }, { @@ -1507,8 +1548,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "idolo", "url.original": "https://example.org/etconsec/elillum.htm?mporinc=onsectet#idolo", - "url.query": "atemUte", + "url.path": "/etconsec/elillum.htm", + "url.query": [ + "atemUte", + "mporinc=onsectet" + ], + "url.scheme": "https", "user.name": "pta" }, { @@ -1541,6 +1590,7 @@ "related.user": [ "doeiu", "mquisn", + "doeiu", "litan" ], "rsa.counters.dclass_c1": 3474, @@ -1638,8 +1688,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "remipsu", "url.original": "https://internal.example.net/atnula/ditautf.jpg?iquidex=olup#remipsu", - "url.query": "tan", + "url.path": "/atnula/ditautf.jpg", + "url.query": [ + "tan", + "iquidex=olup" + ], + "url.scheme": "https", "user.name": "emUte" }, { @@ -1672,6 +1730,7 @@ "related.user": [ "eFini", "urau", + "eFini", "eufug" ], "rsa.counters.dclass_c1": 3348, @@ -1856,8 +1915,8 @@ "10.165.182.111" ], "related.user": [ - "ames", "sis", + "ames", "Bonorum" ], "rsa.counters.dclass_c1": 6401, @@ -1984,8 +2043,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "madmi", "url.original": "https://www.example.net/hender/ptatemU.htm?mquisnos=tnulapa#madmi", - "url.query": "tlabore", + "url.path": "/hender/ptatemU.htm", + "url.query": [ + "tlabore", + "mquisnos=tnulapa" + ], + "url.scheme": "https", "user.name": "uian" }, { @@ -2012,8 +2079,8 @@ "turQuis4046.api.test" ], "related.ip": [ - "10.90.50.149", - "10.168.225.209" + "10.168.225.209", + "10.90.50.149" ], "related.user": [ "olu", @@ -2079,8 +2146,8 @@ ], "related.user": [ "luptat", - "qua", - "mtota" + "mtota", + "qua" ], "rsa.counters.dclass_c1": 6112, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2325,8 +2392,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "temqui", "url.original": "https://example.net/itati/oidentsu.gif?eporroqu=aturve#temqui", - "url.query": "lup", + "url.path": "/itati/oidentsu.gif", + "url.query": [ + "lup", + "eporroqu=aturve" + ], + "url.scheme": "https", "user.name": "ttenb" }, { @@ -2387,8 +2462,8 @@ "10.116.1.130" ], "related.user": [ - "eturadip", "reseo", + "eturadip", "amco" ], "rsa.counters.event_counter": 1295, @@ -2423,8 +2498,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "pteurs", "url.original": "https://api.example.org/uamqua/Neq.gif?eumiu=nim#pteurs", - "url.query": "ercitati", + "url.path": "/uamqua/Neq.gif", + "url.query": [ + "ercitati", + "eumiu=nim" + ], + "url.scheme": "https", "user.name": "eturadip" }, { @@ -2517,9 +2600,9 @@ "10.152.213.228" ], "related.user": [ - "itationu", + "ptatev", "velillum", - "ptatev" + "itationu" ], "rsa.counters.dclass_c1": 7245, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2607,9 +2690,9 @@ "10.208.33.55" ], "related.user": [ - "inimv", + "ulapari", "mremaper", - "ulapari" + "inimv" ], "rsa.counters.dclass_c1": 6433, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2789,13 +2872,13 @@ "tat50.mail.host" ], "related.ip": [ - "10.45.152.205", - "10.224.217.153" + "10.224.217.153", + "10.45.152.205" ], "related.user": [ "eriti", - "imav", - "utlabo" + "utlabo", + "imav" ], "rsa.counters.dclass_c1": 922, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2892,8 +2975,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "emipsumq", "url.original": "https://api.example.net/quam/saute.htm?nostru=docons#emipsumq", - "url.query": "orinr", + "url.path": "/quam/saute.htm", + "url.query": [ + "orinr", + "nostru=docons" + ], + "url.scheme": "https", "user.name": "ugi" }, { @@ -2981,9 +3072,9 @@ "10.86.121.152" ], "related.user": [ + "nimv", "ine", - "consecte", - "nimv" + "consecte" ], "rsa.counters.dclass_c1": 2771, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3043,9 +3134,9 @@ "10.204.223.184" ], "related.user": [ - "teni", + "tuserror", "rcit", - "tuserror" + "teni" ], "rsa.counters.dclass_c1": 4113, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3105,9 +3196,9 @@ "10.223.56.33" ], "related.user": [ - "elitsedd", "magnido", - "Nequepo" + "Nequepo", + "elitsedd" ], "rsa.counters.dclass_c1": 3243, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3178,8 +3269,8 @@ "rsa.internal.event_desc": "atuse", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "cancel", - "eruntmol" + "eruntmol", + "cancel" ], "rsa.misc.category": "imad", "rsa.misc.disposition": "tura", @@ -3205,8 +3296,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "asia", "url.original": "https://internal.example.com/imide/uiineav.htm?lloinve=eni#asia", - "url.query": "edquiac", + "url.path": "/imide/uiineav.htm", + "url.query": [ + "edquiac", + "lloinve=eni" + ], + "url.scheme": "https", "user.name": "tuserror" }, { @@ -3236,9 +3335,9 @@ "10.65.174.196" ], "related.user": [ + "iin", "tione", - "uta", - "iin" + "uta" ], "rsa.counters.dclass_c1": 5836, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3296,9 +3395,9 @@ "10.224.148.48" ], "related.user": [ - "niam", "iosamn", - "equepor" + "equepor", + "niam" ], "rsa.counters.event_counter": 7468, "rsa.db.database": "erspicia", @@ -3332,8 +3431,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iumd", "url.original": "https://api.example.com/ptatem/mporain.gif?corpo=commod#iumd", - "url.query": "ntore", + "url.path": "/ptatem/mporain.gif", + "url.query": [ + "ntore", + "corpo=commod" + ], + "url.scheme": "https", "user.name": "niam" }, { @@ -3427,8 +3534,8 @@ ], "related.user": [ "tevelite", - "iarchit", - "iamquisn" + "iamquisn", + "iarchit" ], "rsa.counters.dclass_c1": 639, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3491,8 +3598,8 @@ ], "related.user": [ "animide", - "modtempo", - "nofde" + "nofde", + "modtempo" ], "rsa.counters.event_counter": 7580, "rsa.db.database": "Lore", @@ -3525,8 +3632,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "onsequa", "url.original": "https://www5.example.net/aUten/edutpers.gif?apariatu=mnisis#onsequa", - "url.query": "sunt", + "url.path": "/aUten/edutpers.gif", + "url.query": [ + "sunt", + "apariatu=mnisis" + ], + "url.scheme": "https", "user.name": "modtempo" }, { @@ -3555,8 +3670,8 @@ "mquis319.api.local" ], "related.ip": [ - "10.111.22.134", - "10.178.79.217" + "10.178.79.217", + "10.111.22.134" ], "related.user": [ "tqui", @@ -3594,8 +3709,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "iaeco", "url.original": "https://www.example.net/mvolup/pidat.jpg?ents=nsec#iaeco", - "url.query": "ommodoco", + "url.path": "/mvolup/pidat.jpg", + "url.query": [ + "ommodoco", + "ents=nsec" + ], + "url.scheme": "https", "user.name": "ccusan" }, { @@ -3687,9 +3810,9 @@ "10.186.133.184" ], "related.user": [ - "boriosa", + "acons", "sci", - "acons" + "boriosa" ], "rsa.counters.dclass_c1": 1578, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3864,9 +3987,9 @@ "10.108.130.106" ], "related.user": [ - "uisautei", + "colab", "exeacomm", - "colab" + "uisautei" ], "rsa.counters.dclass_c1": 1044, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3963,8 +4086,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "rautod", "url.original": "https://internal.example.org/aev/uovolup.txt?aqueip=aqueip#rautod", - "url.query": "tur", + "url.path": "/aev/uovolup.txt", + "url.query": [ + "tur", + "aqueip=aqueip" + ], + "url.scheme": "https", "user.name": "Sedut" }, { @@ -4047,8 +4178,8 @@ "qui5978.api.test" ], "related.ip": [ - "10.115.203.143", - "10.134.135.22" + "10.134.135.22", + "10.115.203.143" ], "related.user": [ "orpori", @@ -4115,7 +4246,9 @@ "related.user": [ "inculp", "gnido", - "uptat" + "uptat", + "gnido", + "inculp" ], "rsa.counters.dclass_c1": 6947, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4205,6 +4338,7 @@ "related.user": [ "mqu", "tesseq", + "mqu", "uatDuisa" ], "rsa.counters.dclass_c1": 1623, @@ -4353,9 +4487,9 @@ "10.106.166.105" ], "related.user": [ - "avolup", "usa", - "olupt" + "olupt", + "avolup" ], "rsa.counters.dclass_c1": 2658, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4473,9 +4607,9 @@ "10.129.234.200" ], "related.user": [ - "dquia", "tevelit", - "tisundeo" + "tisundeo", + "dquia" ], "rsa.counters.dclass_c1": 6709, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4536,8 +4670,8 @@ ], "related.user": [ "scive", - "ali", - "oloremi" + "oloremi", + "ali" ], "rsa.counters.dclass_c1": 6155, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4660,8 +4794,8 @@ ], "related.user": [ "apar", - "isn", - "ptasn" + "ptasn", + "isn" ], "rsa.counters.dclass_c1": 758, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4717,8 +4851,8 @@ "iatisund424.mail.localdomain" ], "related.ip": [ - "10.42.135.34", - "10.178.190.123" + "10.178.190.123", + "10.42.135.34" ], "related.user": [ "orsi", @@ -4874,6 +5008,7 @@ "related.user": [ "nseq", "litesseq", + "nseq", "oNe" ], "rsa.counters.dclass_c1": 3218, @@ -4930,9 +5065,9 @@ "10.86.180.150" ], "related.user": [ - "itasper", "mnisis", - "etconsec" + "etconsec", + "itasper" ], "rsa.counters.dclass_c1": 4564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4994,8 +5129,8 @@ "10.220.175.201" ], "related.user": [ - "dolo", - "rrors" + "rrors", + "dolo" ], "rsa.counters.event_counter": 4098, "rsa.db.database": "tsed", @@ -5028,8 +5163,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "amnisiu", "url.original": "https://internal.example.com/architec/incul.txt?aborios=mco#amnisiu", - "url.query": "suntincu", + "url.path": "/architec/incul.txt", + "url.query": [ + "suntincu", + "aborios=mco" + ], + "url.scheme": "https", "user.name": "dolo" }, { @@ -5211,9 +5354,9 @@ "10.69.5.227" ], "related.user": [ + "ntocc", "rumw", - "doloreme", - "ntocc" + "doloreme" ], "rsa.counters.dclass_c1": 5201, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5270,8 +5413,8 @@ ], "related.user": [ "ate", - "nrep", - "epteurs" + "epteurs", + "nrep" ], "rsa.counters.dclass_c1": 6260, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5368,8 +5511,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "edut", "url.original": "https://www.example.com/sit/ugi.gif?sitametc=rur#edut", - "url.query": "sitametc", + "url.path": "/sit/ugi.gif", + "url.query": [ + "sitametc", + "sitametc=rur" + ], + "url.scheme": "https", "user.name": "aboris" }, { @@ -5402,9 +5553,9 @@ "10.81.108.232" ], "related.user": [ + "uaturve", "neavolup", - "aco", - "uaturve" + "aco" ], "rsa.counters.event_counter": 5098, "rsa.db.database": "lapa", @@ -5438,8 +5589,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "amrem", "url.original": "https://internal.example.net/obeatae/sedqui.jpg?nulap=onseq#amrem", - "url.query": "plicab", + "url.path": "/obeatae/sedqui.jpg", + "url.query": [ + "plicab", + "nulap=onseq" + ], + "url.scheme": "https", "user.name": "aco" }, { @@ -5472,9 +5631,9 @@ "10.230.48.97" ], "related.user": [ + "usmodte", "untex", - "erit", - "usmodte" + "erit" ], "rsa.counters.event_counter": 4029, "rsa.db.database": "ommodi", @@ -5508,8 +5667,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "itinvo", "url.original": "https://mail.example.com/iaturE/inc.htm?uisaut=mnihilm#itinvo", - "url.query": "lestia", + "url.path": "/iaturE/inc.htm", + "url.query": [ + "lestia", + "uisaut=mnihilm" + ], + "url.scheme": "https", "user.name": "erit" }, { @@ -5540,9 +5707,9 @@ "10.115.42.231" ], "related.user": [ + "res", "sequamn", - "tasnul", - "res" + "tasnul" ], "rsa.counters.dclass_c1": 4846, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5604,9 +5771,9 @@ "10.226.75.20" ], "related.user": [ - "fugia", + "tema", "maccusan", - "tema" + "fugia" ], "rsa.counters.event_counter": 3711, "rsa.db.database": "psa", @@ -5640,8 +5807,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "eip", "url.original": "https://www5.example.org/elaud/temsequ.htm?dolo=iciatisu#eip", - "url.query": "iquaUte", + "url.path": "/elaud/temsequ.htm", + "url.query": [ + "iquaUte", + "dolo=iciatisu" + ], + "url.scheme": "https", "user.name": "tema" }, { @@ -5766,8 +5941,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "ema", "url.original": "https://mail.example.net/teturad/nimide.htm?ueporroq=writ#ema", - "url.query": "ioffici", + "url.path": "/teturad/nimide.htm", + "url.query": [ + "ioffici", + "ueporroq=writ" + ], + "url.scheme": "https", "user.name": "ide" }, { @@ -5794,8 +5977,8 @@ "colabo6686.internal.invalid" ], "related.ip": [ - "10.28.77.79", - "10.144.14.15" + "10.144.14.15", + "10.28.77.79" ], "related.user": [ "rspic", @@ -5889,4 +6072,4 @@ ], "user.name": "quei" } -] \ No newline at end of file +] diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 8a25a657a0a9..12ba933ea0ec 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -10,6 +10,37 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json index 7e500a270946..2937ed39d2df 100644 --- a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json +++ b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json @@ -323,7 +323,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.net/uam/untutl.jpg?llu=uptassi#tamremap" + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "tamremap", + "url.original": "https://mail.example.net/uam/untutl.jpg?llu=uptassi#tamremap", + "url.path": "/uam/untutl.jpg", + "url.query": "llu=uptassi", + "url.scheme": "https" }, { "event.action": "Script mitigation", @@ -379,7 +385,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.org/seq/olorema.jpg?quid=fug#uatDuis" + "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "uatDuis", + "url.original": "https://www5.example.org/seq/olorema.jpg?quid=fug#uatDuis", + "url.path": "/seq/olorema.jpg", + "url.query": "quid=fug", + "url.scheme": "https" }, { "event.action": "Fault Occured", @@ -458,7 +470,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://example.org/isiu/nimadmi.gif?ari=equun#suntinc" + "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "suntinc", + "url.original": "https://example.org/isiu/nimadmi.gif?ari=equun#suntinc", + "url.path": "/isiu/nimadmi.gif", + "url.query": "ari=equun", + "url.scheme": "https" }, { "event.code": "Hardware", @@ -848,7 +866,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/nse/sinto.gif?CSed=lupt#psaquae" + "url.domain": "www.example.net", + "url.extension": "gif", + "url.fragment": "psaquae", + "url.original": "https://www.example.net/nse/sinto.gif?CSed=lupt#psaquae", + "url.path": "/nse/sinto.gif", + "url.query": "CSed=lupt", + "url.scheme": "https" }, { "event.code": "BGP", @@ -980,7 +1004,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.net/corp/modtemp.jpg?oluptas=tNequepo#lup" + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "lup", + "url.original": "https://mail.example.net/corp/modtemp.jpg?oluptas=tNequepo#lup", + "url.path": "/corp/modtemp.jpg", + "url.query": "oluptas=tNequepo", + "url.scheme": "https" }, { "event.code": "Change_Log", @@ -1036,7 +1066,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://example.net/temqu/edol.jpg?ipi=reseos#pariatu" + "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "pariatu", + "url.original": "https://example.net/temqu/edol.jpg?ipi=reseos#pariatu", + "url.path": "/temqu/edol.jpg", + "url.query": "ipi=reseos", + "url.scheme": "https" }, { "destination.ip": [ @@ -1161,7 +1197,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.org/atem/gnido.txt?tmollita=fde#nsecte" + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "nsecte", + "url.original": "https://www5.example.org/atem/gnido.txt?tmollita=fde#nsecte", + "url.path": "/atem/gnido.txt", + "url.query": "tmollita=fde", + "url.scheme": "https" }, { "destination.ip": [ @@ -1402,7 +1444,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://api.example.com/eufugi/pici.html?ccaecat=tquiin#tse" + "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "tse", + "url.original": "https://api.example.com/eufugi/pici.html?ccaecat=tquiin#tse", + "url.path": "/eufugi/pici.html", + "url.query": "ccaecat=tquiin", + "url.scheme": "https" }, { "event.code": "Change_Log", @@ -1766,7 +1814,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.org/idunt/luptat.txt?ica=lillum#remips" + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "remips", + "url.original": "https://mail.example.org/idunt/luptat.txt?ica=lillum#remips", + "url.path": "/idunt/luptat.txt", + "url.query": "ica=lillum", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1790,7 +1844,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://internal.example.com/ineavol/iosa.html?usc=rem#amvolupt" + "url.domain": "internal.example.com", + "url.extension": "html", + "url.fragment": "amvolupt", + "url.original": "https://internal.example.com/ineavol/iosa.html?usc=rem#amvolupt", + "url.path": "/ineavol/iosa.html", + "url.query": "usc=rem", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1814,7 +1874,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://internal.example.org/quaUten/nisiut.txt?teturad=perspici#itation" + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "itation", + "url.original": "https://internal.example.org/quaUten/nisiut.txt?teturad=perspici#itation", + "url.path": "/quaUten/nisiut.txt", + "url.query": "teturad=perspici", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1838,7 +1904,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.org/nofd/dipisci.txt?ilmol=eri#quunt" + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "quunt", + "url.original": "https://mail.example.org/nofd/dipisci.txt?ilmol=eri#quunt", + "url.path": "/nofd/dipisci.txt", + "url.query": "ilmol=eri", + "url.scheme": "https" }, { "destination.ip": [ @@ -1870,7 +1942,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.org/ptateve/enderi.html?toccaec=fugi#labo" + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "labo", + "url.original": "https://www.example.org/ptateve/enderi.html?toccaec=fugi#labo", + "url.path": "/ptateve/enderi.html", + "url.query": "toccaec=fugi", + "url.scheme": "https" }, { "destination.ip": [ @@ -1902,7 +1980,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo" + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "umetMalo", + "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo", + "url.path": "/tanimi/rumSecti.jpg", + "url.query": "emporain=ntiumto", + "url.scheme": "https" }, { "event.code": "configuration", @@ -2183,7 +2267,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/itanim/nesciun.txt?mollita=tatem#iae" + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "iae", + "url.original": "https://www.example.net/itanim/nesciun.txt?mollita=tatem#iae", + "url.path": "/itanim/nesciun.txt", + "url.query": "mollita=tatem", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -2207,7 +2297,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/rever/ore.jpg?oluptat=metco#acom" + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "acom", + "url.original": "https://www.example.net/rever/ore.jpg?oluptat=metco#acom", + "url.path": "/rever/ore.jpg", + "url.query": "oluptat=metco", + "url.scheme": "https" }, { "event.code": "Host", @@ -2314,7 +2410,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://api.example.org/eosquira/pta.htm?econs=lmolesti#apariatu" + "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "apariatu", + "url.original": "https://api.example.org/eosquira/pta.htm?econs=lmolesti#apariatu", + "url.path": "/eosquira/pta.htm", + "url.query": "econs=lmolesti", + "url.scheme": "https" }, { "event.action": "Script mitigation", @@ -2362,7 +2464,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.com/atcupida/tessequa.htm?dolores=equamnih#taliqui" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "taliqui", + "url.original": "https://www5.example.com/atcupida/tessequa.htm?dolores=equamnih#taliqui", + "url.path": "/atcupida/tessequa.htm", + "url.query": "dolores=equamnih", + "url.scheme": "https" }, { "event.code": "Change_Log", diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 493713469254..681ae839bbc3 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -7,6 +7,37 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json index 4a6f14974fc4..40660e0d5188 100644 --- a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json @@ -220,7 +220,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "aspx", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -298,7 +302,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "aspx", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -377,7 +385,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -455,7 +467,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -612,7 +628,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -690,7 +710,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -768,7 +792,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -846,7 +874,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json index 75f16b9867e3..710d6ad3a8b0 100644 --- a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json @@ -80,7 +80,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -176,7 +180,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -268,7 +276,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -364,7 +376,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -460,7 +476,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL%20CREDIT%20CARD%20NUMBERS.docx", + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL CREDIT CARD NUMBERS.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL CREDIT CARD NUMBERS.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -556,7 +576,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", + "url.domain": "testsiem2.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared Documents/Document.docx", + "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", @@ -652,7 +676,11 @@ "tags": [ "forwarded" ], - "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", + "url.domain": "testsiem2.sharepoint.com", + "url.extension": "docx", + "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared Documents/Document.docx", + "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json b/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json index 941adb1e1ac6..7996a2d808df 100644 --- a/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json @@ -791,7 +791,11 @@ "tags": [ "forwarded" ], + "url.domain": "example.net", + "url.extension": "com/sharepoint", "url.original": "https://example.net/testsiem2.onmicrosoft.com/sharepoint", + "url.path": "/testsiem2.onmicrosoft.com/sharepoint", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml index 3c41092be69f..bc5967880a50 100644 --- a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml @@ -10,6 +10,37 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json index e0caf6d7abbf..d02fc7982b35 100644 --- a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json @@ -103,7 +103,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "iciatisu", "url.original": "https://mail.example.net/tati/utaliqu.html?iquaUten=santium#iciatisu", + "url.path": "/tati/utaliqu.html", + "url.query": "iquaUten=santium", + "url.scheme": "https", "user.name": "sunt" }, { @@ -638,8 +644,8 @@ "10.54.169.175" ], "related.user": [ - "scipit", - "taspe" + "taspe", + "scipit" ], "rsa.identity.logon_type": "olores", "rsa.internal.event_desc": "secil", @@ -1033,13 +1039,13 @@ "nostrum6305.internal.localhost" ], "related.ip": [ - "10.17.51.153", - "10.89.41.97" + "10.89.41.97", + "10.17.51.153" ], "related.user": [ "tcustatuscode=eumiu", - "tio", - "pteurs" + "pteurs", + "tio" ], "rsa.db.index": "eavolupt", "rsa.identity.logon_type": "ursintoc", @@ -1084,7 +1090,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "ssequamn", "url.original": "https://mail.example.net/velillu/ecatcupi.txt?rsitamet=leumiur#ssequamn", + "url.path": "/velillu/ecatcupi.txt", + "url.query": "rsitamet=leumiur", + "url.scheme": "https", "user.name": "tio" }, { @@ -1237,7 +1249,13 @@ "sophos.utm", "forwarded" ], - "url.original": "https://www5.example.com/dantium/ors.htm?sinto=edi#eumiure" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "eumiure", + "url.original": "https://www5.example.com/dantium/ors.htm?sinto=edi#eumiure", + "url.path": "/dantium/ors.htm", + "url.query": "sinto=edi", + "url.scheme": "https" }, { "@timestamp": "2017-05-29T07:37:24.000Z", @@ -1899,7 +1917,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "enbyCi", "url.original": "https://api.example.org/iono/secillum.txt?apariat=tse#enbyCi", + "url.path": "/iono/secillum.txt", + "url.query": "apariat=tse", + "url.scheme": "https", "user.name": "hen" }, { @@ -2033,10 +2057,10 @@ "10.92.93.236" ], "related.user": [ - "dolorsistatuscode=acc", - "Sedutper", "ntoccae", - "ulpaq" + "Sedutper", + "ulpaq", + "dolorsistatuscode=acc" ], "rsa.db.index": "snisiut", "rsa.identity.logon_type": "umdol", @@ -2082,8 +2106,16 @@ "sophos.utm", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "tru", "url.original": "https://www.example.org/imipsamv/doeiu.jpg?nderit=ficia#tru", - "url.query": "exerc", + "url.path": "/imipsamv/doeiu.jpg", + "url.query": [ + "exerc", + "nderit=ficia" + ], + "url.scheme": "https", "user.name": "Sedutper" }, { @@ -2124,9 +2156,9 @@ "10.180.169.49" ], "related.user": [ - "atatno", + "iscivelistatuscode=urve", "tasu", - "iscivelistatuscode=urve" + "atatno" ], "rsa.db.index": "amrem", "rsa.identity.logon_type": "nulamcol", @@ -2168,7 +2200,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "adm", "url.original": "https://mail.example.net/isqua/deF.html?iameaq=orainci#adm", + "url.path": "/isqua/deF.html", + "url.query": "iameaq=orainci", + "url.scheme": "https", "user.name": "atatno" }, { @@ -2362,7 +2400,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "oditautf", "url.original": "https://www5.example.org/etcons/aecatc.jpg?ditem=tut#oditautf", + "url.path": "/etcons/aecatc.jpg", + "url.query": "ditem=tut", + "url.scheme": "https", "user.name": "eturadip" }, { @@ -2481,8 +2525,8 @@ "10.32.85.21" ], "related.user": [ - "antium", - "etconsec" + "etconsec", + "antium" ], "rsa.identity.logon_type": "umiurere", "rsa.internal.event_desc": "serro", @@ -2632,6 +2676,8 @@ "10.210.175.52" ], "related.user": [ + "reetd", + "inimastatuscode=emipsum", "Loremi", "inimastatuscode=emipsum", "reetd", @@ -2678,8 +2724,16 @@ "sophos.utm", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "agnaaliq", "url.original": "https://internal.example.com/ritat/dipi.jpg?aliquide=aliqui#agnaaliq", - "url.query": "iatquovo", + "url.path": "/ritat/dipi.jpg", + "url.query": [ + "iatquovo", + "aliquide=aliqui" + ], + "url.scheme": "https", "user.name": "reetd" }, { @@ -2707,8 +2761,8 @@ "itametc1599.api.test" ], "related.ip": [ - "10.115.166.48", - "10.133.45.45" + "10.133.45.45", + "10.115.166.48" ], "rsa.internal.event_desc": "Authentication", "rsa.internal.messageid": "ulogd", @@ -3724,4 +3778,4 @@ "forwarded" ] } -] \ No newline at end of file +] diff --git a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml index 845a93c945ed..287892f11902 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml @@ -192,6 +192,33 @@ processors: target_field: url.domain ignore_missing: true if: "ctx.sophos?.xg?.src_domainname != null" +- uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null +- script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } +- remove: + field: _temp_ + ignore_missing: true +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true ############################ ## ECS User Agent Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml index c659264d633e..64c3e52bf03f 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml @@ -120,6 +120,33 @@ processors: target_field: url.original ignore_missing: true if: "ctx.sophos?.xg?.url != null" +- uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null && ctx?.url?.original != ctx?.destination?.ip +- script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } +- remove: + field: _temp_ + ignore_missing: true +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true ############################# ## ECS Network/Geo Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json index 4afefcee9b49..8ab666cc94bc 100644 --- a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json @@ -77,7 +77,10 @@ "forwarded" ], "url.domain": "sophostest.com", + "url.extension": "pdf", "url.original": "http://sophostest.com/Sandstorm/SBTestFile1.pdf", + "url.path": "/Sandstorm/SBTestFile1.pdf", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" }, { @@ -158,7 +161,10 @@ "forwarded" ], "url.domain": "sophostest.com", + "url.extension": "html", "url.original": "http://sophostest.com/eicar/index.html", + "url.path": "/eicar/index.html", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" }, { diff --git a/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json index a0230cb1dc49..969ca99d7935 100644 --- a/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json @@ -140,7 +140,11 @@ "sophos-xg", "forwarded" ], - "url.original": "http://sophostest.com/callhome/index.html" + "url.domain": "sophostest.com", + "url.extension": "html", + "url.original": "http://sophostest.com/callhome/index.html", + "url.path": "/callhome/index.html", + "url.scheme": "http" }, { "@timestamp": "2020-05-18T14:38:35.000-02:00", @@ -211,7 +215,11 @@ "sophos-xg", "forwarded" ], - "url.original": "http://sophostest.com/callhome/index.html" + "url.domain": "sophostest.com", + "url.extension": "html", + "url.original": "http://sophostest.com/callhome/index.html", + "url.path": "/callhome/index.html", + "url.scheme": "http" }, { "@timestamp": "2018-06-05T08:49:00.000-02:00", diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index d2bf90581b81..7bf0484a46f2 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -10,6 +10,41 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true + - remove: + field: url.scheme + ignore_missing: true + if: ctx?.url?.scheme == ctx?.url?.domain # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index bcced9c85a3e..420f48270f21 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -134,7 +134,9 @@ ], "url.domain": "www.goonernews.com", "url.original": "http://www.goonernews.com/", + "url.path": "/", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -206,8 +208,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "css", "url.original": "http://www.goonernews.com/styles.css", + "url.path": "/styles.css", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -265,8 +270,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "css", "url.original": "http://www.goonernews.com/styles.css", + "url.path": "/styles.css", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -324,8 +332,11 @@ "forwarded" ], "url.domain": "www.google-analytics.com", + "url.extension": "js", "url.original": "http://www.google-analytics.com/urchin.js", + "url.path": "/urchin.js", "url.registered_domain": "google-analytics.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -397,7 +408,9 @@ ], "url.domain": "www.goonernews.com", "url.original": "http://www.goonernews.com/", + "url.path": "/", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -466,8 +479,12 @@ "forwarded" ], "url.domain": "www.google-analytics.com", + "url.extension": "gif", "url.original": "http://www.google-analytics.com/__utm.gif?", + "url.path": "/__utm.gif", + "url.query": "", "url.registered_domain": "google-analytics.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -539,8 +556,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/graphics/newslogo.gif", + "url.path": "/graphics/newslogo.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -612,8 +632,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "jpg", "url.original": "http://www.goonernews.com/shop/arsenal_shop_ad.jpg", + "url.path": "/shop/arsenal_shop_ad.jpg", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -685,8 +708,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FUS.gif", + "url.path": "/flags/FUS.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -744,8 +770,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FGB.gif", + "url.path": "/flags/FGB.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -818,7 +847,10 @@ ], "url.domain": "as.casalemedia.com", "url.original": "http://as.casalemedia.com/s?", + "url.path": "/s", + "url.query": "", "url.registered_domain": "casalemedia.com", + "url.scheme": "http", "url.subdomain": "as", "url.top_level_domain": "com", "user.name": "badeyek" @@ -954,7 +986,9 @@ ], "url.domain": "impgb.tradedoubler.com", "url.original": "http://impgb.tradedoubler.com/imp/img/16349696/992098", + "url.path": "/imp/img/16349696/992098", "url.registered_domain": "tradedoubler.com", + "url.scheme": "http", "url.subdomain": "impgb", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1026,8 +1060,12 @@ "forwarded" ], "url.domain": "4.adbrite.com", + "url.extension": "php", "url.original": "http://4.adbrite.com/mb/text_group.php?", + "url.path": "/mb/text_group.php", + "url.query": "", "url.registered_domain": "adbrite.com", + "url.scheme": "http", "url.subdomain": "4", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1061,8 +1099,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -1085,8 +1123,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FFR.gif", + "url.path": "/flags/FFR.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1158,8 +1199,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FAU.gif", + "url.path": "/flags/FAU.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1231,8 +1275,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/graphics/spacer.gif", + "url.path": "/graphics/spacer.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1299,8 +1346,12 @@ "forwarded" ], "url.domain": "4.adbrite.com", + "url.extension": "php", "url.original": "http://4.adbrite.com/mb/text_group.php?", + "url.path": "/mb/text_group.php", + "url.query": "", "url.registered_domain": "adbrite.com", + "url.scheme": "http", "url.subdomain": "4", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1373,7 +1424,10 @@ ], "url.domain": "ff.connextra.com", "url.original": "http://ff.connextra.com/Ladbrokes/selector/image?", + "url.path": "/Ladbrokes/selector/image", + "url.query": "", "url.registered_domain": "connextra.com", + "url.scheme": "http", "url.subdomain": "ff", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1446,7 +1500,10 @@ ], "url.domain": "dd.connextra.com", "url.original": "http://dd.connextra.com/servlet/controller?", + "url.path": "/servlet/controller", + "url.query": "", "url.registered_domain": "connextra.com", + "url.scheme": "http", "url.subdomain": "dd", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1503,7 +1560,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "-" }, @@ -1628,8 +1687,11 @@ "forwarded" ], "url.domain": "update.messenger.yahoo.com", + "url.extension": "html", "url.original": "http://update.messenger.yahoo.com/msgrcli7.html", + "url.path": "/msgrcli7.html", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "update.messenger", "url.top_level_domain": "com", "user.name": "-" @@ -1672,8 +1734,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -1697,7 +1759,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -1768,7 +1832,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -1815,8 +1881,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -1838,8 +1904,11 @@ "forwarded" ], "url.domain": "hi5.com", + "url.extension": "css", "url.original": "http://hi5.com/friend/styles/homepage.css", + "url.path": "/friend/styles/homepage.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -1906,7 +1975,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -1964,8 +2035,11 @@ "forwarded" ], "url.domain": "rms.adobe.com", + "url.extension": "xml", "url.original": "http://rms.adobe.com/read/0600/win_/ENU/read0600win_ENUadbe0000.xml", + "url.path": "/read/0600/win_/ENU/read0600win_ENUadbe0000.xml", "url.registered_domain": "adobe.com", + "url.scheme": "http", "url.subdomain": "rms", "url.top_level_domain": "com", "user.name": "-" @@ -2023,8 +2097,11 @@ "forwarded" ], "url.domain": "images.hi5.com", + "url.extension": "css", "url.original": "http://images.hi5.com/styles/style.css", + "url.path": "/styles/style.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.subdomain": "images", "url.top_level_domain": "com", "user.name": "nazsoau" @@ -2082,8 +2159,11 @@ "forwarded" ], "url.domain": "images.hi5.com", + "url.extension": "css", "url.original": "http://images.hi5.com/friend/styles/buttons_en_us.css", + "url.path": "/friend/styles/buttons_en_us.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.subdomain": "images", "url.top_level_domain": "com", "user.name": "nazsoau" @@ -2154,7 +2234,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -2224,8 +2306,11 @@ "forwarded" ], "url.domain": "hi5.com", + "url.extension": "css", "url.original": "http://hi5.com/friend/styles/headernav.css", + "url.path": "/friend/styles/headernav.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -2292,7 +2377,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2359,7 +2446,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/?", + "url.path": "/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2428,8 +2518,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/playmessenger.asp", + "url.path": "/radio/play/playmessenger.asp", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2497,7 +2590,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2565,7 +2660,10 @@ ], "url.domain": "address.yahoo.com", "url.original": "http://address.yahoo.com/yab/us?", + "url.path": "/yab/us", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "address", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2637,8 +2735,11 @@ "forwarded" ], "url.domain": "fxfeeds.mozilla.org", + "url.extension": "xml", "url.original": "http://fxfeeds.mozilla.org/rss20.xml", + "url.path": "/rss20.xml", "url.registered_domain": "mozilla.org", + "url.scheme": "http", "url.subdomain": "fxfeeds", "url.top_level_domain": "org", "user.name": "badeyek" @@ -2682,8 +2783,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2707,7 +2808,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/ycontent/?", + "url.path": "/ycontent/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2749,8 +2853,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2774,7 +2878,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/ycontent/?", + "url.path": "/ycontent/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2888,8 +2995,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appinstru.asp", + "url.path": "/apps/agent/submgr/appinstru.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -2946,8 +3056,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appsync.asp", + "url.path": "/apps/agent/submgr/appsync.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3073,8 +3186,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_stations.gif", + "url.path": "/radio/clientdata/538/images/btn_stations.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3200,8 +3316,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_stations_over.gif", + "url.path": "/radio/clientdata/538/images/btn_stations_over.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3269,7 +3388,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3338,8 +3459,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_left.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_left.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3408,8 +3532,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/launchcast_radio.gif", + "url.path": "/radio/clientdata/538/images/launchcast_radio.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3466,8 +3593,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appinstru.asp", + "url.path": "/apps/agent/submgr/appinstru.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3524,8 +3654,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appsync.asp", + "url.path": "/apps/agent/submgr/appsync.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3594,8 +3727,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_right.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_right.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3664,8 +3800,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_center.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_center.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3710,8 +3849,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3734,8 +3873,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_controls_off.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_controls_off.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3804,8 +3946,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", + "url.path": "/radio/common_radio/resources/images/t.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3863,8 +4008,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_off_state_station.gif", + "url.path": "/radio/clientdata/538/images/btn_off_state_station.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3922,8 +4070,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_controls_fill.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_controls_fill.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3957,8 +4108,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -3981,8 +4132,11 @@ "forwarded" ], "url.domain": "us.i1.yimg.com", + "url.extension": "gif", "url.original": "http://us.i1.yimg.com/us.yimg.com/i/us/toolbar50x50.gif", + "url.path": "/us.yimg.com/i/us/toolbar50x50.gif", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.i1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4051,8 +4205,11 @@ "forwarded" ], "url.domain": "newsrss.bbc.co.uk", + "url.extension": "xml", "url.original": "http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml", + "url.path": "/rss/newsonline_world_edition/front_page/rss.xml", "url.registered_domain": "bbc.co.uk", + "url.scheme": "http", "url.subdomain": "newsrss", "url.top_level_domain": "co.uk", "user.name": "badeyek" @@ -4121,8 +4278,11 @@ "forwarded" ], "url.domain": "insider.msg.yahoo.com", + "url.extension": "php", "url.original": "http://insider.msg.yahoo.com/ycontent/beacon.php", + "url.path": "/ycontent/beacon.php", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4180,8 +4340,11 @@ "forwarded" ], "url.domain": "us.ent1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.ent1.yimg.com/images.launch.yahoo.com/000/032/457/32457654.jpg", + "url.path": "/images.launch.yahoo.com/000/032/457/32457654.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.ent1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4239,8 +4402,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060906/thumb.71d29ded334347c48ac88433d033c9a9.pakistan_bin_laden_nyol440.jpg", + "url.path": "/us.yimg.com/p/ap/20060906/thumb.71d29ded334347c48ac88433d033c9a9.pakistan_bin_laden_nyol440.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4308,8 +4474,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/authplay.asp", + "url.path": "/radio/play/authplay.asp", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4381,8 +4550,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060908/thumb.443f57762d7349669f609fbf0c97a5f1.academy_awards_host_cacp101.jpg", + "url.path": "/us.yimg.com/p/ap/20060908/thumb.443f57762d7349669f609fbf0c97a5f1.academy_awards_host_cacp101.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4451,8 +4623,12 @@ "forwarded" ], "url.domain": "radio.music.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.music.yahoo.com/radio/player/ymsgr/initstationfeed.asp?", + "url.path": "/radio/player/ymsgr/initstationfeed.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.music", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4521,8 +4697,12 @@ "forwarded" ], "url.domain": "radio.music.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.music.yahoo.com/radio/player/ymsgr/initstationfeed.asp?", + "url.path": "/radio/player/ymsgr/initstationfeed.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.music", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4591,8 +4771,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/authplay.asp?", + "url.path": "/radio/play/authplay.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4651,7 +4835,11 @@ ], "url.domain": "natrocket.kmip.net", "url.original": "http://natrocket.kmip.net:5288/iesocks?", + "url.path": "/iesocks", + "url.port": 5288, + "url.query": "", "url.registered_domain": "kmip.net", + "url.scheme": "http", "url.subdomain": "natrocket", "url.top_level_domain": "net", "user.name": "-" @@ -4710,7 +4898,11 @@ ], "url.domain": "natrocket.kmip.net", "url.original": "http://natrocket.kmip.net:5288/return?", + "url.path": "/return", + "url.port": 5288, + "url.query": "", "url.registered_domain": "kmip.net", + "url.scheme": "http", "url.subdomain": "natrocket", "url.top_level_domain": "net", "user.name": "-" @@ -4782,8 +4974,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060907/thumb.1caf18e56db54eafb16da58356eb3382.amazon_com_online_video_watw101.jpg", + "url.path": "/us.yimg.com/p/ap/20060907/thumb.1caf18e56db54eafb16da58356eb3382.amazon_com_online_video_watw101.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4852,8 +5047,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/515/starter.asp?", + "url.path": "/radio/clientdata/515/starter.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4922,8 +5121,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/player/default.asp?", + "url.path": "/radio/player/default.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4995,8 +5198,12 @@ "forwarded" ], "url.domain": "us.a2.yimg.com", + "url.extension": "swf", "url.original": "http://us.a2.yimg.com/us.yimg.com/a/ya/yahoo_messenger/081106_lrec_msgr_interophitchhiker.swf?", + "url.path": "/us.yimg.com/a/ya/yahoo_messenger/081106_lrec_msgr_interophitchhiker.swf", + "url.query": "", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.a2", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5065,8 +5272,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/player/stickwall.asp?", + "url.path": "/radio/player/stickwall.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5134,7 +5345,10 @@ ], "url.domain": "us.bc.yahoo.com", "url.original": "http://us.bc.yahoo.com/b?", + "url.path": "/b", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "us.bc", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5201,8 +5415,12 @@ "forwarded" ], "url.domain": "insider.msg.yahoo.com", + "url.extension": "php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw", "url.original": "http://insider.msg.yahoo.com/ycontent/beacon.php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw?", + "url.path": "/ycontent/beacon.php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5326,8 +5544,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "js", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", + "url.path": "/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5375,8 +5596,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "304", @@ -5399,8 +5620,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "css", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", + "url.path": "/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5472,8 +5696,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5613,8 +5840,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", + "url.path": "/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5686,8 +5916,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5745,8 +5978,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", + "url.path": "/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5818,8 +6054,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5891,8 +6130,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", + "url.path": "/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5964,8 +6206,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -6024,7 +6269,9 @@ ], "url.domain": "www.google.com", "url.original": "http://www.google.com/supported_domains", + "url.path": "/supported_domains", "url.registered_domain": "google.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "-" @@ -6082,8 +6329,12 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/en-us/agent5/chknews.asp?", + "url.path": "/apps/agent/en-us/agent5/chknews.asp", + "url.query": "", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -6151,7 +6402,10 @@ ], "url.domain": "launch.adserver.yahoo.com", "url.original": "http://launch.adserver.yahoo.com/l?", + "url.path": "/l", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "launch.adserver", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -6221,7 +6475,10 @@ ], "url.domain": "uk.f250.mail.yahoo.com", "url.original": "http://uk.f250.mail.yahoo.com/dc/launch?", + "url.path": "/dc/launch", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "uk.f250.mail", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6350,8 +6607,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/77cf3e56414f974dfd8616f56f0f632c_1.js", + "url.path": "/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/77cf3e56414f974dfd8616f56f0f632c_1.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6409,8 +6669,11 @@ "forwarded" ], "url.domain": "us.js1.yimg.com", + "url.extension": "css", "url.original": "http://us.js1.yimg.com/us.yimg.com/lib/hdr/ygma5.css", + "url.path": "/us.yimg.com/lib/hdr/ygma5.css", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js1", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6482,8 +6745,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/f7fc76100697c9c2d25dd0ec35e563b0_1.js", + "url.path": "/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/f7fc76100697c9c2d25dd0ec35e563b0_1.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6541,8 +6807,11 @@ "forwarded" ], "url.domain": "us.js1.yimg.com", + "url.extension": "js", "url.original": "http://us.js1.yimg.com/us.yimg.com/lib/pim/r/medici/13_15/mail/ac.js", + "url.path": "/us.yimg.com/lib/pim/r/medici/13_15/mail/ac.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js1", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6600,8 +6869,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/common/utils/2/yahoo_2.0.0-b4.js", + "url.path": "/us.js.yimg.com/lib/common/utils/2/yahoo_2.0.0-b4.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6659,8 +6931,11 @@ "forwarded" ], "url.domain": "us.i1.yimg.com", + "url.extension": "gif", "url.original": "http://us.i1.yimg.com/us.yimg.com/i/us/pim/dclient/d/img/liam_ball_1.gif", + "url.path": "/us.yimg.com/i/us/pim/dclient/d/img/liam_ball_1.gif", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.i1", "url.top_level_domain": "com", "user.name": "badeyek" diff --git a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json index 69a23c8a5717..0e23e14e212d 100644 --- a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json @@ -55,9 +55,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "min", "url.original": "https://example.org/exercita/der.htm?odoco=ria#min", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/exercita/der.htm" + ], + "url.query": "odoco=ria", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "tation", "user_agent.device.name": "Samsung SM-A715F", @@ -125,9 +132,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "nidolor", "url.original": "https://www.example.org/enderitq/sperna.txt?billoi=oreetdol#nidolor", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/enderitq/sperna.txt" + ], + "url.query": "billoi=oreetdol", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "doeiu", @@ -162,8 +176,8 @@ "example.net" ], "related.ip": [ - "10.142.172.64", - "10.15.135.248" + "10.15.135.248", + "10.142.172.64" ], "related.user": [ "tia" @@ -196,9 +210,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "iin", "url.original": "https://internal.example.com/aqu/utper.jpg?eFinib=omm#iin", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/aqu/utper.jpg" + ], + "url.query": "eFinib=omm", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "tia", @@ -267,9 +288,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "con", "url.original": "https://api.example.org/ceroinBC/ratvolup.gif?iatu=ionofde#con", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ceroinBC/ratvolup.gif" + ], + "url.query": "iatu=ionofde", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user_agent.device.name": "5024D_RU", @@ -337,9 +365,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "eos", "url.original": "https://api.example.org/isetq/estqui.gif?magn=equuntu#eos", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/isetq/estqui.gif" + ], + "url.query": "magn=equuntu", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "onev", @@ -411,9 +446,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "ect", "url.original": "https://www.example.org/inesci/rsitvolu.txt?pori=occ#ect", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/inesci/rsitvolu.txt" + ], + "url.query": "pori=occ", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "mrema", @@ -456,8 +498,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "allow", - "DELETE" + "DELETE", + "allow" ], "rsa.misc.content_type": "usmodte", "rsa.misc.result_code": "mUt", @@ -482,9 +524,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "atat", "url.original": "https://www5.example.com/ari/eataevit.txt?iam=mqua#atat", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/ari/eataevit.txt" + ], + "url.query": "iam=mqua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "tsed", @@ -552,9 +601,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "orsitame", "url.original": "https://example.org/tatno/imav.htm?ofdeF=tion#orsitame", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/tatno/imav.htm" + ], + "url.query": "ofdeF=tion", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "umdo", "user_agent.device.name": "Notepad_K10", @@ -624,9 +680,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "iumto", "url.original": "https://example.org/uaera/sitas.txt?aedic=atquovo#iumto", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/uaera/sitas.txt" + ], + "url.query": "aedic=atquovo", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "saute", "user_agent.device.name": "STK-L21", @@ -696,9 +759,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "nsequu", "url.original": "https://mail.example.org/edquiano/loru.htm?end=enia#nsequu", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/edquiano/loru.htm" + ], + "url.query": "end=enia", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "Nemoeni", @@ -766,9 +836,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "xercitat", "url.original": "https://www.example.org/iqui/etc.txt?tatiset=eprehen#xercitat", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/iqui/etc.txt" + ], + "url.query": "tatiset=eprehen", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "aliq", @@ -837,9 +914,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "txt", + "url.fragment": "ugiatnu", "url.original": "https://internal.example.net/ommod/sequatur.txt?tlabo=suntexp#ugiatnu", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/ommod/sequatur.txt" + ], + "url.query": "tlabo=suntexp", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "uines", @@ -904,9 +988,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "eporroqu", "url.original": "https://api.example.org/oriosamn/deFinibu.gif?iciatisu=rehender#eporroqu", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/oriosamn/deFinibu.gif" + ], + "url.query": "iciatisu=rehender", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "xeac", @@ -975,9 +1066,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "civeli", "url.original": "https://mail.example.org/rehend/tio.html?numqu=qui#civeli", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/rehend/tio.html" + ], + "url.query": "numqu=qui", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "ipitla", @@ -1046,9 +1144,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "iamquis", "url.original": "https://www.example.org/eav/ionevo.txt?siar=orev#iamquis", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/eav/ionevo.txt" + ], + "url.query": "siar=orev", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "aturve", @@ -1117,9 +1222,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "leumiu", "url.original": "https://internal.example.net/wri/bor.jpg?hitect=dol#leumiu", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/wri/bor.jpg" + ], + "url.query": "hitect=dol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "itame", @@ -1191,9 +1303,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "eriamea", "url.original": "https://api.example.net/equat/doloreme.htm?ione=ihilmole#eriamea", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/equat/doloreme.htm" + ], + "url.query": "ione=ihilmole", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "iquidex", @@ -1262,9 +1381,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "xerc", "url.original": "https://www5.example.com/estia/tper.gif?volupt=osqui#xerc", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/estia/tper.gif" + ], + "url.query": "volupt=osqui", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "etdol", @@ -1336,9 +1462,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "isnis", "url.original": "https://internal.example.net/sin/rvel.htm?nimid=itatione#isnis", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/sin/rvel.htm" + ], + "url.query": "nimid=itatione", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "isnisiu", @@ -1407,9 +1540,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "equat", "url.original": "https://api.example.org/abo/veniamqu.gif?aliquide=ofde#equat", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/abo/veniamqu.gif" + ], + "url.query": "aliquide=ofde", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "lapar", @@ -1478,9 +1618,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "icabo", "url.original": "https://api.example.org/autfu/gnaaliq.jpg?olupta=litse#icabo", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/autfu/gnaaliq.jpg" + ], + "url.query": "olupta=litse", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "volupt", @@ -1551,9 +1698,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "ntocc", "url.original": "https://internal.example.net/ptatemq/luptatev.html?Nequepo=ipsumd#ntocc", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/ptatemq/luptatev.html" + ], + "url.query": "Nequepo=ipsumd", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "reetdolo", @@ -1622,9 +1776,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "cons", "url.original": "https://www5.example.net/quiavol/rrorsi.gif?iatisu=sec#cons", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/quiavol/rrorsi.gif" + ], + "url.query": "iatisu=sec", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "tlab", @@ -1693,9 +1854,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "edquia", "url.original": "https://api.example.net/orain/tiumt.jpg?litessec=itas#edquia", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/orain/tiumt.jpg" + ], + "url.query": "litessec=itas", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "qui", @@ -1764,9 +1932,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "untut", "url.original": "https://www.example.net/str/idolore.txt?eetdolo=cteturad#untut", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/str/idolore.txt" + ], + "url.query": "eetdolo=cteturad", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "ntexplic", @@ -1835,9 +2010,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "mestq", "url.original": "https://mail.example.net/velitse/oditem.html?torever=oremi#mestq", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/velitse/oditem.html" + ], + "url.query": "torever=oremi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "mquelau", @@ -1908,9 +2090,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "uisa", "url.original": "https://internal.example.org/isnisi/ritatise.gif?tamet=quatur#uisa", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/isnisi/ritatise.gif" + ], + "url.query": "tamet=quatur", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "ice", @@ -1979,9 +2168,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "ano", "url.original": "https://www.example.net/ianon/tsed.htm?ameiusm=proide#ano", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/ianon/tsed.htm" + ], + "url.query": "ameiusm=proide", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "edqui", @@ -2050,9 +2246,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "apariat", "url.original": "https://www5.example.org/ncididun/umSe.jpg?ise=itau#apariat", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/ncididun/umSe.jpg" + ], + "url.query": "ise=itau", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "onsecte", @@ -2120,9 +2323,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "tem", "url.original": "https://example.org/uatur/adminimv.gif?exeacom=roidents#tem", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/uatur/adminimv.gif" + ], + "url.query": "exeacom=roidents", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "isis", "user_agent.device.name": "Generic Smartphone", @@ -2188,9 +2398,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "reprehe", "url.original": "https://api.example.com/ven/rQu.html?doloreme=dun#reprehe", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ven/rQu.html" + ], + "url.query": "doloreme=dun", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "oraincid", @@ -2262,9 +2479,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "ate", "url.original": "https://api.example.net/eseru/quamest.html?qua=rsita#ate", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/eseru/quamest.html" + ], + "url.query": "qua=rsita", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "upt", @@ -2333,9 +2557,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "isiu", "url.original": "https://internal.example.org/mvolu/conse.txt?aincidu=nimadmin#isiu", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/mvolu/conse.txt" + ], + "url.query": "aincidu=nimadmin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "xeaco", @@ -2403,9 +2634,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "html", + "url.fragment": "ilmol", "url.original": "https://internal.example.org/isciv/rroqu.html?uisa=tametco#ilmol", - "url.path": "https://internal.example.org", + "url.path": [ + "https://internal.example.org", + "/isciv/rroqu.html" + ], + "url.query": "uisa=tametco", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "autodit", @@ -2474,9 +2712,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "pid", "url.original": "https://www5.example.net/uaeratv/isa.txt?periam=dqu#pid", - "url.path": "https://example.org", + "url.path": [ + "https://example.org", + "/uaeratv/isa.txt" + ], + "url.query": "periam=dqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "reetd", @@ -2545,9 +2790,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "remagn", "url.original": "https://www5.example.com/orroq/vitaedic.txt?orisni=ons#remagn", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/orroq/vitaedic.txt" + ], + "url.query": "orisni=ons", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "atu", @@ -2615,9 +2867,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "ruredo", "url.original": "https://example.net/mven/olorsit.gif?oremag=illu#ruredo", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/mven/olorsit.gif" + ], + "url.query": "oremag=illu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -2684,9 +2943,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "html", + "url.fragment": "oin", "url.original": "https://www5.example.net/rum/eataevi.html?ulla=iqu#oin", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/rum/eataevi.html" + ], + "url.query": "ulla=iqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "Duisa", @@ -2755,9 +3021,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "tinvolup", "url.original": "https://mail.example.net/osquir/mod.txt?fugitse=imad#tinvolup", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/osquir/mod.txt" + ], + "url.query": "fugitse=imad", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "atu", @@ -2792,8 +3065,8 @@ "api.example.com" ], "related.ip": [ - "10.228.77.21", - "10.92.237.93" + "10.92.237.93", + "10.228.77.21" ], "related.user": [ "onse" @@ -2826,9 +3099,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "txt", + "url.fragment": "mve", "url.original": "https://api.example.com/asper/umq.txt?itasper=uae#mve", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/asper/umq.txt" + ], + "url.query": "itasper=uae", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "onse", @@ -2899,9 +3179,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "jpg", + "url.fragment": "veniamqu", "url.original": "https://www5.example.net/quatD/isqua.jpg?oloreseo=iruredol#veniamqu", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/quatD/isqua.jpg" + ], + "url.query": "oloreseo=iruredol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "ntexpl", @@ -2968,9 +3255,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "repreh", "url.original": "https://api.example.net/ation/luptas.html?iatqu=lorsi#repreh", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/ation/luptas.html" + ], + "url.query": "iatqu=lorsi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "tionula", @@ -3039,9 +3333,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "eumi", "url.original": "https://internal.example.com/mpo/unte.jpg?ueipsa=scipitl#eumi", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/mpo/unte.jpg" + ], + "url.query": "ueipsa=scipitl", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "rsp", @@ -3072,8 +3373,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "example.com" + "example.com", + "www.example.org" ], "related.ip": [ "10.101.85.169", @@ -3113,9 +3414,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "olup", "url.original": "https://www.example.org/uov/quaeab.jpg?moles=dipiscin#olup", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uov/quaeab.jpg" + ], + "url.query": "moles=dipiscin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "liquid", @@ -3161,8 +3469,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "HEAD", - "block" + "block", + "HEAD" ], "rsa.misc.content_type": "scipitl", "rsa.misc.result_code": "temaccu", @@ -3186,9 +3494,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "snos", "url.original": "https://example.net/edolo/ugiatquo.jpg?eosquira=pta#snos", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/edolo/ugiatquo.jpg" + ], + "url.query": "eosquira=pta", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user.name": "eavol", "user_agent.device.name": "Asus X01BDA", @@ -3258,9 +3573,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "nrepreh", "url.original": "https://api.example.org/isci/dolor.htm?orinrep=quiavol#nrepreh", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/isci/dolor.htm" + ], + "url.query": "orinrep=quiavol", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "BCS", @@ -3329,9 +3651,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "culpaqui", "url.original": "https://mail.example.com/roide/tem.gif?rerepre=nculpaq#culpaqui", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/roide/tem.gif" + ], + "url.query": "rerepre=nculpaq", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "obea", @@ -3400,9 +3729,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "edquia", "url.original": "https://internal.example.net/rautod/olest.jpg?lapar=ritati#edquia", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/rautod/olest.jpg" + ], + "url.query": "lapar=ritati", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "amvolu", @@ -3471,9 +3807,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "iame", "url.original": "https://api.example.org/iusmodt/enim.txt?aquio=ersp#iame", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/iusmodt/enim.txt" + ], + "url.query": "aquio=ersp", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "mexer", @@ -3543,9 +3886,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "eetdo", "url.original": "https://example.org/abillo/undeom.html?oraincid=quaer#eetdo", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/abillo/undeom.html" + ], + "url.query": "oraincid=quaer", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "nse", "user_agent.device.name": "Samsung SM-A715F", @@ -3613,9 +3963,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "sse", "url.original": "https://mail.example.net/uam/orumSec.jpg?isnisiu=suntincu#sse", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/uam/orumSec.jpg" + ], + "url.query": "isnisiu=suntincu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "untutlab", @@ -3687,9 +4044,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "iamea", "url.original": "https://www5.example.org/umiurer/rere.txt?mnisi=usmo#iamea", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/umiurer/rere.txt" + ], + "url.query": "mnisi=usmo", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "odoco", @@ -3754,9 +4118,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "remipsu", "url.original": "https://internal.example.net/atnula/ditautf.jpg?iquidex=olup#remipsu", - "url.path": "https://internal.example.org", + "url.path": [ + "https://internal.example.org", + "/atnula/ditautf.jpg" + ], + "url.query": "iquidex=olup", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "etdol", @@ -3821,9 +4192,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "xeacommo", "url.original": "https://api.example.net/uscip/umS.txt?quiacons=uisa#xeacommo", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uscip/umS.txt" + ], + "url.query": "quiacons=uisa", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "identsu", @@ -3888,9 +4266,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "temporai", "url.original": "https://mail.example.com/mrema/ullamc.txt?eufug=roquisq#temporai", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/mrema/ullamc.txt" + ], + "url.query": "eufug=roquisq", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "enimadm", @@ -3936,8 +4321,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "GET" + "GET", + "deny" ], "rsa.misc.content_type": "eriam", "rsa.misc.result_code": "sseq", @@ -3962,9 +4347,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "orsitam", "url.original": "https://www5.example.com/isau/itinvol.txt?saquaea=ons#orsitam", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/isau/itinvol.txt" + ], + "url.query": "saquaea=ons", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "riame", @@ -4033,9 +4425,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "onemulla", "url.original": "https://api.example.net/veli/venia.htm?etdolor=uat#onemulla", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/veli/venia.htm" + ], + "url.query": "etdolor=uat", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "asp", @@ -4104,9 +4503,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "expli", "url.original": "https://internal.example.net/ainci/osqu.jpg?sus=imavenia#expli", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/ainci/osqu.jpg" + ], + "url.query": "sus=imavenia", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "ntorever", @@ -4177,9 +4583,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "seos", "url.original": "https://www.example.org/runtm/eturadip.htm?psumd=oloree#seos", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/runtm/eturadip.htm" + ], + "url.query": "psumd=oloree", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "mcorpo", @@ -4244,9 +4657,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "html", + "url.fragment": "umwri", "url.original": "https://internal.example.org/liquipex/uisnos.html?ventor=lupt#umwri", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/liquipex/uisnos.html" + ], + "url.query": "ventor=lupt", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "nvo", @@ -4277,8 +4697,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "api.example.net" + "api.example.net", + "api.example.org" ], "related.ip": [ "10.83.130.95", @@ -4318,9 +4738,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "veniamq", "url.original": "https://api.example.org/mquisnos/lore.txt?siar=isn#veniamq", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/mquisnos/lore.txt" + ], + "url.query": "siar=isn", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "userror", @@ -4389,9 +4816,16 @@ "forwarded" ], "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "uptatem", "url.original": "https://www.example.com/quas/occaeca.htm?ender=dico#uptatem", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/quas/occaeca.htm" + ], + "url.query": "ender=dico", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "radip", @@ -4458,9 +4892,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "sit", "url.original": "https://www5.example.org/uidolo/umdolore.jpg?oquisq=abori#sit", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uidolo/umdolore.jpg" + ], + "url.query": "oquisq=abori", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "tatio", @@ -4529,9 +4970,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "tisu", "url.original": "https://internal.example.net/eniamqu/inimav.htm?imadm=uta#tisu", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/eniamqu/inimav.htm" + ], + "url.query": "imadm=uta", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "aturv", @@ -4600,9 +5048,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "giatquo", "url.original": "https://mail.example.net/sintocca/mipsumqu.htm?tnulapar=ico#giatquo", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/sintocca/mipsumqu.htm" + ], + "url.query": "tnulapar=ico", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "uatu", @@ -4671,9 +5126,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "eleumi", "url.original": "https://api.example.org/rep/remap.html?siarc=fdeFin#eleumi", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/rep/remap.html" + ], + "url.query": "siarc=fdeFin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "odic", @@ -4704,8 +5166,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "api.example.net" + "api.example.net", + "example.com" ], "related.ip": [ "10.62.188.193", @@ -4742,9 +5204,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "osa", "url.original": "https://api.example.net/eturad/tDuis.htm?enimadmi=tateveli#osa", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/eturad/tDuis.htm" + ], + "url.query": "enimadmi=tateveli", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "quaU", @@ -4809,9 +5278,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "madmi", "url.original": "https://www.example.net/hender/ptatemU.htm?mquisnos=tnulapa#madmi", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/hender/ptatemU.htm" + ], + "url.query": "mquisnos=tnulapa", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "onsectet", @@ -4882,9 +5358,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "oid", "url.original": "https://www.example.org/iduntutl/rsitam.htm?ntor=oinBCSed#oid", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/iduntutl/rsitam.htm" + ], + "url.query": "ntor=oinBCSed", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "midestl", @@ -4950,9 +5433,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "dat", "url.original": "https://example.org/itessequ/porissu.html?uip=ectobea#dat", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/itessequ/porissu.html" + ], + "url.query": "uip=ectobea", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "deomnisi", "user_agent.device.name": "LG-$2", @@ -5020,9 +5510,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "nimide", "url.original": "https://api.example.net/enimad/tis.txt?mipsumq=ident#nimide", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/enimad/tis.txt" + ], + "url.query": "mipsumq=ident", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "oeiusmo", @@ -5091,9 +5588,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "nse", "url.original": "https://api.example.org/taspe/yCiceroi.htm?cti=ommodoc#nse", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/taspe/yCiceroi.htm" + ], + "url.query": "cti=ommodoc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user_agent.device.name": "iPhone", @@ -5160,9 +5664,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "iarchite", "url.original": "https://example.com/beat/rro.jpg?uisau=qua#iarchite", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/beat/rro.jpg" + ], + "url.query": "uisau=qua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "tationu", "user_agent.device.name": "Lenovo A2016a40 ", @@ -5232,9 +5743,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "dexea", "url.original": "https://www5.example.org/loremq/turmagni.txt?emUtenim=ende#dexea", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/loremq/turmagni.txt" + ], + "url.query": "emUtenim=ende", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "caboNemo", @@ -5303,9 +5821,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "ecte", "url.original": "https://www5.example.net/deomnisi/ddoe.txt?oremi=ectobeat#ecte", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/deomnisi/ddoe.txt" + ], + "url.query": "oremi=ectobeat", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "cillumdo", @@ -5374,9 +5899,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "lors", "url.original": "https://api.example.net/tiset/sci.jpg?rauto=doloreeu#lors", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/tiset/sci.jpg" + ], + "url.query": "rauto=doloreeu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "roquisq", @@ -5445,9 +5977,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "tconsect", "url.original": "https://api.example.net/tDuisau/aturve.htm?tper=pisciv#tconsect", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/tDuisau/aturve.htm" + ], + "url.query": "tper=pisciv", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "isi", @@ -5516,9 +6055,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "scip", "url.original": "https://internal.example.com/oluptate/todi.jpg?tdolo=ident#scip", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/oluptate/todi.jpg" + ], + "url.query": "tdolo=ident", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "aparia", @@ -5549,8 +6095,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "mail.example.org" + "mail.example.org", + "mail.example.com" ], "related.ip": [ "10.175.138.42", @@ -5587,9 +6133,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "html", + "url.fragment": "utlabore", "url.original": "https://mail.example.com/radipis/lore.html?civeli=eufugia#utlabore", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/radipis/lore.html" + ], + "url.query": "civeli=eufugia", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "onemul", @@ -5632,8 +6185,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "allow", - "PROPFIND" + "PROPFIND", + "allow" ], "rsa.misc.content_type": "llum", "rsa.misc.result_code": "ntut", @@ -5658,9 +6211,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "jpg", + "url.fragment": "emp", "url.original": "https://internal.example.org/olupta/tio.jpg?idestl=litani#emp", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/olupta/tio.jpg" + ], + "url.query": "idestl=litani", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "ittenb", @@ -5729,9 +6289,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "Neq", "url.original": "https://www5.example.com/sequines/cto.gif?temaccu=uamqua#Neq", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/sequines/cto.gif" + ], + "url.query": "temaccu=uamqua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "ercitati", @@ -5800,9 +6367,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "gif", + "url.fragment": "onorume", "url.original": "https://www.example.net/periam/ain.gif?iquipex=mqu#onorume", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/periam/ain.gif" + ], + "url.query": "iquipex=mqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "mve", @@ -5873,9 +6447,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "Duis", "url.original": "https://www.example.net/ecatc/quovolu.jpg?dexe=nemul#Duis", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ecatc/quovolu.jpg" + ], + "url.query": "dexe=nemul", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "ipsa", @@ -5943,9 +6524,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "cipitla", "url.original": "https://example.com/tqui/ssequ.gif?emse=emqui#cipitla", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/tqui/ssequ.gif" + ], + "url.query": "emse=emqui", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "ptate", "user_agent.device.name": "Asus X01BDA", @@ -6015,9 +6603,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "quisnost", "url.original": "https://example.com/caboN/imipsam.jpg?catcupid=ritquiin#quisnost", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/caboN/imipsam.jpg" + ], + "url.query": "catcupid=ritquiin", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "ore", "user_agent.device.name": "Samsung SM-A260G", @@ -6085,9 +6680,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "osqu", "url.original": "https://www.example.org/oinvento/ali.htm?utaliqui=isciv#osqu", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/oinvento/ali.htm" + ], + "url.query": "utaliqui=isciv", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "enimad", @@ -6158,9 +6760,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "rch", "url.original": "https://api.example.net/quido/llo.htm?tpersp=assi#rch", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/quido/llo.htm" + ], + "url.query": "tpersp=assi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "oremeu", @@ -6232,9 +6841,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "deriti", "url.original": "https://www.example.net/tvolu/imve.txt?gnaaliq=quam#deriti", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/tvolu/imve.txt" + ], + "url.query": "gnaaliq=quam", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "mSecti", @@ -6303,9 +6919,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "tquii", "url.original": "https://www5.example.com/ugitsed/dminimve.htm?onse=uiac#tquii", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/ugitsed/dminimve.htm" + ], + "url.query": "onse=uiac", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "ita", @@ -6374,9 +6997,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "usan", "url.original": "https://api.example.com/ntium/ide.htm?tamrema=isautem#usan", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/ntium/ide.htm" + ], + "url.query": "tamrema=isautem", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "tvolu", @@ -6447,9 +7077,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "utaliqui", "url.original": "https://www5.example.org/iad/ngelits.jpg?mporin=orissusc#utaliqui", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/iad/ngelits.jpg" + ], + "url.query": "mporin=orissusc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "lumd", @@ -6492,8 +7129,8 @@ ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "TRACE", - "allow" + "allow", + "TRACE" ], "rsa.misc.content_type": "rinci", "rsa.misc.result_code": "iscivel", @@ -6518,9 +7155,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "eprehe", "url.original": "https://mail.example.com/iatnulap/roi.htm?uine=loreeu#eprehe", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/iatnulap/roi.htm" + ], + "url.query": "uine=loreeu", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "rem", @@ -6591,9 +7235,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "illoin", "url.original": "https://mail.example.org/turv/use.jpg?mtot=macc#illoin", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/turv/use.jpg" + ], + "url.query": "mtot=macc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "umq", @@ -6662,9 +7313,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "ineavol", "url.original": "https://mail.example.com/umdol/rerepr.txt?emipsumq=orinr#ineavol", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/umdol/rerepr.txt" + ], + "url.query": "emipsumq=orinr", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "upta", @@ -6733,9 +7391,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "nevolup", "url.original": "https://internal.example.com/rautod/onorumet.htm?mvo=agnidol#nevolup", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/rautod/onorumet.htm" + ], + "url.query": "mvo=agnidol", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "tinv", @@ -6803,9 +7468,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "asiar", "url.original": "https://example.net/dun/xce.jpg?nsequat=mvol#asiar", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/dun/xce.jpg" + ], + "url.query": "nsequat=mvol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user.name": "squamest", "user_agent.device.name": "Generic Smartphone", @@ -6833,12 +7505,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www.example.org" + "www.example.org", + "api.example.com" ], "related.ip": [ - "10.83.154.75", - "10.37.33.179" + "10.37.33.179", + "10.83.154.75" ], "related.user": [ "eatae" @@ -6871,9 +7543,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "sequu", "url.original": "https://api.example.com/teiru/mquamei.jpg?pta=uradi#sequu", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/teiru/mquamei.jpg" + ], + "url.query": "pta=uradi", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "eatae", @@ -6942,9 +7621,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "umdolo", "url.original": "https://www5.example.net/dtempor/rroquisq.gif?liquid=uidex#umdolo", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/dtempor/rroquisq.gif" + ], + "url.query": "liquid=uidex", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "usmod", @@ -7015,9 +7701,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "rchite", "url.original": "https://api.example.com/ore/adeser.htm?pre=aute#rchite", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/ore/adeser.htm" + ], + "url.query": "pre=aute", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "ptasnul", @@ -7086,9 +7779,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "aer", "url.original": "https://api.example.com/liqu/dolor.htm?ess=umdo#aer", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/liqu/dolor.htm" + ], + "url.query": "ess=umdo", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "edictas", diff --git a/x-pack/filebeat/module/suricata/eve/ingest/http.yml b/x-pack/filebeat/module/suricata/eve/ingest/http.yml new file mode 100644 index 000000000000..b1420dd42cf5 --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/http.yml @@ -0,0 +1,90 @@ +--- +description: Pipeline for Suricata DNS Events + +processors: + - set: + value: "{{suricata.eve.http.http_method}}" + field: http.request.method + if: "ctx?.suricata?.eve?.http?.http_method != null" + - rename: + field: suricata.eve.http.status + target_field: http.response.status_code + ignore_missing: true + - rename: + field: suricata.eve.http.hostname + target_field: destination.domain + ignore_missing: true + # - grok: + # field: suricata.eve.http.url + # patterns: + # - '%{PATH:url.path}(?:\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?' + # ignore_missing: true + # pattern_definitions: + # PATH: '[^?#]*' + # QUERY: '[^#]*' + # ANY: '.*' + - rename: + field: suricata.eve.http.url + target_field: url.original + ignore_missing: true + - rename: + field: suricata.eve.http.http_port + target_field: url.port + ignore_missing: true + + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - set: + field: url.domain + value: '{{destination.domain}}' + ignore_empty_value: true + if: ctx?.url?.domain == null && ctx?.destination?.domain != null + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - rename: + field: suricata.eve.http.http_refer + target_field: http.request.referrer + ignore_missing: true + - urldecode: + field: http.request.referrer + ignore_missing: true + - rename: + field: suricata.eve.http.length + target_field: http.response.body.bytes + ignore_missing: true + - user_agent: + field: suricata.eve.http.http_user_agent + ignore_missing: true + +on_failure: + - append: + field: error.message + value: >- + error in HTTP pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index a24a5df3ff67..e957d177f38f 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -152,6 +152,11 @@ processors: - pipeline: if: ctx?.network?.protocol == "tls" name: '{< IngestPipeline "tls" >}' + ## HTTP + - pipeline: + if: >- + ctx?.network?.protocol == "http" + name: '{< IngestPipeline "http" >}' ## Flow - append: if: ctx?.suricata?.eve?.flow?.state == "new" @@ -163,24 +168,6 @@ processors: field: event.type value: - end - - - - set: - value: "{{suricata.eve.http.http_method}}" - field: http.request.method - if: "ctx?.suricata?.eve?.http?.http_method != null" - - rename: - field: suricata.eve.http.status - target_field: http.response.status_code - ignore_missing: true - - append: - if: ctx.suricata?.eve?.http?.hostname != null - value: '{{suricata.eve.http.hostname}}' - field: destination.domain - allow_duplicates: false - - remove: - field: suricata.eve.http.hostname - ignore_failure: true - script: lang: painless tag: suricata_deduplicate_dest_domain @@ -194,36 +181,6 @@ processors: ctx.destination.domain = domain; } ignore_failure: true - - set: - if: ctx?.network?.protocol == 'http' - field: url.domain - value: '{{destination.domain}}' - ignore_empty_value: true - - grok: - field: suricata.eve.http.url - patterns: - - '%{PATH:url.path}(?:\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?' - ignore_missing: true - pattern_definitions: - PATH: '[^?#]*' - QUERY: '[^#]*' - ANY: '.*' - - rename: - field: suricata.eve.http.url - target_field: url.original - ignore_missing: true - - rename: - field: suricata.eve.http.http_port - target_field: url.port - ignore_missing: true - - rename: - field: suricata.eve.http.http_refer - target_field: http.request.referrer - ignore_missing: true - - rename: - field: suricata.eve.http.length - target_field: http.response.body.bytes - ignore_missing: true - rename: field: suricata.eve.fileinfo.filename target_field: file.path @@ -342,9 +299,6 @@ processors: field: suricata.eve.proto target_field: network.transport ignore_missing: true - - user_agent: - field: suricata.eve.http.http_user_agent - ignore_missing: true - geoip: if: ctx?.source?.geo == null field: source.ip diff --git a/x-pack/filebeat/module/suricata/eve/manifest.yml b/x-pack/filebeat/module/suricata/eve/manifest.yml index 8d7f87d83735..a712ab663364 100644 --- a/x-pack/filebeat/module/suricata/eve/manifest.yml +++ b/x-pack/filebeat/module/suricata/eve/manifest.yml @@ -19,6 +19,7 @@ ingest_pipeline: - ingest/dns-answer-v1.yml - ingest/dns-answer-v2.yml - ingest/tls.yml + - ingest/http.yml input: config/eve.yml requires.processors: diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json index e8f77f9033a7..b294e0cbdca8 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json @@ -78,6 +78,7 @@ "suricata" ], "url.domain": "testmynids.org", + "url.extension": "html", "url.original": "/uid/index.html", "url.path": "/uid/index.html", "user_agent.device.name": "Other", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log b/x-pack/filebeat/module/suricata/eve/test/eve-small.log index 9cc157a9e757..28104b077ecc 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log @@ -1,7 +1,7 @@ {"timestamp":"2018-07-05T15:01:09.820360-0400","flow_id":298824096901438,"in_iface":"en0","event_type":"ssh","src_ip":"192.168.86.85","src_port":55406,"dest_ip":"192.168.253.112","dest_port":22,"proto":"TCP","ssh":{"client":{"proto_version":"2.0","software_version":"OpenSSH_7.6"},"server":{"proto_version":"2.0","software_version":"libssh_0.7.0"}}} {"timestamp":"2018-07-05T15:07:20.910626-0400","flow_id":904992230150281,"in_iface":"en0","event_type":"alert","src_ip":"192.168.86.85","src_port":55641,"dest_ip":"192.168.156.70","dest_port":443,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2024833,"rev":3,"signature":"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)","category":"Potential Corporate Privacy Violation","severity":1},"tls":{"session_resumed":true,"sni":"l2.io","version":"TLS 1.2"},"app_proto":"tls","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":793,"bytes_toclient":343,"start":"2018-07-05T15:07:19.659593-0400"}} {"timestamp":"2018-07-05T15:43:47.690014-0400","flow_id":2115002772430095,"in_iface":"en0","event_type":"http","src_ip":"192.168.86.85","src_port":56119,"dest_ip":"192.168.86.28","dest_port":63963,"proto":"TCP","tx_id":0,"http":{"hostname":"192.168.86.28","url":"\/dd.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"text\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1155}} -{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.28","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} +{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.85","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} {"timestamp":"2018-07-05T15:51:20.213418-0400","flow_id":1684780223079543,"in_iface":"en0","event_type":"dns","src_ip":"192.168.86.1","src_port":53,"dest_ip":"192.168.86.85","dest_port":39464,"proto":"UDP","dns":{"type":"answer","id":12308,"rcode":"NOERROR","rrname":"clients.l.google.com","rrtype":"A","ttl":299,"rdata":"172.217.13.110"}} {"timestamp":"2018-07-05T15:51:23.009510-0400","event_type":"stats","stats":{"uptime":5400,"capture":{"kernel_packets":430313,"kernel_drops":0,"kernel_ifdrops":0},"decoder":{"pkts":430313,"bytes":335138381,"invalid":2,"ipv4":425873,"ipv6":3785,"ethernet":430313,"raw":0,"null":0,"sll":0,"tcp":370093,"udp":58337,"sctp":0,"icmpv4":186,"icmpv6":1019,"ppp":0,"pppoe":0,"gre":0,"vlan":0,"vlan_qinq":0,"ieee8021ah":0,"teredo":1,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":778,"max_pkt_size":1514,"erspan":0,"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"dce":{"pkt_too_small":0}},"flow":{"memcap":0,"tcp":1113,"udp":1881,"icmpv4":0,"icmpv6":677,"spare":10000,"emerg_mode_entered":0,"emerg_mode_over":0,"tcp_reuse":0,"memuse":11537312},"defrag":{"ipv4":{"fragments":0,"reassembled":0,"timeouts":0},"ipv6":{"fragments":0,"reassembled":0,"timeouts":0},"max_frag_hits":0},"tcp":{"sessions":842,"ssn_memcap_drop":0,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"no_flow":0,"syn":1138,"synack":656,"rst":1165,"segment_memcap_drop":0,"stream_depth_reached":63,"reassembly_gap":0,"overlap":5979,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"insert_list_fail":0,"memuse":4587520,"reassembly_memuse":768000},"detect":{"alert":2},"app_layer":{"flow":{"http":22,"ftp":0,"smtp":0,"tls":560,"ssh":4,"imap":0,"msn":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"failed_tcp":2,"dcerpc_udp":0,"dns_udp":762,"failed_udp":1119},"tx":{"http":25,"ftp":0,"smtp":0,"tls":0,"ssh":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"dcerpc_udp":0,"dns_udp":762}},"flow_mgr":{"closed_pruned":729,"new_pruned":1879,"est_pruned":975,"bypassed_pruned":0,"flows_checked":8,"flows_notimeout":8,"flows_timeout":0,"flows_timeout_inuse":0,"flows_removed":0,"rows_checked":65536,"rows_skipped":65530,"rows_empty":0,"rows_busy":0,"rows_maxlen":2},"file_store":{"open_files":0},"dns":{"memuse":7749,"memcap_state":0,"memcap_global":0},"http":{"memuse":17861,"memcap":0}}} {"timestamp":"2018-07-05T15:51:50.666597-0400","flow_id":89751777876473,"in_iface":"en0","event_type":"tls","src_ip":"192.168.86.85","src_port":56187,"dest_ip":"17.142.164.13","dest_port":443,"proto":"TCP","tls":{"subject":"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US","issuerdn":"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US","serial":"5C:9C:E1:09:78:87:F8:07","fingerprint":"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47","sni":"p33-btmmdns.icloud.com.","version":"TLS 1.2","notbefore":"2017-02-27T17:54:31","notafter":"2019-03-29T17:54:31"}} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 50125bc3f3c2..cfd6fa5ff832 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -151,6 +151,7 @@ "suricata" ], "url.domain": "192.168.86.28", + "url.extension": "xml", "url.original": "/dd.xml", "url.path": "/dd.xml", "user_agent.device.name": "Mac", @@ -164,7 +165,7 @@ { "@timestamp": "2018-07-05T19:44:33.222Z", "destination.address": "192.168.86.85", - "destination.domain": "192.168.86.28", + "destination.domain": "192.168.86.85", "destination.ip": "192.168.86.85", "destination.port": 56118, "event.category": [ @@ -173,7 +174,7 @@ "event.dataset": "suricata.eve", "event.kind": "event", "event.module": "suricata", - "event.original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.28\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}", + "event.original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.85\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}", "file.path": "/ssdp/device-desc.xml", "file.size": 1071, "fileset.name": "eve", @@ -186,7 +187,7 @@ "network.protocol": "http", "network.transport": "tcp", "related.hosts": [ - "192.168.86.28" + "192.168.86.85" ], "related.ip": [ "192.168.86.28", @@ -211,7 +212,8 @@ "tags": [ "suricata" ], - "url.domain": "192.168.86.28", + "url.domain": "192.168.86.85", + "url.extension": "xml", "url.original": "/ssdp/device-desc.xml", "url.path": "/ssdp/device-desc.xml", "user_agent.device.name": "Mac", @@ -582,9 +584,11 @@ "suricata" ], "url.domain": "ctldl.windowsupdate.com", + "url.extension": "cab", "url.original": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111", - "url.path": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", + "url.path": "/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", "url.query": "111111111111", + "url.scheme": "http", "user_agent.device.name": "Other", "user_agent.name": "Microsoft-CryptoAPI", "user_agent.original": "Microsoft-CryptoAPI/10.0", diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index b6105a0fddd8..f9dec729bc44 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -9,7 +9,38 @@ processors: # User agent - user_agent: field: user_agent.original - ignore_missing: true + ignore_missing: + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - urldecode: + field: url.original + ignore_missing: true + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json index 54841870df77..2629b307fc3c 100644 --- a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json +++ b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json @@ -65,7 +65,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "amremap", "url.original": "https://api.example.com/ivelitse/ritin.htm?utl=vol#amremap", + "url.path": "/ivelitse/ritin.htm", + "url.query": "utl=vol", + "url.scheme": "https", "user.name": "sumdo", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -139,7 +145,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "mquia", "url.original": "https://internal.example.net/isiutal/moenimi.jpg?gnaali=enatus#mquia", + "url.path": "/isiutal/moenimi.jpg", + "url.query": "gnaali=enatus", + "url.scheme": "https", "user.name": "eataevi", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -215,7 +227,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "iquaUte", "url.original": "https://example.com/taspe/mvolu.gif?atcup=snos#iquaUte", + "url.path": "/taspe/mvolu.gif", + "url.query": "atcup=snos", + "url.scheme": "https", "user.name": "tenima", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -291,7 +309,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "uid", "url.original": "https://api.example.org/doloreeu/pori.jpg?itati=mfu#uid", + "url.path": "/doloreeu/pori.jpg", + "url.query": "itati=mfu", + "url.scheme": "https", "user.name": "equun", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -367,7 +391,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iinea", "url.original": "https://api.example.com/ele/tenbyCic.gif?porainc=amquisno#iinea", + "url.path": "/ele/tenbyCic.gif", + "url.query": "porainc=amquisno", + "url.scheme": "https", "user.name": "ercit", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -443,7 +473,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "aboreetd", "url.original": "https://mail.example.org/sitas/ehenderi.jpg?atquovo=iumto#aboreetd", + "url.path": "/sitas/ehenderi.jpg", + "url.query": "atquovo=iumto", + "url.scheme": "https", "user.name": "tessec", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -495,8 +531,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "lupt", "rsa.misc.action": [ - "dun", - "Blocked" + "Blocked", + "dun" ], "rsa.misc.category": "rsitamet", "rsa.misc.filter": "usmod", @@ -519,7 +555,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "ctetura", "url.original": "https://mail.example.net/aborumSe/luptat.txt?antiumto=strude#ctetura", + "url.path": "/aborumSe/luptat.txt", + "url.query": "antiumto=strude", + "url.scheme": "https", "user.name": "xercitat", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -595,7 +637,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "jpg", + "url.fragment": "des", "url.original": "https://www5.example.net/ntutla/equa.jpg?civeli=errorsi#des", + "url.path": "/ntutla/equa.jpg", + "url.query": "civeli=errorsi", + "url.scheme": "https", "user.name": "erc", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -632,8 +680,8 @@ "aperia4409.www5.invalid" ], "related.ip": [ - "10.78.151.178", - "10.25.192.202" + "10.25.192.202", + "10.78.151.178" ], "related.user": [ "quip" @@ -671,7 +719,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "oriosamn", "url.original": "https://api.example.net/atvol/umiur.txt?tati=utaliqu#oriosamn", + "url.path": "/atvol/umiur.txt", + "url.query": "tati=utaliqu", + "url.scheme": "https", "user.name": "quip", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -747,7 +801,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "uipe", "url.original": "https://mail.example.net/equep/iavolu.gif?aqu=rpo#uipe", + "url.path": "/equep/iavolu.gif", + "url.query": "aqu=rpo", + "url.scheme": "https", "user.name": "atu", "user_agent.device.name": "POCOPHONE F1", "user_agent.name": "Chrome Mobile", @@ -823,7 +883,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "namali", "url.original": "https://example.org/bor/occa.htm?dol=leumiu#namali", + "url.path": "/bor/occa.htm", + "url.query": "dol=leumiu", + "url.scheme": "https", "user.name": "tNequepo", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -899,7 +965,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "reseos", "url.original": "https://internal.example.net/oru/temqu.htm?etMalor=ipi#reseos", + "url.path": "/oru/temqu.htm", + "url.query": "etMalor=ipi", + "url.scheme": "https", "user.name": "ihilmo", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -975,7 +1047,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "osamn", "url.original": "https://mail.example.net/itatione/isnis.html?oluptate=issus#osamn", + "url.path": "/itatione/isnis.html", + "url.query": "oluptate=issus", + "url.scheme": "https", "user.name": "ratvolu", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1051,7 +1129,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "rau", "url.original": "https://www.example.org/eporr/xeacomm.html?aturQui=utlabor#rau", + "url.path": "/eporr/xeacomm.html", + "url.query": "aturQui=utlabor", + "url.scheme": "https", "user.name": "volupta", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -1127,7 +1211,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "ntocc", "url.original": "https://internal.example.net/ptatemq/luptatev.html?Nequepo=ipsumd#ntocc", + "url.path": "/ptatemq/luptatev.html", + "url.query": "Nequepo=ipsumd", + "url.scheme": "https", "user.name": "saute", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1203,7 +1293,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "ipsumd", "url.original": "https://www5.example.com/tateve/itinvol.txt?tenatus=cipitlab#ipsumd", + "url.path": "/tateve/itinvol.txt", + "url.query": "tenatus=cipitlab", + "url.scheme": "https", "user.name": "inibusB", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1279,7 +1375,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "taedicta", "url.original": "https://api.example.net/tquiin/tse.jpg?ovol=ptasn#taedicta", + "url.path": "/tquiin/tse.jpg", + "url.query": "ovol=ptasn", + "url.scheme": "https", "user.name": "exercita", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -1355,7 +1457,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "amcol", "url.original": "https://mail.example.org/olor/ineavo.gif?mquelau=iadolor#amcol", + "url.path": "/olor/ineavo.gif", + "url.query": "mquelau=iadolor", + "url.scheme": "https", "user.name": "str", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -1431,7 +1539,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "seq", "url.original": "https://www.example.com/its/ender.gif?oles=edic#seq", + "url.path": "/its/ender.gif", + "url.query": "oles=edic", + "url.scheme": "https", "user.name": "aturve", "user_agent.device.name": "Samsung SM-S337TL", "user_agent.name": "Chrome Mobile", @@ -1507,7 +1621,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "psum", "url.original": "https://www5.example.org/oeni/tdol.gif?llamco=nea#psum", + "url.path": "/oeni/tdol.gif", + "url.query": "llamco=nea", + "url.scheme": "https", "user.name": "ulapar", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -1583,7 +1703,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "lor", "url.original": "https://www.example.com/uasiar/utlab.htm?loremqu=dantium#lor", + "url.path": "/uasiar/utlab.htm", + "url.query": "loremqu=dantium", + "url.scheme": "https", "user.name": "lor", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -1659,7 +1785,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "quamest", "url.original": "https://api.example.org/icabo/gna.html?urerepr=eseru#quamest", + "url.path": "/icabo/gna.html", + "url.query": "urerepr=eseru", + "url.scheme": "https", "user.name": "snost", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -1735,7 +1867,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "onsequ", "url.original": "https://api.example.org/oremi/elites.html?iosa=boNemoe#onsequ", + "url.path": "/oremi/elites.html", + "url.query": "iosa=boNemoe", + "url.scheme": "https", "user.name": "olori", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1787,8 +1925,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ulpa", "rsa.misc.action": [ - "Allowed", - "gnaal" + "gnaal", + "Allowed" ], "rsa.misc.category": "nte", "rsa.misc.filter": "pid", @@ -1811,7 +1949,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "dqu", "url.original": "https://example.com/luptatem/uaeratv.gif?dat=periam#dqu", + "url.path": "/luptatem/uaeratv.gif", + "url.query": "dat=periam", + "url.scheme": "https", "user.name": "fugi", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -1887,7 +2031,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "umetMalo", "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo", + "url.path": "/tanimi/rumSecti.jpg", + "url.query": "emporain=ntiumto", + "url.scheme": "https", "user.name": "boreetdo", "user_agent.device.name": "Samsung SM-A715F", "user_agent.name": "Facebook", @@ -1963,7 +2113,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "ulla", "url.original": "https://api.example.com/ori/tconsect.html?ercit=eporroq#ulla", + "url.path": "/ori/tconsect.html", + "url.query": "ercit=eporroq", + "url.scheme": "https", "user.name": "agnaaliq", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -2039,7 +2195,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "utfug", "url.original": "https://api.example.org/rure/asiarchi.txt?loremeu=aturve#utfug", + "url.path": "/rure/asiarchi.txt", + "url.query": "loremeu=aturve", + "url.scheme": "https", "user.name": "onse", "user_agent.device.name": "POCOPHONE F1", "user_agent.name": "Chrome Mobile", @@ -2115,7 +2277,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "htm", + "url.fragment": "labore", "url.original": "https://example.com/emUte/molestia.htm?orroqu=elitsed#labore", + "url.path": "/emUte/molestia.htm", + "url.query": "orroqu=elitsed", + "url.scheme": "https", "user.name": "Cic", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -2191,7 +2359,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "tionula", "url.original": "https://mail.example.net/repreh/plic.jpg?utlabo=tetur#tionula", + "url.path": "/repreh/plic.jpg", + "url.query": "utlabo=tetur", + "url.scheme": "https", "user.name": "ueipsa", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -2267,7 +2441,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ema", "url.original": "https://api.example.com/tcu/iatqu.jpg?quovo=urExcep#ema", + "url.path": "/tcu/iatqu.jpg", + "url.query": "quovo=urExcep", + "url.scheme": "https", "user.name": "rroqu", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -2319,8 +2499,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mag", "rsa.misc.action": [ - "Allowed", - "tali" + "tali", + "Allowed" ], "rsa.misc.category": "oconse", "rsa.misc.filter": "npr", @@ -2343,7 +2523,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "lupta", "url.original": "https://www5.example.com/apariatu/lorsita.gif?msequ=uat#lupta", + "url.path": "/apariatu/lorsita.gif", + "url.query": "msequ=uat", + "url.scheme": "https", "user.name": "stenatus", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -2419,7 +2605,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "tesseq", "url.original": "https://internal.example.net/ende/abor.jpg?riameaqu=ame#tesseq", + "url.path": "/ende/abor.jpg", + "url.query": "riameaqu=ame", + "url.scheme": "https", "user.name": "itasp", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -2495,7 +2687,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "txt", + "url.fragment": "mvolupta", "url.original": "https://example.com/ame/amvolu.txt?equaturv=lamc#mvolupta", + "url.path": "/ame/amvolu.txt", + "url.query": "equaturv=lamc", + "url.scheme": "https", "user.name": "iusmodt", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -2571,7 +2769,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "nisi", "url.original": "https://mail.example.org/uisnostr/reetdol.txt?ugi=niamquis#nisi", + "url.path": "/uisnostr/reetdol.txt", + "url.query": "ugi=niamquis", + "url.scheme": "https", "user.name": "mUteni", "user_agent.device.name": "STK-L21", "user_agent.name": "Chrome Mobile", @@ -2647,7 +2851,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "stiaecon", "url.original": "https://api.example.com/radipis/cive.gif?orumSec=nisiuta#stiaecon", + "url.path": "/radipis/cive.gif", + "url.query": "orumSec=nisiuta", + "url.scheme": "https", "user.name": "umdolo", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -2723,7 +2933,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "eursinto", "url.original": "https://mail.example.org/oin/itseddoe.html?citati=uamei#eursinto", + "url.path": "/oin/itseddoe.html", + "url.query": "citati=uamei", + "url.scheme": "https", "user.name": "olori", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -2799,7 +3015,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "html", + "url.fragment": "umS", "url.original": "https://mail.example.com/eseruntm/lpaquiof.html?magnaal=uscip#umS", + "url.path": "/eseruntm/lpaquiof.html", + "url.query": "magnaal=uscip", + "url.scheme": "https", "user.name": "etur", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -2875,7 +3097,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.net", + "url.extension": "html", + "url.fragment": "ine", "url.original": "https://example.net/snulap/enimadm.html?writte=sitvo#ine", + "url.path": "/snulap/enimadm.html", + "url.query": "writte=sitvo", + "url.scheme": "https", "user.name": "isau", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -2927,8 +3155,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "idolores", "rsa.misc.action": [ - "Blocked", - "lestia" + "lestia", + "Blocked" ], "rsa.misc.category": "risni", "rsa.misc.filter": "emacc", @@ -2951,7 +3179,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "ents", "url.original": "https://internal.example.org/etcon/onsequu.gif?Bonoru=madminim#ents", + "url.path": "/etcon/onsequu.gif", + "url.query": "Bonoru=madminim", + "url.scheme": "https", "user.name": "eiusmo", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -3027,7 +3261,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "seos", "url.original": "https://www.example.org/rur/itse.gif?pisciv=fugiatqu#seos", + "url.path": "/rur/itse.gif", + "url.query": "pisciv=fugiatqu", + "url.scheme": "https", "user.name": "rios", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -3103,7 +3343,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "liquipex", "url.original": "https://mail.example.com/qui/equeporr.jpg?itsedd=texpli#liquipex", + "url.path": "/qui/equeporr.jpg", + "url.query": "itsedd=texpli", + "url.scheme": "https", "user.name": "CSe", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -3179,7 +3425,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "eacommo", "url.original": "https://api.example.net/lup/iumtotam.html?ipitlabo=userror#eacommo", + "url.path": "/lup/iumtotam.html", + "url.query": "ipitlabo=userror", + "url.scheme": "https", "user.name": "Utenima", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -3255,7 +3507,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "umdolore", "url.original": "https://example.org/onproide/uamnih.htm?tatisetq=uidolo#umdolore", + "url.path": "/onproide/uamnih.htm", + "url.query": "tatisetq=uidolo", + "url.scheme": "https", "user.name": "reet", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -3331,7 +3589,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "emip", "url.original": "https://www5.example.org/obeataev/umf.htm?moll=quaeabil#emip", + "url.path": "/obeataev/umf.htm", + "url.query": "moll=quaeabil", + "url.scheme": "https", "user.name": "orissus", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -3407,7 +3671,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "leumiur", "url.original": "https://mail.example.org/consequa/eaqueip.gif?aevitaed=byCic#leumiur", + "url.path": "/consequa/eaqueip.gif", + "url.query": "aevitaed=byCic", + "url.scheme": "https", "user.name": "psaquaea", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -3483,7 +3753,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "uptasnul", "url.original": "https://example.org/tmo/onofdeF.txt?oremip=its#uptasnul", + "url.path": "/tmo/onofdeF.txt", + "url.query": "oremip=its", + "url.scheme": "https", "user.name": "dentsunt", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -3535,8 +3811,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "itesse", "rsa.misc.action": [ - "Allowed", - "uip" + "uip", + "Allowed" ], "rsa.misc.category": "teturad", "rsa.misc.filter": "roquisqu", @@ -3559,7 +3835,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "smodtem", "url.original": "https://mail.example.com/orsitvol/ntor.htm?itqu=minimav#smodtem", + "url.path": "/orsitvol/ntor.htm", + "url.query": "itqu=minimav", + "url.scheme": "https", "user.name": "taliq", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -3611,8 +3893,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mipsumq", "rsa.misc.action": [ - "citation", - "Allowed" + "Allowed", + "citation" ], "rsa.misc.category": "usant", "rsa.misc.filter": "Nem", @@ -3635,7 +3917,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "gna", "url.original": "https://internal.example.org/rumexe/xerci.gif?olor=quiav#gna", + "url.path": "/rumexe/xerci.gif", + "url.query": "olor=quiav", + "url.scheme": "https", "user.name": "lamcolab", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -3709,7 +3997,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "equuntu", "url.original": "https://example.org/tvolu/dutper.html?nbyCicer=scipit#equuntu", + "url.path": "/tvolu/dutper.html", + "url.query": "nbyCicer=scipit", + "url.scheme": "https", "user.name": "ute", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -3785,7 +4079,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "nul", "url.original": "https://www.example.com/uiavo/uisaut.htm?paq=uianon#nul", + "url.path": "/uiavo/uisaut.htm", + "url.query": "paq=uianon", + "url.scheme": "https", "user.name": "ssec", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -3861,7 +4161,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "upt", "url.original": "https://mail.example.com/uasiarch/Malor.jpg?iinea=snos#upt", + "url.path": "/uasiarch/Malor.jpg", + "url.query": "iinea=snos", + "url.scheme": "https", "user.name": "sci", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -3935,7 +4241,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "com", "url.original": "https://api.example.org/ntiumt/sumquia.jpg?lam=asnu#com", + "url.path": "/ntiumt/sumquia.jpg", + "url.query": "lam=asnu", + "url.scheme": "https", "user.name": "duntut", "user_agent.device.name": "Other", "user_agent.name": "Other", @@ -4007,7 +4319,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "todi", "url.original": "https://example.com/itsedqu/paq.jpg?hilmol=oluptate#todi", + "url.path": "/itsedqu/paq.jpg", + "url.query": "hilmol=oluptate", + "url.scheme": "https", "user.name": "picia", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -4044,8 +4362,8 @@ "tamr1693.api.home" ], "related.ip": [ - "10.53.191.49", - "10.133.102.57" + "10.133.102.57", + "10.53.191.49" ], "related.user": [ "onsec" @@ -4083,7 +4401,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "ctionofd", "url.original": "https://api.example.org/remeum/etur.html?Quisa=quiav#ctionofd", + "url.path": "/remeum/etur.html", + "url.query": "Quisa=quiav", + "url.scheme": "https", "user.name": "onsec", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -4159,7 +4483,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "nostrum", "url.original": "https://internal.example.org/ree/itten.gif?rsp=imipsa#nostrum", + "url.path": "/ree/itten.gif", + "url.query": "rsp=imipsa", + "url.scheme": "https", "user.name": "tem", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -4235,7 +4565,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "onemulla", "url.original": "https://www.example.net/ritquiin/reseo.jpg?ari=umtot#onemulla", + "url.path": "/ritquiin/reseo.jpg", + "url.query": "ari=umtot", + "url.scheme": "https", "user.name": "uasiarch", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -4311,7 +4647,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "eavolupt", "url.original": "https://mail.example.com/dexe/nemul.jpg?yCicero=inimave#eavolupt", + "url.path": "/dexe/nemul.jpg", + "url.query": "yCicero=inimave", + "url.scheme": "https", "user.name": "inrepreh", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -4363,8 +4705,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uamquaer", "rsa.misc.action": [ - "Blocked", - "aerat" + "aerat", + "Blocked" ], "rsa.misc.category": "quela", "rsa.misc.filter": "qui", @@ -4387,7 +4729,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "sistena", "url.original": "https://mail.example.org/caecat/uel.html?enim=umq#sistena", + "url.path": "/caecat/uel.html", + "url.query": "enim=umq", + "url.scheme": "https", "user.name": "olup", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -4461,7 +4809,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "rna", "url.original": "https://mail.example.com/runtmoll/busBon.txt?ionev=vitaedi#rna", + "url.path": "/runtmoll/busBon.txt", + "url.query": "ionev=vitaedi", + "url.scheme": "https", "user.name": "oremeu", "user_agent.device.name": "XiaoMi Redmi 4X", "user_agent.name": "MiuiBrowser", @@ -4537,7 +4891,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "quam", "url.original": "https://api.example.com/lits/tvolu.jpg?squir=gnaaliq#quam", + "url.path": "/lits/tvolu.jpg", + "url.query": "squir=gnaaliq", + "url.scheme": "https", "user.name": "tsedquia", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -4613,7 +4973,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "nder", "url.original": "https://mail.example.net/loremi/queporro.jpg?ade=nihilmol#nder", + "url.path": "/loremi/queporro.jpg", + "url.query": "ade=nihilmol", + "url.scheme": "https", "user.name": "gnamali", "user_agent.device.name": "Other", "user_agent.name": "Other", @@ -4685,7 +5051,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iad", "url.original": "https://api.example.com/nge/psum.gif?exerci=isnostru#iad", + "url.path": "/nge/psum.gif", + "url.query": "exerci=isnostru", + "url.scheme": "https", "user.name": "emoen", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -4761,7 +5133,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "onorumet", "url.original": "https://internal.example.org/ddoeiusm/apa.txt?uptatemU=rem#onorumet", + "url.path": "/ddoeiusm/apa.txt", + "url.query": "uptatemU=rem", + "url.scheme": "https", "user.name": "prehend", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -4835,7 +5213,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "htm", + "url.fragment": "quam", "url.original": "https://www5.example.net/yCice/uinesci.htm?taevitae=dminimv#quam", + "url.path": "/yCice/uinesci.htm", + "url.query": "taevitae=dminimv", + "url.scheme": "https", "user.name": "abo", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", @@ -4911,7 +5295,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "txt", + "url.fragment": "nse", "url.original": "https://api.example.com/itinvolu/adeserun.txt?tinv=Utenima#nse", + "url.path": "/itinvolu/adeserun.txt", + "url.query": "tinv=Utenima", + "url.scheme": "https", "user.name": "equamn", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -4948,8 +5338,8 @@ "onproi4354.www5.invalid" ], "related.ip": [ - "10.185.107.27", - "10.29.162.157" + "10.29.162.157", + "10.185.107.27" ], "related.user": [ "evelite" @@ -4987,7 +5377,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "loreeuf", "url.original": "https://www.example.org/sci/isquames.gif?tlabor=itecto#loreeuf", + "url.path": "/sci/isquames.gif", + "url.query": "tlabor=itecto", + "url.scheme": "https", "user.name": "evelite", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -5063,7 +5459,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "usmod", "url.original": "https://mail.example.org/umdolo/nimv.htm?equunt=tutla#usmod", + "url.path": "/umdolo/nimv.htm", + "url.query": "equunt=tutla", + "url.scheme": "https", "user.name": "eavolupt", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", @@ -5100,8 +5502,8 @@ "rvelill1981.www.invalid" ], "related.ip": [ - "10.26.115.88", - "10.12.130.224" + "10.12.130.224", + "10.26.115.88" ], "related.user": [ "Nequepo" @@ -5139,7 +5541,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "byCice", "url.original": "https://mail.example.net/tvol/ostru.htm?oei=iquipex#byCice", + "url.path": "/tvol/ostru.htm", + "url.query": "oei=iquipex", + "url.scheme": "https", "user.name": "Nequepo", "user_agent.device.name": "STK-L21", "user_agent.name": "Chrome Mobile", @@ -5215,7 +5623,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "atemacc", "url.original": "https://mail.example.org/pariatur/cita.html?equuntur=rve#atemacc", + "url.path": "/pariatur/cita.html", + "url.query": "equuntur=rve", + "url.scheme": "https", "user.name": "edict", "user_agent.device.name": "QMobile X700 PRO II", "user_agent.name": "Chrome Mobile", @@ -5267,8 +5681,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "omnisi", "rsa.misc.action": [ - "userro", - "Allowed" + "Allowed", + "userro" ], "rsa.misc.category": "etd", "rsa.misc.filter": "loremeum", @@ -5291,7 +5705,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "elillumq", "url.original": "https://example.org/aturE/aaliqu.gif?nvol=doloreeu#elillumq", + "url.path": "/aturE/aaliqu.gif", + "url.query": "nvol=doloreeu", + "url.scheme": "https", "user.name": "quia", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -5367,7 +5787,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "dese", "url.original": "https://internal.example.org/olorin/orisnisi.gif?eritquii=atevelit#dese", + "url.path": "/olorin/orisnisi.gif", + "url.query": "eritquii=atevelit", + "url.scheme": "https", "user.name": "isciveli", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -5443,7 +5869,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "deo", "url.original": "https://example.org/vel/preh.html?sequamni=edutpers#deo", + "url.path": "/vel/preh.html", + "url.query": "sequamni=edutpers", + "url.scheme": "https", "user.name": "estla", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -5519,7 +5951,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "aUtenim", "url.original": "https://example.com/nonproi/dolor.jpg?molli=oeiusm#aUtenim", + "url.path": "/nonproi/dolor.jpg", + "url.query": "molli=oeiusm", + "url.scheme": "https", "user.name": "litesse", "user_agent.device.name": "Samsung SM-A305FN", "user_agent.name": "YandexSearch", @@ -5595,7 +6033,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "uepor", "url.original": "https://example.com/sedqui/iuntNe.gif?epteu=nvent#uepor", + "url.path": "/sedqui/iuntNe.gif", + "url.query": "epteu=nvent", + "url.scheme": "https", "user.name": "ntore", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -5671,7 +6115,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "imidest", "url.original": "https://www5.example.net/lits/Nemoen.txt?elillu=seruntmo#imidest", + "url.path": "/lits/Nemoen.txt", + "url.query": "elillu=seruntmo", + "url.scheme": "https", "user.name": "squir", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -5747,7 +6197,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "ipis", "url.original": "https://example.org/eius/evo.jpg?iarchit=volupt#ipis", + "url.path": "/eius/evo.jpg", + "url.query": "iarchit=volupt", + "url.scheme": "https", "user.name": "mes", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -5823,7 +6279,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "uiineavo", "url.original": "https://www5.example.net/equun/veli.gif?tem=iadeseru#uiineavo", + "url.path": "/equun/veli.gif", + "url.query": "tem=iadeseru", + "url.scheme": "https", "user.name": "sequatDu", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -5899,7 +6361,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "rese", "url.original": "https://internal.example.com/ecatcu/tMalo.txt?nse=rauto#rese", + "url.path": "/ecatcu/tMalo.txt", + "url.query": "nse=rauto", + "url.scheme": "https", "user.name": "nre", "user_agent.device.name": "Samsung SM-A305FN", "user_agent.name": "YandexSearch", @@ -5975,7 +6443,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "tionevol", "url.original": "https://mail.example.org/ntutlabo/leumiure.htm?eacommo=amqua#tionevol", + "url.path": "/ntutlabo/leumiure.htm", + "url.query": "eacommo=amqua", + "url.scheme": "https", "user.name": "ten", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6051,7 +6525,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "mquis", "url.original": "https://www.example.org/ugitsed/ritatis.jpg?xplic=stenat#mquis", + "url.path": "/ugitsed/ritatis.jpg", + "url.query": "xplic=stenat", + "url.scheme": "https", "user.name": "umwr", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -6127,7 +6607,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "cidu", "url.original": "https://mail.example.com/aute/dictasu.gif?ptas=iadolo#cidu", + "url.path": "/aute/dictasu.gif", + "url.query": "ptas=iadolo", + "url.scheme": "https", "user.name": "evita", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -6164,8 +6650,8 @@ "rsitame4049.internal.corp" ], "related.ip": [ - "10.77.102.206", - "10.34.98.144" + "10.34.98.144", + "10.77.102.206" ], "related.user": [ "tectobe" @@ -6203,7 +6689,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "psamvolu", "url.original": "https://mail.example.net/enbyCic/aturau.gif?orroqui=sci#psamvolu", + "url.path": "/enbyCic/aturau.gif", + "url.query": "orroqui=sci", + "url.scheme": "https", "user.name": "tectobe", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -6255,8 +6747,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "essequa", "rsa.misc.action": [ - "Blocked", - "odic" + "odic", + "Blocked" ], "rsa.misc.category": "cto", "rsa.misc.filter": "odite", @@ -6279,7 +6771,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "aeratvol", "url.original": "https://example.org/olu/mqua.txt?mdolore=ita#aeratvol", + "url.path": "/olu/mqua.txt", + "url.query": "mdolore=ita", + "url.scheme": "https", "user.name": "isnos", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -6355,7 +6853,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "ccusa", "url.original": "https://api.example.net/mnisiut/eabil.jpg?psumqui=trude#ccusa", + "url.path": "/mnisiut/eabil.jpg", + "url.query": "psumqui=trude", + "url.scheme": "https", "user.name": "redolo", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6431,7 +6935,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "ntNeque", "url.original": "https://api.example.net/uamestq/eetdol.html?ctionofd=uianonnu#ntNeque", + "url.path": "/uamestq/eetdol.html", + "url.query": "ctionofd=uianonnu", + "url.scheme": "https", "user.name": "colab", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -6507,7 +7017,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "gnido", "url.original": "https://mail.example.net/ius/msequ.jpg?ptat=tionula#gnido", + "url.path": "/ius/msequ.jpg", + "url.query": "ptat=tionula", + "url.scheme": "https", "user.name": "umdolo", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -6583,7 +7099,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "umqu", "url.original": "https://www.example.org/iat/acom.html?umdolo=oluptass#umqu", + "url.path": "/iat/acom.html", + "url.query": "umdolo=oluptass", + "url.scheme": "https", "user.name": "mnisiuta", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6620,8 +7142,8 @@ "archite4407.mail.invalid" ], "related.ip": [ - "10.234.34.40", - "10.247.255.107" + "10.247.255.107", + "10.234.34.40" ], "related.user": [ "aeabillo" @@ -6659,7 +7181,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "trumexer", "url.original": "https://www.example.com/onorum/umiure.gif?lites=admini#trumexer", + "url.path": "/onorum/umiure.gif", + "url.query": "lites=admini", + "url.scheme": "https", "user.name": "aeabillo", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -6735,7 +7263,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "dquiac", "url.original": "https://mail.example.org/veni/rspi.htm?ntium=imadmi#dquiac", + "url.path": "/veni/rspi.htm", + "url.query": "ntium=imadmi", + "url.scheme": "https", "user.name": "tNequ", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -6811,7 +7345,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "beat", "url.original": "https://www.example.com/tem/litsedq.htm?ium=utfugit#beat", + "url.path": "/tem/litsedq.htm", + "url.query": "ium=utfugit", + "url.scheme": "https", "user.name": "uptat", "user_agent.device.name": "Spider", "user_agent.name": "Other", @@ -6883,7 +7423,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "laborisn", "url.original": "https://www.example.com/amcola/eumiurer.gif?stiaeco=equu#laborisn", + "url.path": "/amcola/eumiurer.gif", + "url.query": "stiaeco=equu", + "url.scheme": "https", "user.name": "sintocca", "user_agent.device.name": "Spider", "user_agent.name": "Other", @@ -6955,7 +7501,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "veleum", "url.original": "https://api.example.net/sedquian/lamcorpo.html?sequatD=Nequepo#veleum", + "url.path": "/sedquian/lamcorpo.html", + "url.query": "sequatD=Nequepo", + "url.scheme": "https", "user.name": "usm", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -7031,7 +7583,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "uisnostr", "url.original": "https://www5.example.com/ictasun/iumto.txt?erro=admin#uisnostr", + "url.path": "/ictasun/iumto.txt", + "url.query": "erro=admin", + "url.scheme": "https", "user.name": "ptassit", "user_agent.device.name": "Samsung SM-A715F", "user_agent.name": "Facebook", @@ -7107,7 +7665,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "isi", "url.original": "https://www5.example.org/oriosa/ssusc.htm?atemacc=rsitvolu#isi", + "url.path": "/oriosa/ssusc.htm", + "url.query": "atemacc=rsitvolu", + "url.scheme": "https", "user.name": "eroi", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -7183,7 +7747,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "ame", "url.original": "https://mail.example.net/tseddoei/byCi.gif?assitas=nul#ame", + "url.path": "/tseddoei/byCi.gif", + "url.query": "assitas=nul", + "url.scheme": "https", "user.name": "unt", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -7220,8 +7790,8 @@ "oremeum4231.internal.host" ], "related.ip": [ - "10.139.90.218", - "10.131.81.172" + "10.131.81.172", + "10.139.90.218" ], "related.user": [ "hende" @@ -7259,7 +7829,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "suntin", "url.original": "https://www5.example.org/liquipe/rehe.gif?niamqu=uioffi#suntin", + "url.path": "/liquipe/rehe.gif", + "url.query": "niamqu=uioffi", + "url.scheme": "https", "user.name": "hende", "user_agent.device.name": "Samsung GT-P3100 ", "user_agent.name": "Android", @@ -7335,7 +7911,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "siuta", "url.original": "https://www.example.org/erit/asiarch.gif?tdolor=oremagna#siuta", + "url.path": "/erit/asiarch.gif", + "url.query": "tdolor=oremagna", + "url.scheme": "https", "user.name": "mquiado", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -7411,7 +7993,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "ainci", "url.original": "https://mail.example.org/maven/tectob.jpg?litsedd=mnis#ainci", + "url.path": "/maven/tectob.jpg", + "url.query": "litsedd=mnis", + "url.scheme": "https", "user.name": "uisa", "user_agent.device.name": "QMobile X700 PRO II", "user_agent.name": "Chrome Mobile", @@ -7487,7 +8075,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "tNe", "url.original": "https://www.example.com/siarch/oloremi.htm?one=iduntutl#tNe", + "url.path": "/siarch/oloremi.htm", + "url.query": "one=iduntutl", + "url.scheme": "https", "user.name": "eprehen", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -7563,7 +8157,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "umtota", "url.original": "https://mail.example.com/ostr/liqu.txt?niam=mullamc#umtota", + "url.path": "/ostr/liqu.txt", + "url.query": "niam=mullamc", + "url.scheme": "https", "user.name": "ore", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", From d90b7d419032a79fe2aba2da0579ca1959d5a397 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Sun, 28 Mar 2021 17:47:59 +0000 Subject: [PATCH 08/16] additional fixes --- .../module/apache/error/ingest/pipeline.yml | 13 ++------ .../apache/error/test/test.log-expected.json | 12 +++---- .../test/ubuntu-2.2.22.log-expected.json | 33 +++++++------------ .../module/iis/access/ingest/pipeline.yml | 5 +++ .../iis/access/test/test.log-expected.json | 1 + filebeat/module/iis/error/ingest/pipeline.yml | 5 +++ .../test/iis_error_url.log-expected.json | 9 +++-- 7 files changed, 35 insertions(+), 43 deletions(-) diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index f6e1abc05749..367e4440d4e6 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -20,19 +20,10 @@ processors: - grok: field: message patterns: - - "File does not exist: %{URIPATH:_tmp.url_orig}, referer: %{URI:http.request.referrer}" - - "File does not exist: %{URIPATH:_tmp.url_orig}" + - "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}" + - "File does not exist: %{URIPATH:file.path}" ignore_missing: true ignore_failure: true -- uri_parts: - field: _tmp.url_orig - ignore_failure: true - if: ctx._tmp?.url_orig != null -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - if: ctx._tmp?.url?.original != null - urldecode: field: http.request.referrer ignore_missing: true diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json index 95155ce5f88d..6bbb6067469b 100644 --- a/filebeat/module/apache/error/test/test.log-expected.json +++ b/filebeat/module/apache/error/test/test.log-expected.json @@ -8,6 +8,7 @@ "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -15,10 +16,7 @@ "message": "File does not exist: /var/www/favicon.ico", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.extension": "ico", - "url.original": "/var/www/favicon.ico", - "url.path": "/var/www/favicon.ico" + "source.ip": "192.168.33.1" }, { "@timestamp": "2016-12-26T16:15:55.103-02:00", @@ -48,6 +46,7 @@ "event.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/usr/local/apache2/htdocs/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -67,10 +66,7 @@ "source.geo.location.lon": -84.8154, "source.geo.region_iso_code": "US-GA", "source.geo.region_name": "Georgia", - "source.ip": "72.15.99.187", - "url.extension": "ico", - "url.original": "/usr/local/apache2/htdocs/favicon.ico", - "url.path": "/usr/local/apache2/htdocs/favicon.ico" + "source.ip": "72.15.99.187" }, { "@timestamp": "2019-06-27T06:58:09.169-02:00", diff --git a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json index 7d2d731ecc6c..c946f0c739ad 100644 --- a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json @@ -24,6 +24,7 @@ "event.original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "http.request.referrer": "http://192.168.33.72/", "input.type": "log", @@ -32,10 +33,7 @@ "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.extension": "ico", - "url.original": "/var/www/favicon.ico", - "url.path": "/var/www/favicon.ico" + "source.ip": "192.168.33.1" }, { "@timestamp": "2016-12-26T16:22:08.000-02:00", @@ -46,6 +44,7 @@ "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -53,10 +52,7 @@ "message": "File does not exist: /var/www/favicon.ico", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.extension": "ico", - "url.original": "/var/www/favicon.ico", - "url.path": "/var/www/favicon.ico" + "source.ip": "192.168.33.1" }, { "@timestamp": "2016-12-26T16:22:08.000-02:00", @@ -67,6 +63,7 @@ "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -74,10 +71,7 @@ "message": "File does not exist: /var/www/favicon.ico", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.extension": "ico", - "url.original": "/var/www/favicon.ico", - "url.path": "/var/www/favicon.ico" + "source.ip": "192.168.33.1" }, { "@timestamp": "2016-12-26T16:22:10.000-02:00", @@ -88,6 +82,7 @@ "event.original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/test", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -95,9 +90,7 @@ "message": "File does not exist: /var/www/test", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.original": "/var/www/test", - "url.path": "/var/www/test" + "source.ip": "192.168.33.1" }, { "@timestamp": "2016-12-26T16:22:13.000-02:00", @@ -108,6 +101,7 @@ "event.original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/hello", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -115,9 +109,7 @@ "message": "File does not exist: /var/www/hello", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.original": "/var/www/hello", - "url.path": "/var/www/hello" + "source.ip": "192.168.33.1" }, { "@timestamp": "2016-12-26T16:22:17.000-02:00", @@ -128,6 +120,7 @@ "event.original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/crap", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -135,8 +128,6 @@ "message": "File does not exist: /var/www/crap", "service.type": "apache", "source.address": "192.168.33.1", - "source.ip": "192.168.33.1", - "url.original": "/var/www/crap", - "url.path": "/var/www/crap" + "source.ip": "192.168.33.1" } ] \ No newline at end of file diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 2426360f1a17..73bc5bd47903 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -44,6 +44,11 @@ processors: - uri_parts: field: _tmp.url_path ignore_failure: true +- set: + field: url.original + value: "{{_tmp.url_path}}" + ignore_failure: true + if: ctx?._tmp?.url_path != null && ctx?.url?.original == null - urldecode: field: url.original ignore_missing: true diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index ca7809e8571d..cd809c61bc7e 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -226,6 +226,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index f5ba07111d88..eee6b827317f 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -22,6 +22,11 @@ processors: - uri_parts: field: _tmp.url_orig ignore_failure: true +- set: + field: url.original + value: "{{_tmp.url_orig}}" + ignore_failure: true + if: ctx?._tmp?.url_orig != null && ctx?.url?.original == null - urldecode: field: url.original ignore_missing: true diff --git a/filebeat/module/iis/error/test/iis_error_url.log-expected.json b/filebeat/module/iis/error/test/iis_error_url.log-expected.json index fddcdc433525..ebf47868ce5c 100644 --- a/filebeat/module/iis/error/test/iis_error_url.log-expected.json +++ b/filebeat/module/iis/error/test/iis_error_url.log-expected.json @@ -117,7 +117,8 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", - "source.port": 12345 + "source.port": 12345, + "url.original": "/..\\\\pixfir~1\\\\how_to_login.html" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -155,7 +156,8 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", - "source.port": 12345 + "source.port": 12345, + "url.original": "..\\\\..\\\\..\\\\..\\\\..\\\\..\\\\winnt\\\\win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -234,7 +236,8 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", - "source.port": 12345 + "source.port": 12345, + "url.original": "/nessus\\\\..\\\\..\\\\..\\\\..\\\\..\\\\..\\\\winnt\\\\win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", From 2af6e090f2202234901351fcc27ee6299691d264 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Mon, 5 Apr 2021 18:24:01 +0000 Subject: [PATCH 09/16] update pipelines --- .../apache/access/test/test.log-expected.json | 1 + .../access/test/access.log-expected.json | 2 + .../test/test.log-expected.json | 1 + .../meraki/test/generated.log-expected.json | 48 +++++++++---------- .../module/cisco/umbrella/ingest/pipeline.yml | 2 + ...brella-cloudfirewalllogs.log-expected.json | 2 + 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index c0a03f36aa48..2ef2c7a74160 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -175,6 +175,7 @@ "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user.name": "-", "user_agent.device.name": "Other", + "user_agent.device.type": "Desktop", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "user_agent.os.full": "Windows 7", diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index f0f3078e8132..48da3b70c109 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -600,6 +600,7 @@ "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Kindle", + "user_agent.device.type": "Tablet", "user_agent.name": "Amazon Silk", "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", "user_agent.os.full": "Android 5.1.1", @@ -643,6 +644,7 @@ "url.original": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", "user_agent.device.name": "Kindle", + "user_agent.device.type": "Tablet", "user_agent.name": "Amazon Silk", "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", "user_agent.os.full": "Android 5.1.1", diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index a934be97f15f..bf4843ca9aa4 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1391,6 +1391,7 @@ "url.original": "/v2/some", "url.path": "/v2/some", "user_agent.device.name": "Mac", + "user_agent.device.type": "Desktop", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json index 041ff5b2a743..773dfeca049d 100644 --- a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json @@ -137,8 +137,8 @@ "appliance" ], "related.ip": [ - "10.155.236.240", - "10.112.46.169" + "10.112.46.169", + "10.155.236.240" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -962,8 +962,8 @@ "appliance" ], "related.ip": [ - "10.135.217.12", - "10.93.68.231" + "10.93.68.231", + "10.135.217.12" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1094,8 +1094,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.221.102.245", - "10.173.136.186" + "10.173.136.186", + "10.221.102.245" ], "rsa.internal.event_desc": "idestlab", "rsa.internal.messageid": "security_event", @@ -1402,8 +1402,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.150.245.88", - "10.242.77.170" + "10.242.77.170", + "10.150.245.88" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1501,8 +1501,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.147.15.213", - "10.94.6.140" + "10.94.6.140", + "10.147.15.213" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1540,8 +1540,8 @@ "appliance" ], "related.ip": [ - "10.111.157.56", - "10.230.6.127" + "10.230.6.127", + "10.111.157.56" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1582,8 +1582,8 @@ "appliance" ], "related.ip": [ - "10.179.40.170", - "10.193.219.34" + "10.193.219.34", + "10.179.40.170" ], "rsa.counters.dclass_r1": "emip", "rsa.internal.messageid": "events", @@ -1904,8 +1904,8 @@ "remips188.api.invalid" ], "related.ip": [ - "10.78.199.43", - "10.40.101.224" + "10.40.101.224", + "10.78.199.43" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2209,8 +2209,8 @@ "uames4985.mail.localdomain" ], "related.ip": [ - "10.144.57.239", - "10.150.163.151" + "10.150.163.151", + "10.144.57.239" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2755,8 +2755,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.244.32.189", - "10.121.9.5" + "10.121.9.5", + "10.244.32.189" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -3067,8 +3067,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.199.103.185", - "10.51.121.223" + "10.51.121.223", + "10.199.103.185" ], "rsa.internal.event_desc": "dipi security_event ecatc", "rsa.internal.messageid": "security_event", @@ -3407,8 +3407,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.137.166.97", - "10.162.202.14" + "10.162.202.14", + "10.137.166.97" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml index 2a602ff23317..203126319862 100644 --- a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -198,6 +198,8 @@ processors: - "(?:%{IP:destination.ip}|%{GREEDYDATA:destination.domain})" ignore_failure: true +- community_id: + ignore_missing: true ###################### # Related ECS Fields # ###################### diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json index 65aabab5a88d..6cedb19cced9 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json @@ -16,6 +16,7 @@ "input.type": "log", "log.offset": 0, "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "network.community_id": "1:9s/mgn3WChavy6kpCk25Ed4VCcU=", "network.direction": "outbound", "network.transport": "1", "observer.product": "Umbrella", @@ -52,6 +53,7 @@ "input.type": "log", "log.offset": 128, "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "network.community_id": "1:9s/mgn3WChavy6kpCk25Ed4VCcU=", "network.direction": "inbound", "network.transport": "1", "observer.product": "Umbrella", From 24d5811e9f5abf51955083aba199ac0ed4c8a7a2 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 6 Apr 2021 16:13:25 +0000 Subject: [PATCH 10/16] unescape \ --- filebeat/module/iis/error/ingest/pipeline.yml | 2 +- .../module/iis/error/test/iis_error_url.log-expected.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index eee6b827317f..4fc628e1cfb4 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -24,7 +24,7 @@ processors: ignore_failure: true - set: field: url.original - value: "{{_tmp.url_orig}}" + value: "{{{_tmp.url_orig}}}" ignore_failure: true if: ctx?._tmp?.url_orig != null && ctx?.url?.original == null - urldecode: diff --git a/filebeat/module/iis/error/test/iis_error_url.log-expected.json b/filebeat/module/iis/error/test/iis_error_url.log-expected.json index ebf47868ce5c..dcfa5353878e 100644 --- a/filebeat/module/iis/error/test/iis_error_url.log-expected.json +++ b/filebeat/module/iis/error/test/iis_error_url.log-expected.json @@ -118,7 +118,7 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/..\\\\pixfir~1\\\\how_to_login.html" + "url.original": "/..\\pixfir~1\\how_to_login.html" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -157,7 +157,7 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "..\\\\..\\\\..\\\\..\\\\..\\\\..\\\\winnt\\\\win.ini" + "url.original": "..\\..\\..\\..\\..\\..\\winnt\\win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -237,7 +237,7 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/nessus\\\\..\\\\..\\\\..\\\\..\\\\..\\\\..\\\\winnt\\\\win.ini" + "url.original": "/nessus\\..\\..\\..\\..\\..\\..\\winnt\\win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", From e936522ab43ceb156f99c6fcefc5387f84fc013f Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 8 Apr 2021 23:03:58 +0000 Subject: [PATCH 11/16] remove urldecodes for url.original --- CHANGELOG.next.asciidoc | 1 + .../module/apache/access/ingest/pipeline.yml | 4 -- .../access/test/ssl-request.log-expected.json | 4 +- .../apache/access/test/test.log-expected.json | 2 +- .../module/apache/error/ingest/pipeline.yml | 1 - .../module/iis/access/ingest/pipeline.yml | 4 -- filebeat/module/iis/error/ingest/pipeline.yml | 4 -- .../module/nginx/access/ingest/pipeline.yml | 4 -- .../access/test/access.log-expected.json | 4 +- .../ingress_controller/ingest/pipeline.yml | 35 +++++++++++++++++ .../test/test.log-expected.json | 4 +- .../module/traefik/access/ingest/pipeline.yml | 4 -- .../module/cisco/meraki/ingest/pipeline.yml | 4 -- .../cisco/shared/ingest/asa-ftd-pipeline.yml | 4 -- .../module/cisco/umbrella/ingest/pipeline.yml | 5 +++ .../test/umbrella-proxylogs.log-expected.json | 16 ++++++-- .../module/f5/bigipafm/ingest/pipeline.yml | 4 -- .../module/f5/bigipapm/ingest/pipeline.yml | 4 -- .../fortinet/fortimanager/ingest/pipeline.yml | 4 -- .../admin-sites-test.json.log-expected.json | 12 ++++-- .../module/google_workspace/ingest/common.yml | 4 ++ .../imperva/securesphere/ingest/pipeline.yml | 4 -- .../defender_atp/ingest/pipeline.yml | 4 ++ .../m365_defender/ingest/pipeline.yml | 5 ++- .../netscout/sightline/ingest/pipeline.yml | 4 -- .../module/o365/audit/ingest/pipeline.yml | 4 -- .../test/11-dlp-sharepoint.log-expected.json | 14 +++---- .../module/sophos/utm/ingest/pipeline.yml | 4 -- .../module/sophos/xg/ingest/antivirus.yml | 4 -- .../module/sophos/xg/ingest/cfilter.yml | 38 ++++++++++++++++--- .../filebeat/module/sophos/xg/ingest/waf.yml | 38 ++++++++++++++++--- .../sophos/xg/test/cfilter.log-expected.json | 35 ++++++++++++++--- .../sophos/xg/test/waf.log-expected.json | 13 ++++++- .../module/squid/log/ingest/pipeline.yml | 4 -- .../module/suricata/eve/ingest/http.yml | 4 -- .../module/zeek/sip/ingest/pipeline.yml | 9 +++++ .../module/zoom/webhook/ingest/meeting.yml | 16 ++++++-- .../module/zoom/webhook/ingest/phone.yml | 4 ++ .../module/zoom/webhook/ingest/recording.yml | 4 ++ .../test/meeting.ndjson.log-expected.json | 8 ++++ .../test/phone.ndjson.log-expected.json | 5 +++ .../test/recording.ndjson.log-expected.json | 20 ++++++++++ .../module/zscaler/zia/ingest/pipeline.yml | 4 -- 43 files changed, 258 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0fe7883d70af..1fc8f53e75e8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -849,6 +849,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] - Update URI decoding and parsing across multiple modules. {issue}19088[19088] {pull}24699[24699] +- Add `uri_parts` processor to multiple modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 5befc027d630..588d22b0937e 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -27,10 +27,6 @@ processors: - uri_parts: field: _tmp.url_orig ignore_failure: true -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index c815ede3c46c..6b4288724b18 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -21,7 +21,7 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "url.extension": "php", - "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", "url.path": "/nagiosxi/ajaxhelper.php", "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" }, @@ -51,7 +51,7 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "url.extension": "php", - "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", "url.path": "/appl/ajaxhelper.php", "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" } diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index 2ef2c7a74160..9c5f9e04f52c 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -171,7 +171,7 @@ "source.address": "monitoring-server", "source.domain": "monitoring-server", "url.extension": "mp4", - "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user.name": "-", "user_agent.device.name": "Other", diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index 367e4440d4e6..ee6ef739bdc6 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -28,7 +28,6 @@ processors: field: http.request.referrer ignore_missing: true ignore_failure: true - if: ctx._tmp?.http?.request?.referrer != null - date: if: ctx.event.timezone == null field: apache.error.timestamp diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 73bc5bd47903..1f1bd7ab2305 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -49,10 +49,6 @@ processors: value: "{{_tmp.url_path}}" ignore_failure: true if: ctx?._tmp?.url_path != null && ctx?.url?.original == null -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4fc628e1cfb4..6a09d937fed0 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -27,10 +27,6 @@ processors: value: "{{{_tmp.url_orig}}}" ignore_failure: true if: ctx?._tmp?.url_orig != null && ctx?.url?.original == null -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - remove: field: - _tmp diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index db8c54f8e28d..fc2e93d699b5 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -28,10 +28,6 @@ processors: - uri_parts: field: _tmp.url_orig ignore_failure: true -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index 48da3b70c109..9ae1da3d91e8 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -597,7 +597,7 @@ "source.ip": "192.168.0.1", "url.domain": "lessons.example.com", "url.extension": "mp4", - "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Kindle", "user_agent.device.type": "Tablet", @@ -641,7 +641,7 @@ "source.address": "192.168.0.1", "source.ip": "192.168.0.1", "url.domain": "lessons.example.com", - "url.original": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", + "url.original": "/%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/", "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", "user_agent.device.name": "Kindle", "user_agent.device.type": "Tablet", diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 64fd7567ba1e..96f170ac14a2 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -31,6 +31,17 @@ processors: - "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}" - "" ignore_missing: true + - uri_parts: + field: url.original + ignore_failure: true + - urldecode: + field: http.request.referrer + ignore_missing: true + ignore_failure: true + - set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: field: nginx.ingress_controller.info - split: @@ -266,6 +277,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index bf4843ca9aa4..47bdacc80f59 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1133,7 +1133,7 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42?address=delhi technological university", + "url.original": "/products/42?address=delhi+technological+university", "url.path": "/products/42", "url.query": "address=delhi+technological+university", "user_agent.device.name": "Other", @@ -1456,7 +1456,7 @@ "source.address": "192.168.64.14", "source.ip": "192.168.64.14", "url.extension": "mp4", - "url.original": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index 482367c66438..840bff74192c 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -23,10 +23,6 @@ processors: - uri_parts: field: temp.url_orig ignore_failure: true -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml index 0065fe86f8e5..5257eb1956a8 100644 --- a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml @@ -29,10 +29,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index 86dfb21c7289..59cdbe38fe7c 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1662,10 +1662,6 @@ processors: ignore_failure: true allow_duplicates: false if: ctx?._temp_?.url_domain != null - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml index 203126319862..b659b2fd5edd 100644 --- a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -120,6 +120,11 @@ processors: value: firewall if: ctx?.log?.file?.path.contains('cloudfirewalllogs') +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null + # Identifies is a field that includes any sort of username, device or other asset that is included in the request. # Converting this to an array to make it easier to use in searches and visualizations - split: diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json index fd474d2d0293..4f0b9552eb48 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json @@ -34,7 +34,10 @@ "source.address": "192.168.1.1", "source.ip": "192.168.1.1", "source.nat.ip": "1.1.1.1", - "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "url.domain": "elastic.co", + "url.original": "https://elastic.co/blog/ext_id=Anyclip", + "url.path": "/blog/ext_id=Anyclip", + "url.scheme": "https", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" }, { @@ -72,7 +75,10 @@ "source.address": "192.168.1.1", "source.ip": "192.168.1.1", "source.nat.ip": "1.1.1.1", - "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "url.domain": "elastic.co", + "url.original": "https://elastic.co/blog/ext_id=Anyclip", + "url.path": "/blog/ext_id=Anyclip", + "url.scheme": "https", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" }, { @@ -109,7 +115,11 @@ "source.address": "ActiveDirectoryUserName,ADSite,Network", "source.domain": "ActiveDirectoryUserName,ADSite,Network", "source.nat.ip": "192.192.192.135", - "url.full": "http://google.com/the.js", + "url.domain": "google.com", + "url.extension": "js", + "url.original": "http://google.com/the.js", + "url.path": "/the.js", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml index 43486b2e0570..f47b01f10cd2 100644 --- a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 2c878a87f973..33e5ebbe399d 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml index 5ee19350ce83..0391eea6ae97 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json b/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json index 23436a2de5fc..1e9362659942 100644 --- a/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json +++ b/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json @@ -50,8 +50,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" @@ -107,8 +109,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" @@ -219,8 +223,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" diff --git a/x-pack/filebeat/module/google_workspace/ingest/common.yml b/x-pack/filebeat/module/google_workspace/ingest/common.yml index ec7a9f8bbcfc..d9ee4e9f86c6 100644 --- a/x-pack/filebeat/module/google_workspace/ingest/common.yml +++ b/x-pack/filebeat/module/google_workspace/ingest/common.yml @@ -1,5 +1,9 @@ description: Pipeline for parsing google_workspace logs processors: + - uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null - geoip: field: source.ip target_field: source.geo diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index 18e4409c4245..621c1255bd8d 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index b04d4b5d67fc..aa8e3f976928 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -232,6 +232,10 @@ processors: ignore_missing: true if: ctx?.json?.evidence?.url != null +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS Host Mapping ## ###################### diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index ae33c77d6d5e..5103d12de9df 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -221,7 +221,10 @@ processors: target_field: url.full ignore_missing: true if: ctx?.json?.entities?.url != null - +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS User Mapping ## ###################### diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 12ba933ea0ec..cabc18d6a058 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 681ae839bbc3..c032cf0bf301 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -31,10 +31,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json index 710d6ad3a8b0..c440645b3604 100644 --- a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json @@ -82,7 +82,7 @@ ], "url.domain": "testsiem2-my.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", @@ -182,7 +182,7 @@ ], "url.domain": "testsiem2-my.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", @@ -278,7 +278,7 @@ ], "url.domain": "testsiem2-my.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", @@ -378,7 +378,7 @@ ], "url.domain": "testsiem2-my.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", @@ -478,7 +478,7 @@ ], "url.domain": "testsiem2-my.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL CREDIT CARD NUMBERS.docx", + "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL%20CREDIT%20CARD%20NUMBERS.docx", "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL CREDIT CARD NUMBERS.docx", "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", @@ -578,7 +578,7 @@ ], "url.domain": "testsiem2.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared Documents/Document.docx", + "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", @@ -678,7 +678,7 @@ ], "url.domain": "testsiem2.sharepoint.com", "url.extension": "docx", - "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared Documents/Document.docx", + "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml index bc5967880a50..7cb1b02a18ac 100644 --- a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml index 287892f11902..6eb60e460bbc 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml @@ -215,10 +215,6 @@ processors: - remove: field: _temp_ ignore_missing: true -- urldecode: - field: url.original - ignore_missing: true - ignore_failure: true ############################ ## ECS User Agent Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml index 0b5f92c1e699..7b4853ce5bf4 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml @@ -122,16 +122,19 @@ processors: ##################### ## ECS URL Mapping ## ##################### +- uri_parts: + field: sophos.xg.url + ignore_failure: true + if: "ctx.sophos?.xg?.url != null" - rename: field: sophos.xg.url target_field: url.full ignore_missing: true if: "ctx.sophos?.xg?.url != null" -- rename: - field: sophos.xg.domain - target_field: url.domain - ignore_missing: true - if: "ctx.sophos?.xg?.domain != null" +- set: + field: url.domain + value: "{{sophos.xg.domain}}" + ignore_empty_value: true ############################ ## ECS User Agent Mapping ## @@ -268,7 +271,32 @@ processors: - sophos.xg.src_port - sophos.xg.recv_bytes - sophos.xg.sent_bytes + - sophos.xg.domain ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml index e0ea89c41dc7..6462ac5d6bf2 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml @@ -118,16 +118,19 @@ processors: ##################### ## ECS URL Mapping ## ##################### +- uri_parts: + field: sophos.xg.url + ignore_failure: true + if: "ctx.sophos?.xg?.url != null" - rename: field: sophos.xg.url target_field: url.full ignore_missing: true if: "ctx.sophos?.xg?.url != null" -- rename: - field: sophos.xg.domain - target_field: url.domain - ignore_missing: true - if: "ctx.sophos?.xg?.domain != null" +- set: + field: url.domain + value: "{{sophos.xg.domain}}" + ignore_empty_value: true ############################ ## ECS User Agent Mapping ## @@ -279,7 +282,32 @@ processors: - sophos.xg.bytesrcv - sophos.xg.bytessent - sophos.xg.httpstatus + - sophos.xg.domain ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json index c8bb6001058b..3194c309b5b7 100644 --- a/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json @@ -71,7 +71,10 @@ "forwarded" ], "url.domain": "r8---sn-ci5gup-qxas.googlevideo.com", - "url.full": "https://r8---sn-ci5gup-qxas.googlevideo.com/" + "url.full": "https://r8---sn-ci5gup-qxas.googlevideo.com/", + "url.original": "https://r8---sn-ci5gup-qxas.googlevideo.com/", + "url.path": "/", + "url.scheme": "https" }, { "@timestamp": "2017-02-01T18:20:21.000-02:00", @@ -152,7 +155,10 @@ "forwarded" ], "url.domain": "hanuman.com", - "url.full": "http://hanuman.com/" + "url.full": "http://hanuman.com/", + "url.original": "http://hanuman.com/", + "url.path": "/", + "url.scheme": "http" }, { "@timestamp": "2017-02-01T18:13:29.000-02:00", @@ -309,7 +315,10 @@ "forwarded" ], "url.domain": "his-eur1-neur1.servicebus.windows.net", - "url.full": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket" + "url.full": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket", + "url.original": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket", + "url.path": "/$servicebus/websocket", + "url.scheme": "https" }, { "@timestamp": "2020-05-18T14:38:52.000-02:00", @@ -386,7 +395,10 @@ "forwarded" ], "url.domain": "40.90.137.127", - "url.full": "https://40.90.137.127/" + "url.full": "https://40.90.137.127/", + "url.original": "https://40.90.137.127/", + "url.path": "/", + "url.scheme": "https" }, { "@timestamp": "2020-05-18T14:38:53.000-02:00", @@ -462,7 +474,11 @@ "forwarded" ], "url.domain": "update.eset.com", + "url.extension": "signed", "url.full": "http://update.eset.com/eset_upd/ep7/dll/update.ver.signed", + "url.original": "http://update.eset.com/eset_upd/ep7/dll/update.ver.signed", + "url.path": "/eset_upd/ep7/dll/update.ver.signed", + "url.scheme": "http", "user_agent.original": "EFSW Update (Windows; U; 64bit; BPC 7.1.12010.0; OS: 10.0.17763 SP 0.0 NT; TDB 45511; CL 1.1.1; x64s; APP efsw; PX 1; PUA 1; CD 1; RA 1; PEV 0; UNS 1; UBR 1158; HVCI 0; SHA256 1; WU 3; HWF: 01009DAA-757A-D666-EFD2-92DD0D501284; PLOC de_de; PCODE 211.0.0; " }, { @@ -592,7 +608,10 @@ "forwarded" ], "url.domain": "www.google.com", - "url.full": "http://www.google.com/" + "url.full": "http://www.google.com/", + "url.original": "http://www.google.com/", + "url.path": "/", + "url.scheme": "http" }, { "@timestamp": "2016-12-02T18:50:22.000-02:00", @@ -669,6 +688,10 @@ "forwarded" ], "url.domain": "www.google.ca", - "url.full": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw" + "url.full": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.original": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.path": "/", + "url.query": "gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.scheme": "http" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json index 055f255a15a1..9950baa2a62a 100644 --- a/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json @@ -81,6 +81,8 @@ "forwarded" ], "url.full": "/mapi/nspi/", + "url.original": "/mapi/nspi/", + "url.path": "/mapi/nspi/", "user_agent.original": "Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.4954; Pro)" }, { @@ -166,6 +168,8 @@ "forwarded" ], "url.full": "/mapi/nspi/", + "url.original": "/mapi/nspi/", + "url.path": "/mapi/nspi/", "user_agent.original": "Microsoft Office/16.0 (Windows NT 6.2; Microsoft Outlook 16.0.4954; Pro)" }, { @@ -236,6 +240,8 @@ "forwarded" ], "url.full": "/", + "url.original": "/", + "url.path": "/", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" }, { @@ -308,7 +314,10 @@ "sophos-xg", "forwarded" ], + "url.extension": "zip", "url.full": "/download/eicarcom2.zip", + "url.original": "/download/eicarcom2.zip", + "url.path": "/download/eicarcom2.zip", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" }, { @@ -392,6 +401,8 @@ "sophos-xg", "forwarded" ], - "url.full": "/" + "url.full": "/", + "url.original": "/", + "url.path": "/" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index 7bf0484a46f2..bc79b7991361 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true diff --git a/x-pack/filebeat/module/suricata/eve/ingest/http.yml b/x-pack/filebeat/module/suricata/eve/ingest/http.yml index b1420dd42cf5..7dcb7d4b4ef5 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/http.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/http.yml @@ -61,10 +61,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - rename: field: suricata.eve.http.http_refer target_field: http.request.referrer diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index d8408c511333..5af1cd6ae59a 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -20,6 +20,15 @@ processors: - remove: field: zeek.sip.seq ignore_missing: true +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null +- set: + field: url.full + copy_from: url.original + ignore_failure: true + if: ctx?.url?.original != null - geoip: field: destination.ip target_field: destination.geo diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml index 9291add35933..62f05fa073f2 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml @@ -38,14 +38,22 @@ processors: ignore_missing: true - rename: field: zoom.meeting.join_url - target_field: url.full + target_field: url.original ignore_missing: true - rename: field: zoom.registrant.join_url - target_field: url.full + target_field: url.original ignore_missing: true - if: ctx?.url?.full == null - + if: ctx?.original?.full == null +- uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null +- set: + field: url.full + copy_from: url.original + ignore_failure: true + if: ctx?.url?.original != null # # Set user.* from participant, if any. # diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml b/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml index b836cd9c96c4..e5fca522f334 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml @@ -19,6 +19,10 @@ processors: field: zoom.object target_field: zoom.phone ignore_missing: true +- uri_parts: + field: zoom.phone.download_url + ignore_failure: true + if: ctx?.zoom?.phone?.download_url != null - rename: field: zoom.phone.download_url target_field: url.full diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml b/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml index 715f46bcbd93..f9e5b1c30422 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml @@ -35,6 +35,10 @@ processors: field: zoom.object target_field: zoom.recording ignore_missing: true +- uri_parts: + field: zoom.recording.share_url + ignore_failure: true + if: ctx?.zoom?.recording?.share_url != null - rename: field: zoom.recording.share_url target_field: url.full diff --git a/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json index 123de911c51b..723a37a326fd 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json @@ -99,7 +99,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/j/00000000", + "url.original": "https://zoom.us/j/00000000", + "url.path": "/j/00000000", + "url.scheme": "https", "user.email": "someemail@email.com", "user.id": "BBBBBBBBBB", "zoom.account_id": "AAAAAAAAAAA", @@ -254,7 +258,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/w/someendpointhere", + "url.original": "https://zoom.us/w/someendpointhere", + "url.path": "/w/someendpointhere", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.meeting.host_id": "uLobbbbbbbbbb_qQsQ", diff --git a/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json index 507943735b29..ee24b279b3c1 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json @@ -320,7 +320,12 @@ "zoom-webhook", "forwarded" ], + "url.domain": "testurl.com", + "url.extension": "mp4", "url.full": "https://testurl.com/file.mp4", + "url.original": "https://testurl.com/file.mp4", + "url.path": "/file.mp4", + "url.scheme": "https", "zoom.account_id": "test", "zoom.phone.callee.extension_type": "2", "zoom.phone.callee.id": "1234", diff --git a/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json index f9be7349ab4f..a0296121e1a8 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json @@ -175,7 +175,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -254,7 +258,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -293,7 +301,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -332,7 +344,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -371,7 +387,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index f9dec729bc44..7f869714ff45 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -34,10 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: url.original - ignore_missing: true - ignore_failure: true - urldecode: field: http.request.referrer ignore_missing: true From c42a73eee1b9dd796b04e3b361d3e2ff38f69df4 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 9 Apr 2021 00:21:09 +0000 Subject: [PATCH 12/16] updates after rebase --- .../module/nginx/ingress_controller/ingest/pipeline.yml | 7 ++++--- .../nginx/ingress_controller/test/test.log-expected.json | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 96f170ac14a2..3ef6cb3e7414 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -5,8 +5,11 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - grok: + - rename: field: message + target_field: event.original + - grok: + field: event.original patterns: - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" @@ -209,8 +212,6 @@ processors: patterns: - ^%{IP:source.ip}$ ignore_failure: true - - remove: - field: message - rename: field: "@timestamp" target_field: event.created diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 47bdacc80f59..0074ce0553b7 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1414,6 +1414,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "835136ae24486dbb4156dcbe21f5d402", "http.request.method": "GET", "http.response.body.bytes": 61, "http.response.status_code": 200, From e49a4c45f9beb4642d7a9cc1fec55fb4bd28cd9e Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 9 Apr 2021 13:33:21 +0000 Subject: [PATCH 13/16] update zeek SIP --- .../module/zeek/sip/ingest/pipeline.yml | 36 ++++++++++++++----- .../zeek/sip/test/sip-json.log-expected.json | 9 +++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index 5af1cd6ae59a..045d5afe760b 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -20,15 +20,11 @@ processors: - remove: field: zeek.sip.seq ignore_missing: true -- uri_parts: - field: url.full - ignore_failure: true - if: ctx?.url?.full != null -- set: +- grok: field: url.full - copy_from: url.original - ignore_failure: true - if: ctx?.url?.original != null + patterns: + - '%{WORD:url.scheme}:(%{USERNAME:url.username}@)?%{IPORHOST:url.domain}(:%{POSINT:url.port:long})?' + ignore_missing: true - geoip: field: destination.ip target_field: destination.geo @@ -89,6 +85,30 @@ processors: field: event.outcome value: success if: "ctx?.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code < 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json index 71061cd293bc..a9e15731ebc3 100644 --- a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json @@ -42,7 +42,10 @@ "tags": [ "zeek.sip" ], + "url.domain": "newyork.voip.ms", "url.full": "sip:newyork.voip.ms:5060", + "url.port": 5060, + "url.scheme": "sip", "zeek.session_id": "CPRLCB4eWHdjP852Bk", "zeek.sip.call_id": "8694cd7e-976e4fc3-d76f6e38@172.16.133.19", "zeek.sip.request.body_length": 0, @@ -120,7 +123,11 @@ "tags": [ "zeek.sip" ], + "url.domain": "bestel.com", "url.full": "sip:francisco@bestel.com:55060", + "url.port": 55060, + "url.scheme": "sip", + "url.username": "francisco", "zeek.session_id": "ComJz236lSOcuOmix3", "zeek.sip.call_id": "12013223@200.57.7.195", "zeek.sip.request.body_length": 229, @@ -201,7 +208,9 @@ "tags": [ "zeek.sip" ], + "url.domain": "Verso.com", "url.full": "sip:Verso.com", + "url.scheme": "sip", "zeek.session_id": "CJZDWgixtwqXctWEg", "zeek.sip.call_id": "46E1C3CB36304F84A020CF6DD3F96461@Verso.com", "zeek.sip.request.body_length": 0, From e4fe9b8aea6d73dcedc2f84ee977e46486d22b76 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Sat, 17 Apr 2021 22:00:20 +0000 Subject: [PATCH 14/16] update changelog as requested by @andrewstucki --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1fc8f53e75e8..7ea7243aa488 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] +- All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699] *Heartbeat* From aabbfa3fde6daa3380dbabbf7e56aedfe82036c9 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 20 Apr 2021 02:13:30 +0000 Subject: [PATCH 15/16] remove `url_decode` for `http.request.referrer` --- CHANGELOG.next.asciidoc | 3 +- .../module/apache/access/ingest/pipeline.yml | 4 - .../apache/access/test/test.log-expected.json | 1 - .../module/apache/error/ingest/pipeline.yml | 4 - .../module/iis/access/ingest/pipeline.yml | 4 - .../module/nginx/access/ingest/pipeline.yml | 4 - .../access/test/access.log-expected.json | 6 +- .../ingress_controller/ingest/pipeline.yml | 4 - .../test/test.log-expected.json | 1 - .../module/traefik/access/ingest/pipeline.yml | 4 - .../additional_messages.log-expected.json | 10 +- .../module/cisco/meraki/ingest/pipeline.yml | 4 - .../meraki/test/generated.log-expected.json | 76 +-- .../cisco/shared/ingest/asa-ftd-pipeline.yml | 4 - .../module/f5/bigipafm/ingest/pipeline.yml | 3 - .../bigipafm/test/generated.log-expected.json | 438 +++++++++--------- .../module/f5/bigipapm/ingest/pipeline.yml | 3 - .../test/generated.log-expected.json | 132 +++--- .../test/generated.log-expected.json | 16 +- .../fortinet/fortimanager/ingest/pipeline.yml | 3 - .../test/generated.log-expected.json | 323 +++++++------ .../imperva/securesphere/ingest/pipeline.yml | 3 - .../test/generated.log-expected.json | 335 +++++++------- .../netscout/sightline/ingest/pipeline.yml | 3 - .../test/generated.log-expected.json | 16 +- .../module/o365/audit/ingest/pipeline.yml | 3 - .../06-sharepointfileop.log-expected.json | 12 + .../test/25-ms-teams-groups.log-expected.json | 6 + .../module/sophos/utm/ingest/pipeline.yml | 3 - .../utm/test/generated.log-expected.json | 60 ++- .../module/squid/log/ingest/pipeline.yml | 3 - .../squid/log/test/access1.log-expected.json | 184 ++++---- .../log/test/generated.log-expected.json | 312 ++++++------- .../module/suricata/eve/ingest/http.yml | 3 - .../intel/test/intel-json.log-expected.json | 4 +- .../module/zscaler/zia/ingest/pipeline.yml | 3 - .../zia/test/generated.log-expected.json | 252 +++++----- 37 files changed, 1097 insertions(+), 1152 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7ea7243aa488..5c3b795b3c50 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -849,8 +849,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] -- Update URI decoding and parsing across multiple modules. {issue}19088[19088] {pull}24699[24699] -- Add `uri_parts` processor to multiple modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] +- Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 588d22b0937e..37543d119e33 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -27,10 +27,6 @@ processors: - uri_parts: field: _tmp.url_orig ignore_failure: true -- urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true - set: field: url.domain value: "{{destination.domain}}" diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index 9c5f9e04f52c..84bf4ea81a13 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -175,7 +175,6 @@ "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user.name": "-", "user_agent.device.name": "Other", - "user_agent.device.type": "Desktop", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "user_agent.os.full": "Windows 7", diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index ee6ef739bdc6..4b8495dd9c89 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -24,10 +24,6 @@ processors: - "File does not exist: %{URIPATH:file.path}" ignore_missing: true ignore_failure: true -- urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true - date: if: ctx.event.timezone == null field: apache.error.timestamp diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 1f1bd7ab2305..afbd20c78baf 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -49,10 +49,6 @@ processors: value: "{{_tmp.url_path}}" ignore_failure: true if: ctx?._tmp?.url_path != null && ctx?.url?.original == null -- urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true - urldecode: field: _tmp.url_query target_field: url.query diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index fc2e93d699b5..4a597e30d391 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -28,10 +28,6 @@ processors: - uri_parts: field: _tmp.url_orig ignore_failure: true -- urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true - set: field: url.domain value: "{{destination.domain}}" diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index 9ae1da3d91e8..6a0af6f499c7 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -580,7 +580,7 @@ ], "fileset.name": "access", "http.request.method": "GET", - "http.request.referrer": "http://lessons.example.com/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", "http.response.body.bytes": 7648063, "http.response.status_code": 206, "http.version": "1.1", @@ -600,7 +600,6 @@ "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Kindle", - "user_agent.device.type": "Tablet", "user_agent.name": "Amazon Silk", "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", "user_agent.os.full": "Android 5.1.1", @@ -625,7 +624,7 @@ ], "fileset.name": "access", "http.request.method": "GET", - "http.request.referrer": "http://lessons.example.com/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", "http.response.body.bytes": 7648063, "http.response.status_code": 206, "http.version": "1.1", @@ -644,7 +643,6 @@ "url.original": "/%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/", "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", "user_agent.device.name": "Kindle", - "user_agent.device.type": "Tablet", "user_agent.name": "Amazon Silk", "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", "user_agent.os.full": "Android 5.1.1", diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 3ef6cb3e7414..14a5da9cae96 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -37,10 +37,6 @@ processors: - uri_parts: field: url.original ignore_failure: true - - urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true - set: field: url.domain value: "{{destination.domain}}" diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 0074ce0553b7..7a9e13bf58ea 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1391,7 +1391,6 @@ "url.original": "/v2/some", "url.path": "/v2/some", "user_agent.device.name": "Mac", - "user_agent.device.type": "Desktop", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", "user_agent.os.full": "Mac OS X 10.14", diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index 840bff74192c..3b63a05fe8d1 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -23,10 +23,6 @@ processors: - uri_parts: field: temp.url_orig ignore_failure: true -- urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true - rename: field: '@timestamp' target_field: event.created diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 652da70d3512..7f641d9becdb 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -2546,7 +2546,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/IOFUHSIU98[0]" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/IOFUHSIU98[0]", + "url.path": "/IOFUHSIU98[0]", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2590,7 +2593,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23", + "url.path": "/some/longer/url-asd-er9789870[0]_=23", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", diff --git a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml index 5257eb1956a8..b80d31afad41 100644 --- a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml @@ -29,10 +29,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json index 773dfeca049d..9871b8013dde 100644 --- a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json @@ -137,8 +137,8 @@ "appliance" ], "related.ip": [ - "10.112.46.169", - "10.155.236.240" + "10.155.236.240", + "10.112.46.169" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -587,8 +587,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.187.77.245", - "10.88.231.224" + "10.88.231.224", + "10.187.77.245" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -662,8 +662,8 @@ "appliance" ], "related.ip": [ - "10.205.47.51", - "10.219.84.37" + "10.219.84.37", + "10.205.47.51" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -734,8 +734,8 @@ "appliance" ], "related.ip": [ - "10.182.178.217", - "10.63.194.87" + "10.63.194.87", + "10.182.178.217" ], "rsa.counters.dclass_r1": "fdeFi", "rsa.internal.messageid": "events", @@ -1094,8 +1094,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.173.136.186", - "10.221.102.245" + "10.221.102.245", + "10.173.136.186" ], "rsa.internal.event_desc": "idestlab", "rsa.internal.messageid": "security_event", @@ -1133,8 +1133,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.54.37.86", - "10.58.64.108" + "10.58.64.108", + "10.54.37.86" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1214,8 +1214,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.0.200.27", - "10.183.44.198" + "10.183.44.198", + "10.0.200.27" ], "rsa.internal.event_desc": "uradi security_event tot", "rsa.internal.messageid": "security_event", @@ -1402,8 +1402,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.242.77.170", - "10.150.245.88" + "10.150.245.88", + "10.242.77.170" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1501,8 +1501,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.94.6.140", - "10.147.15.213" + "10.147.15.213", + "10.94.6.140" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1540,8 +1540,8 @@ "appliance" ], "related.ip": [ - "10.230.6.127", - "10.111.157.56" + "10.111.157.56", + "10.230.6.127" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1740,8 +1740,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.124.63.4", - "10.90.99.245" + "10.90.99.245", + "10.124.63.4" ], "rsa.internal.event_desc": "etconsec", "rsa.internal.messageid": "security_event", @@ -2018,8 +2018,8 @@ "appliance" ], "related.ip": [ - "10.201.168.116", - "10.86.188.179" + "10.86.188.179", + "10.201.168.116" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2209,8 +2209,8 @@ "uames4985.mail.localdomain" ], "related.ip": [ - "10.150.163.151", - "10.144.57.239" + "10.144.57.239", + "10.150.163.151" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2383,8 +2383,8 @@ "appliance" ], "related.ip": [ - "10.103.49.129", - "10.2.110.73" + "10.2.110.73", + "10.103.49.129" ], "rsa.counters.dclass_r1": "orumS", "rsa.internal.messageid": "events", @@ -2480,8 +2480,8 @@ "lors2232.api.example" ], "related.ip": [ - "10.105.136.146", - "10.46.217.155" + "10.46.217.155", + "10.105.136.146" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2524,8 +2524,8 @@ "appliance" ], "related.ip": [ - "10.245.199.23", - "10.123.62.215" + "10.123.62.215", + "10.245.199.23" ], "rsa.db.index": "iusmodt", "rsa.internal.messageid": "flows", @@ -2755,8 +2755,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.121.9.5", - "10.244.32.189" + "10.244.32.189", + "10.121.9.5" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2929,8 +2929,8 @@ "appliance" ], "related.ip": [ - "10.17.111.91", - "10.65.0.157" + "10.65.0.157", + "10.17.111.91" ], "rsa.db.index": "nostrum", "rsa.internal.messageid": "flows", @@ -3344,8 +3344,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.147.165.30", - "10.195.90.73" + "10.195.90.73", + "10.147.165.30" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index 59cdbe38fe7c..19775fe0be0b 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1662,10 +1662,6 @@ processors: ignore_failure: true allow_duplicates: false if: ctx?._temp_?.url_domain != null - - urldecode: - field: http.request.referrer - ignore_missing: true - ignore_failure: true # # Populate ECS event.code diff --git a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml index f47b01f10cd2..ae772c9b6a33 100644 --- a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json index 0b756af87b85..76300c43427f 100644 --- a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json @@ -25,9 +25,9 @@ ], "related.ip": [ "10.165.201.71", - "10.208.121.85", + "10.11.196.142", "10.228.193.207", - "10.11.196.142" + "10.208.121.85" ], "related.user": [ "billoi" @@ -93,8 +93,8 @@ ], "related.ip": [ "10.92.202.200", - "10.51.132.10", "10.94.67.230", + "10.51.132.10", "10.162.9.235" ], "related.user": [ @@ -160,10 +160,10 @@ "gelit6728.api.invalid" ], "related.ip": [ - "10.122.116.161", - "10.209.155.149", "10.82.56.117", - "10.191.68.244" + "10.209.155.149", + "10.191.68.244", + "10.122.116.161" ], "related.user": [ "seq" @@ -227,10 +227,10 @@ "uid545.www5.localhost" ], "related.ip": [ - "10.12.44.169", - "10.50.112.141", + "10.202.66.28", "10.131.233.27", - "10.202.66.28" + "10.12.44.169", + "10.50.112.141" ], "related.user": [ "elits" @@ -295,8 +295,8 @@ "emquiavo452.internal.localhost" ], "related.ip": [ - "10.206.197.113", "10.151.111.38", + "10.206.197.113", "10.159.182.171", "10.96.35.212" ], @@ -363,9 +363,9 @@ "sun1403.www.invalid" ], "related.ip": [ - "10.126.177.162", "10.89.163.114", "10.213.113.28", + "10.126.177.162", "10.169.144.147" ], "related.user": [ @@ -430,10 +430,10 @@ "ittenbyC7838.api.localdomain" ], "related.ip": [ + "10.103.107.47", "10.146.88.52", "10.101.223.43", - "10.18.124.28", - "10.103.107.47" + "10.18.124.28" ], "related.user": [ "rudexerc" @@ -498,10 +498,10 @@ "ume465.corp" ], "related.ip": [ - "10.69.57.206", "10.150.220.75", - "10.110.99.17", - "10.189.109.245" + "10.189.109.245", + "10.69.57.206", + "10.110.99.17" ], "related.user": [ "onse" @@ -565,11 +565,10 @@ "iciatisu1463.www5.localdomain" ], "related.ip": [ - "10.153.136.222", - "10.19.194.101", "10.121.219.204", - "10.153.136.222", - "10.199.34.241" + "10.199.34.241", + "10.19.194.101", + "10.153.136.222" ], "related.user": [ "temveleu" @@ -633,10 +632,10 @@ "aliqu6801.api.localdomain" ], "related.ip": [ - "10.46.27.57", - "10.182.199.231", + "10.57.103.192", "10.64.141.105", - "10.57.103.192" + "10.182.199.231", + "10.46.27.57" ], "related.user": [ "ice" @@ -700,10 +699,10 @@ "itame189.domain" ], "related.ip": [ - "10.3.134.237", + "10.32.67.231", "10.164.6.207", - "10.160.210.31", - "10.32.67.231" + "10.3.134.237", + "10.160.210.31" ], "related.user": [ "pic" @@ -770,8 +769,8 @@ "related.ip": [ "10.235.101.253", "10.182.178.217", - "10.201.6.10", - "10.42.138.192" + "10.42.138.192", + "10.201.6.10" ], "related.user": [ "giatnu" @@ -836,10 +835,10 @@ "stlabo1228.mail.host" ], "related.ip": [ + "10.22.102.198", "10.86.101.235", "10.194.247.171", - "10.151.161.70", - "10.22.102.198" + "10.151.161.70" ], "related.user": [ "nse" @@ -904,10 +903,10 @@ "ecte4762.local" ], "related.ip": [ - "10.204.35.15", + "10.174.252.105", "10.107.168.60", "10.167.172.155", - "10.174.252.105" + "10.204.35.15" ], "related.user": [ "mnisi" @@ -971,10 +970,10 @@ "smo7167.www.test" ], "related.ip": [ - "10.182.191.174", + "10.99.249.210", "10.214.249.164", - "10.81.26.208", - "10.99.249.210" + "10.182.191.174", + "10.81.26.208" ], "related.user": [ "upta" @@ -1038,10 +1037,10 @@ "sauteiru4554.api.domain" ], "related.ip": [ + "10.201.238.90", "10.220.5.143", - "10.88.101.53", "10.101.226.128", - "10.201.238.90" + "10.88.101.53" ], "related.user": [ "porro" @@ -1105,10 +1104,10 @@ "untut4046.internal.domain" ], "related.ip": [ - "10.30.133.66", "10.243.218.215", "10.157.18.252", - "10.217.150.196" + "10.217.150.196", + "10.30.133.66" ], "related.user": [ "evit" @@ -1172,10 +1171,10 @@ "quid3147.mail.home" ], "related.ip": [ - "10.181.133.187", - "10.66.181.6", "10.167.227.44", - "10.148.161.250" + "10.148.161.250", + "10.66.181.6", + "10.181.133.187" ], "related.user": [ "adipisc" @@ -1241,9 +1240,9 @@ ], "related.ip": [ "10.107.9.163", + "10.74.11.43", "10.54.17.32", - "10.84.163.178", - "10.74.11.43" + "10.84.163.178" ], "related.user": [ "mquisno" @@ -1307,10 +1306,10 @@ "lorsita2019.internal.home" ], "related.ip": [ - "10.230.129.252", - "10.192.229.221", + "10.184.73.211", "10.112.32.213", - "10.184.73.211" + "10.230.129.252", + "10.192.229.221" ], "related.user": [ "odi" @@ -1376,9 +1375,9 @@ ], "related.ip": [ "10.7.200.140", + "10.198.213.189", "10.199.216.143", - "10.161.148.64", - "10.198.213.189" + "10.161.148.64" ], "related.user": [ "ccaeca" @@ -1443,9 +1442,9 @@ ], "related.ip": [ "10.65.232.27", + "10.128.157.27", "10.22.187.69", - "10.206.96.56", - "10.128.157.27" + "10.206.96.56" ], "related.user": [ "uaeab" @@ -1509,10 +1508,10 @@ "avolupt7576.api.corp" ], "related.ip": [ - "10.194.210.62", + "10.68.253.120", "10.71.114.14", "10.183.130.225", - "10.68.253.120" + "10.194.210.62" ], "related.user": [ "admin" @@ -1577,10 +1576,9 @@ "loi7596.www5.home" ], "related.ip": [ - "10.107.45.175", + "10.45.253.103", "10.31.177.226", "10.107.45.175", - "10.45.253.103", "10.47.255.237" ], "related.user": [ @@ -1647,9 +1645,9 @@ ], "related.ip": [ "10.213.94.135", - "10.225.212.189", + "10.55.105.113", "10.44.58.106", - "10.55.105.113" + "10.225.212.189" ], "related.user": [ "dquia" @@ -1713,10 +1711,10 @@ "ectiono2241.lan" ], "related.ip": [ - "10.2.114.9", "10.163.209.70", + "10.69.161.78", "10.255.74.136", - "10.69.161.78" + "10.2.114.9" ], "related.user": [ "olabor" @@ -1780,10 +1778,10 @@ "umetMal1664.mail.lan" ], "related.ip": [ + "10.12.129.137", "10.46.115.216", "10.184.59.148", - "10.252.102.110", - "10.12.129.137" + "10.252.102.110" ], "related.user": [ "perspici" @@ -1848,10 +1846,10 @@ "derit5270.mail.local" ], "related.ip": [ - "10.105.52.140", - "10.199.194.79", "10.155.204.243", - "10.81.184.7" + "10.199.194.79", + "10.81.184.7", + "10.105.52.140" ], "related.user": [ "eetd" @@ -1916,9 +1914,9 @@ "orisni5238.mail.lan" ], "related.ip": [ - "10.251.231.142", "10.177.238.45", "10.18.226.72", + "10.251.231.142", "10.110.2.166" ], "related.user": [ @@ -1984,10 +1982,10 @@ "iutali7297.www.domain" ], "related.ip": [ + "10.99.202.229", "10.100.199.226", "10.190.122.27", - "10.192.98.247", - "10.99.202.229" + "10.192.98.247" ], "related.user": [ "lloinven" @@ -2053,9 +2051,9 @@ ], "related.ip": [ "10.248.111.207", - "10.37.193.70", + "10.172.154.97", "10.162.97.197", - "10.172.154.97" + "10.37.193.70" ], "related.user": [ "culpaq" @@ -2119,10 +2117,10 @@ "oinv5493.internal.domain" ], "related.ip": [ - "10.36.63.31", + "10.171.221.230", "10.45.35.180", "10.222.165.250", - "10.171.221.230" + "10.36.63.31" ], "related.user": [ "otamr" @@ -2186,10 +2184,10 @@ "tnonproi195.api.home" ], "related.ip": [ - "10.238.4.219", "10.199.127.211", - "10.1.171.61", - "10.83.238.145" + "10.83.238.145", + "10.238.4.219", + "10.1.171.61" ], "related.user": [ "reetdolo" @@ -2254,9 +2252,9 @@ ], "related.ip": [ "10.170.252.219", - "10.74.213.42", + "10.44.226.104", "10.65.141.244", - "10.44.226.104" + "10.74.213.42" ], "related.user": [ "Nequepo" @@ -2320,10 +2318,10 @@ "uido492.www5.home" ], "related.ip": [ - "10.225.255.211", "10.183.223.149", "10.225.141.172", - "10.180.48.221" + "10.180.48.221", + "10.225.255.211" ], "related.user": [ "nihil" @@ -2387,9 +2385,9 @@ "redo6311.api.invalid" ], "related.ip": [ - "10.169.123.103", - "10.176.64.28", "10.205.174.181", + "10.176.64.28", + "10.169.123.103", "10.97.138.181" ], "related.user": [ @@ -2455,10 +2453,10 @@ "dolorem1698.www.domain" ], "related.ip": [ - "10.53.101.131", "10.169.101.161", - "10.204.4.40", - "10.75.120.11" + "10.75.120.11", + "10.53.101.131", + "10.204.4.40" ], "related.user": [ "tquo" @@ -2524,8 +2522,8 @@ ], "related.ip": [ "10.6.222.112", - "10.87.120.87", "10.156.117.169", + "10.87.120.87", "10.28.51.219" ], "related.user": [ @@ -2591,9 +2589,9 @@ ], "related.ip": [ "10.247.44.59", - "10.4.126.103", "10.57.89.155", - "10.253.167.17" + "10.253.167.17", + "10.4.126.103" ], "related.user": [ "ntorever" @@ -2657,10 +2655,10 @@ "olorsi2746.internal.localhost" ], "related.ip": [ - "10.143.183.208", "10.15.240.220", - "10.248.206.210", - "10.36.69.125" + "10.36.69.125", + "10.143.183.208", + "10.248.206.210" ], "related.user": [ "met" @@ -2725,9 +2723,9 @@ "edqu2208.www.localhost" ], "related.ip": [ + "10.142.186.43", "10.34.133.2", "10.69.170.107", - "10.142.186.43", "10.6.32.7" ], "related.user": [ @@ -2793,10 +2791,10 @@ "ender5647.www5.example" ], "related.ip": [ - "10.142.22.24", - "10.170.165.164", "10.121.153.197", - "10.59.103.10" + "10.142.22.24", + "10.59.103.10", + "10.170.165.164" ], "related.user": [ "borumSec" @@ -2862,9 +2860,9 @@ ], "related.ip": [ "10.247.114.30", - "10.133.10.122", "10.19.99.129", - "10.176.83.7" + "10.176.83.7", + "10.133.10.122" ], "related.user": [ "quaeabil" @@ -2929,8 +2927,8 @@ "uatu2894.api.lan" ], "related.ip": [ - "10.70.7.23", "10.8.29.219", + "10.70.7.23", "10.64.139.17", "10.40.177.138" ], @@ -2996,10 +2994,10 @@ "rmagnido5483.local" ], "related.ip": [ - "10.180.62.222", - "10.2.189.20", "10.67.173.228", - "10.67.221.220" + "10.180.62.222", + "10.67.221.220", + "10.2.189.20" ], "related.user": [ "uptasnul" @@ -3064,10 +3062,10 @@ "uian521.www.example" ], "related.ip": [ - "10.209.52.47", "10.56.134.118", - "10.147.127.181", - "10.196.176.243" + "10.209.52.47", + "10.196.176.243", + "10.147.127.181" ], "related.user": [ "tasu" @@ -3131,9 +3129,9 @@ "taliq5213.api.corp" ], "related.ip": [ + "10.226.24.84", "10.85.13.237", "10.231.18.90", - "10.226.24.84", "10.248.140.59" ], "related.user": [ @@ -3199,10 +3197,10 @@ "ntsunt4894.mail.domain" ], "related.ip": [ - "10.203.46.215", "10.59.215.207", + "10.207.183.204", "10.8.224.72", - "10.207.183.204" + "10.203.46.215" ], "related.user": [ "eruntmo" @@ -3267,10 +3265,10 @@ "mexer3864.api.corp" ], "related.ip": [ - "10.73.84.95", - "10.230.38.148", "10.98.154.146", - "10.255.145.22" + "10.73.84.95", + "10.255.145.22", + "10.230.38.148" ], "related.user": [ "sitam" @@ -3334,10 +3332,10 @@ "oluptat6960.www5.test" ], "related.ip": [ - "10.211.29.187", - "10.175.181.138", + "10.166.142.198", "10.105.120.162", - "10.166.142.198" + "10.175.181.138", + "10.211.29.187" ], "related.user": [ "tium" @@ -3403,8 +3401,8 @@ ], "related.ip": [ "10.195.139.25", - "10.122.133.162", "10.220.202.102", + "10.122.133.162", "10.182.213.195" ], "related.user": [ @@ -3470,11 +3468,10 @@ "ptat3230.domain" ], "related.ip": [ - "10.247.144.9", "10.156.208.5", - "10.33.143.163", "10.53.72.161", - "10.33.143.163" + "10.33.143.163", + "10.247.144.9" ], "related.user": [ "scip" @@ -3538,10 +3535,10 @@ "exer447.internal.localhost" ], "related.ip": [ - "10.21.58.162", "10.241.143.145", + "10.113.65.192", "10.35.190.164", - "10.113.65.192" + "10.21.58.162" ], "related.user": [ "porin" @@ -3607,8 +3604,8 @@ ], "related.ip": [ "10.19.154.103", - "10.53.27.253", "10.75.113.240", + "10.53.27.253", "10.129.16.166" ], "related.user": [ @@ -3675,8 +3672,8 @@ ], "related.ip": [ "10.22.213.196", - "10.120.50.13", "10.125.150.220", + "10.120.50.13", "10.150.153.61" ], "related.user": [ @@ -3742,10 +3739,10 @@ "edquiaco6562.api.lan" ], "related.ip": [ - "10.238.171.184", "10.113.2.13", - "10.229.155.171", - "10.85.52.249" + "10.238.171.184", + "10.85.52.249", + "10.229.155.171" ], "related.user": [ "tatiset" @@ -3810,10 +3807,10 @@ "tatis7315.mail.home" ], "related.ip": [ + "10.198.150.185", "10.249.174.35", - "10.51.245.225", "10.220.1.249", - "10.198.150.185" + "10.51.245.225" ], "related.user": [ "quela" @@ -3878,10 +3875,10 @@ "eosqui3723.api.localdomain" ], "related.ip": [ - "10.251.82.195", + "10.38.185.31", "10.152.157.32", - "10.190.96.181", - "10.38.185.31" + "10.251.82.195", + "10.190.96.181" ], "related.user": [ "olorese" @@ -3945,10 +3942,10 @@ "itaedict199.mail.corp" ], "related.ip": [ + "10.211.198.50", "10.103.102.242", - "10.190.247.194", "10.230.112.179", - "10.211.198.50" + "10.190.247.194" ], "related.user": [ "tDuisaut" @@ -4012,9 +4009,10 @@ "xeaco7887.www.localdomain" ], "related.ip": [ - "10.47.223.155", + "10.251.101.61", "10.101.13.122", - "10.251.101.61" + "10.47.223.155", + "10.219.83.199" ], "related.user": [ "ectetur" @@ -4079,10 +4077,10 @@ "saute7421.www.invalid" ], "related.ip": [ - "10.21.30.43", - "10.21.80.157", + "10.83.136.233", "10.31.86.83", - "10.83.136.233" + "10.21.80.157", + "10.21.30.43" ], "related.user": [ "litsed" @@ -4147,10 +4145,10 @@ "oluptas1637.home" ], "related.ip": [ - "10.194.197.107", - "10.27.181.27", "10.45.152.205", - "10.195.90.73" + "10.195.90.73", + "10.27.181.27", + "10.194.197.107" ], "related.user": [ "datatn" @@ -4215,10 +4213,10 @@ "ididu5505.api.localdomain" ], "related.ip": [ - "10.129.161.18", - "10.222.2.132", "10.43.239.97", - "10.183.90.25" + "10.183.90.25", + "10.222.2.132", + "10.129.161.18" ], "related.user": [ "aedicta" @@ -4282,10 +4280,10 @@ "mqui1099.api.corp" ], "related.ip": [ + "10.67.129.100", "10.248.156.138", "10.231.167.171", - "10.189.162.131", - "10.67.129.100" + "10.189.162.131" ], "related.user": [ "sedquia" @@ -4350,10 +4348,10 @@ "siuta2155.lan" ], "related.ip": [ + "10.63.103.30", "10.6.146.184", "10.142.106.66", - "10.185.107.27", - "10.63.103.30" + "10.185.107.27" ], "related.user": [ "sequu" @@ -4417,10 +4415,10 @@ "tatiset4191.localdomain" ], "related.ip": [ - "10.93.39.237", "10.214.93.200", + "10.119.179.182", "10.0.202.9", - "10.119.179.182" + "10.93.39.237" ], "related.user": [ "tionofd" @@ -4485,9 +4483,9 @@ "aute2433.mail.lan" ], "related.ip": [ + "10.30.189.166", "10.252.204.162", "10.28.145.163", - "10.30.189.166", "10.123.154.140" ], "related.user": [ @@ -4553,8 +4551,8 @@ ], "related.ip": [ "10.29.122.183", - "10.145.128.250", "10.46.162.198", + "10.145.128.250", "10.79.49.3" ], "related.user": [ @@ -4620,10 +4618,10 @@ "one7728.api.localdomain" ], "related.ip": [ + "10.166.169.167", "10.65.174.196", "10.177.232.136", - "10.142.235.217", - "10.166.169.167" + "10.142.235.217" ], "related.user": [ "olors" @@ -4688,9 +4686,9 @@ "uptatem4446.internal.localhost" ], "related.ip": [ - "10.53.188.140", - "10.29.217.44", "10.215.184.154", + "10.29.217.44", + "10.53.188.140", "10.191.78.86" ], "related.user": [ @@ -4756,10 +4754,10 @@ "emq2514.api.localhost" ], "related.ip": [ - "10.74.74.129", "10.135.77.156", - "10.46.222.149", - "10.76.148.147" + "10.74.74.129", + "10.76.148.147", + "10.46.222.149" ], "related.user": [ "urve" @@ -4823,10 +4821,10 @@ "agna5654.www.corp" ], "related.ip": [ - "10.96.200.223", + "10.145.49.29", "10.130.203.37", - "10.11.146.253", - "10.145.49.29" + "10.96.200.223", + "10.11.146.253" ], "related.user": [ "mvele" @@ -4890,8 +4888,8 @@ "ipi4827.mail.lan" ], "related.ip": [ - "10.162.78.48", "10.162.2.180", + "10.162.78.48", "10.24.23.209", "10.48.75.140" ], @@ -4957,9 +4955,9 @@ "sequatD163.internal.example" ], "related.ip": [ - "10.66.92.83", "10.151.206.38", "10.119.12.186", + "10.66.92.83", "10.97.105.115" ], "related.user": [ @@ -5025,9 +5023,9 @@ ], "related.ip": [ "10.201.132.114", - "10.64.76.142", + "10.169.139.250", "10.12.148.73", - "10.169.139.250" + "10.64.76.142" ], "related.user": [ "borisnis" @@ -5092,10 +5090,10 @@ "epr3512.internal.domain" ], "related.ip": [ - "10.111.128.11", "10.9.236.18", - "10.200.116.191", - "10.35.38.185" + "10.111.128.11", + "10.35.38.185", + "10.200.116.191" ], "related.user": [ "umfug" @@ -5159,10 +5157,10 @@ "uredol2174.home" ], "related.ip": [ - "10.240.62.238", - "10.236.67.227", + "10.134.238.8", "10.191.27.182", - "10.134.238.8" + "10.240.62.238", + "10.236.67.227" ], "related.user": [ "tlabo" @@ -5227,9 +5225,9 @@ ], "related.ip": [ "10.22.231.91", - "10.165.66.92", + "10.65.35.64", "10.109.14.142", - "10.65.35.64" + "10.165.66.92" ], "related.user": [ "perna" @@ -5293,10 +5291,10 @@ "inimav5557.www5.test" ], "related.ip": [ - "10.29.230.203", + "10.89.221.90", "10.71.112.86", "10.64.161.215", - "10.89.221.90" + "10.29.230.203" ], "related.user": [ "rnatur" @@ -5360,10 +5358,10 @@ "nonn1650.www.test" ], "related.ip": [ - "10.221.199.137", "10.140.118.182", "10.88.226.76", - "10.79.208.135" + "10.79.208.135", + "10.221.199.137" ], "related.user": [ "erspic" @@ -5429,9 +5427,9 @@ ], "related.ip": [ "10.126.61.230", + "10.133.48.55", "10.35.73.208", - "10.189.244.22", - "10.133.48.55" + "10.189.244.22" ], "related.user": [ "tia" @@ -5495,10 +5493,10 @@ "suscipit587.www.localhost" ], "related.ip": [ - "10.81.154.115", - "10.35.65.72", "10.240.94.109", - "10.239.194.105" + "10.239.194.105", + "10.81.154.115", + "10.35.65.72" ], "related.user": [ "reseo" @@ -5565,8 +5563,8 @@ "related.ip": [ "10.38.253.213", "10.248.72.104", - "10.52.70.192", - "10.150.56.227" + "10.150.56.227", + "10.52.70.192" ], "related.user": [ "ionem" @@ -5631,10 +5629,10 @@ "borios1067.www5.home" ], "related.ip": [ - "10.218.15.164", - "10.73.172.186", "10.203.193.134", - "10.62.218.239" + "10.62.218.239", + "10.218.15.164", + "10.73.172.186" ], "related.user": [ "reh" @@ -5698,10 +5696,10 @@ "msequ323.www.example" ], "related.ip": [ - "10.131.127.113", - "10.60.20.76", "10.10.46.43", - "10.136.211.234" + "10.60.20.76", + "10.136.211.234", + "10.131.127.113" ], "related.user": [ "nev" @@ -5766,10 +5764,10 @@ "tdolorem813.internal.host" ], "related.ip": [ - "10.233.181.250", - "10.187.237.220", "10.248.0.74", - "10.50.177.151" + "10.233.181.250", + "10.50.177.151", + "10.187.237.220" ], "related.user": [ "ugiatq" @@ -5834,9 +5832,9 @@ "volupt4626.internal.test" ], "related.ip": [ - "10.96.223.46", - "10.248.248.120", "10.80.129.81", + "10.248.248.120", + "10.96.223.46", "10.189.43.11" ], "related.user": [ @@ -5902,10 +5900,10 @@ "ntium5103.www5.localhost" ], "related.ip": [ - "10.66.106.186", - "10.91.115.139", "10.102.109.199", - "10.173.114.63" + "10.173.114.63", + "10.66.106.186", + "10.91.115.139" ], "related.user": [ "tNequ" @@ -5970,10 +5968,10 @@ "orpori3334.www.local" ], "related.ip": [ - "10.221.223.127", - "10.198.157.122", "10.0.175.17", - "10.159.155.88" + "10.159.155.88", + "10.221.223.127", + "10.198.157.122" ], "related.user": [ "iquipex" @@ -6038,8 +6036,8 @@ ], "related.ip": [ "10.7.212.201", - "10.189.70.237", "10.30.20.187", + "10.189.70.237", "10.252.136.130" ], "related.user": [ @@ -6106,9 +6104,9 @@ ], "related.ip": [ "10.83.105.69", + "10.242.121.165", "10.102.109.194", - "10.60.224.93", - "10.242.121.165" + "10.60.224.93" ], "related.user": [ "mni" @@ -6173,10 +6171,10 @@ "uisnostr2390.mail.domain" ], "related.ip": [ - "10.17.20.93", - "10.251.167.219", "10.181.134.69", - "10.219.174.45" + "10.219.174.45", + "10.17.20.93", + "10.251.167.219" ], "related.user": [ "Uteni" @@ -6241,9 +6239,9 @@ "luptate4811.mail.example" ], "related.ip": [ - "10.28.233.253", - "10.37.14.20", "10.30.117.82", + "10.37.14.20", + "10.28.233.253", "10.223.99.90" ], "related.user": [ @@ -6310,9 +6308,9 @@ ], "related.ip": [ "10.125.20.22", - "10.57.85.113", "10.8.32.17", - "10.50.61.114" + "10.50.61.114", + "10.57.85.113" ], "related.user": [ "qua" @@ -6377,10 +6375,10 @@ "lorinrep7686.mail.corp" ], "related.ip": [ - "10.181.63.82", - "10.200.28.55", "10.113.78.101", - "10.215.224.27" + "10.200.28.55", + "10.215.224.27", + "10.181.63.82" ], "related.user": [ "ficiade" @@ -6445,10 +6443,10 @@ "nderit6272.mail.example" ], "related.ip": [ - "10.139.20.223", - "10.169.95.128", "10.177.14.106", - "10.243.43.168" + "10.243.43.168", + "10.169.95.128", + "10.139.20.223" ], "related.user": [ "ofd" @@ -6514,8 +6512,8 @@ ], "related.ip": [ "10.90.93.4", - "10.39.100.88", "10.92.168.198", + "10.39.100.88", "10.18.176.44" ], "related.user": [ @@ -6583,8 +6581,8 @@ "related.ip": [ "10.193.43.135", "10.49.68.8", - "10.163.203.191", - "10.173.13.179" + "10.173.13.179", + "10.163.203.191" ], "related.user": [ "tlab" @@ -6648,9 +6646,9 @@ "cipitl2184.localdomain" ], "related.ip": [ - "10.240.47.113", "10.84.64.28", "10.209.226.7", + "10.240.47.113", "10.31.147.51" ], "related.user": [ @@ -6717,9 +6715,9 @@ ], "related.ip": [ "10.32.20.4", - "10.52.13.192", "10.225.189.229", - "10.86.1.244" + "10.86.1.244", + "10.52.13.192" ], "related.user": [ "odtemp" diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 33e5ebbe399d..814b2b0f30dd 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json index 5e403ecc4e6c..21e6b8a0246f 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json @@ -25,8 +25,8 @@ "boNemoe4402.www.invalid" ], "related.ip": [ - "10.150.92.220", - "10.102.123.34" + "10.102.123.34", + "10.150.92.220" ], "related.user": [ "sumdo" @@ -397,8 +397,8 @@ "reetdolo2770.www5.local" ], "related.ip": [ - "10.12.44.169", - "10.214.225.125" + "10.214.225.125", + "10.12.44.169" ], "related.user": [ "erep" @@ -521,8 +521,8 @@ "usmodte1296.www.corp" ], "related.ip": [ - "10.178.244.31", - "10.69.20.77" + "10.69.20.77", + "10.178.244.31" ], "related.user": [ "umdolor" @@ -582,8 +582,8 @@ "tatno4987.www5.localhost" ], "related.ip": [ - "10.54.231.100", - "10.203.5.162" + "10.203.5.162", + "10.54.231.100" ], "related.user": [ "umdolore" @@ -706,8 +706,8 @@ "essecill2595.mail.local" ], "related.ip": [ - "10.210.213.18", - "10.57.40.29" + "10.57.40.29", + "10.210.213.18" ], "related.user": [ "onse" @@ -952,8 +952,8 @@ "errorsi6996.www.domain" ], "related.ip": [ - "10.76.72.111", - "10.70.95.74" + "10.70.95.74", + "10.76.72.111" ], "related.user": [ "ivelits" @@ -1507,8 +1507,8 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ - "10.163.5.243", - "10.178.77.231" + "10.178.77.231", + "10.163.5.243" ], "related.user": [ "liquide" @@ -1568,8 +1568,8 @@ "ris3314.mail.invalid" ], "related.ip": [ - "10.221.89.228", - "10.177.194.18" + "10.177.194.18", + "10.221.89.228" ], "related.user": [ "aliquam" @@ -1630,8 +1630,8 @@ "reme622.mail.example" ], "related.ip": [ - "10.32.239.1", - "10.241.65.49" + "10.241.65.49", + "10.32.239.1" ], "related.user": [ "idata" @@ -1815,8 +1815,8 @@ "stquido5705.api.host" ], "related.ip": [ - "10.60.129.15", - "10.248.101.25" + "10.248.101.25", + "10.60.129.15" ], "related.user": [ "evolup" @@ -2185,8 +2185,8 @@ "eumiu765.api.lan" ], "related.ip": [ - "10.184.18.202", - "10.4.157.1" + "10.4.157.1", + "10.184.18.202" ], "related.user": [ "oditem" @@ -2371,8 +2371,8 @@ "inesci6789.test" ], "related.ip": [ - "10.167.227.44", - "10.38.54.72" + "10.38.54.72", + "10.167.227.44" ], "related.user": [ "riamea" @@ -2556,8 +2556,8 @@ "ian867.internal.corp" ], "related.ip": [ - "10.83.130.226", - "10.41.123.102" + "10.41.123.102", + "10.83.130.226" ], "related.user": [ "tenim" @@ -2742,8 +2742,8 @@ "nsequat1859.internal.localhost" ], "related.ip": [ - "10.28.118.160", - "10.223.119.218" + "10.223.119.218", + "10.28.118.160" ], "related.user": [ "ntsunt" @@ -2803,8 +2803,8 @@ "ritin2495.api.corp" ], "related.ip": [ - "10.110.114.175", - "10.47.28.48" + "10.47.28.48", + "10.110.114.175" ], "related.user": [ "plicab" @@ -3050,8 +3050,8 @@ "dqu6144.api.localhost" ], "related.ip": [ - "10.210.89.183", - "10.150.245.88" + "10.150.245.88", + "10.210.89.183" ], "related.user": [ "sequa" @@ -3112,8 +3112,8 @@ "giatquov1918.internal.example" ], "related.ip": [ - "10.85.185.13", - "10.180.195.43" + "10.180.195.43", + "10.85.185.13" ], "related.user": [ "voluptas" @@ -3236,8 +3236,8 @@ "Sedut1775.www.domain" ], "related.ip": [ - "10.86.11.48", - "10.248.165.185" + "10.248.165.185", + "10.86.11.48" ], "related.user": [ "dquiac" @@ -3484,8 +3484,8 @@ "volupta3552.internal.localhost" ], "related.ip": [ - "10.31.237.225", - "10.6.38.163" + "10.6.38.163", + "10.31.237.225" ], "related.user": [ "olup" @@ -3670,8 +3670,8 @@ "oloreseo5039.test" ], "related.ip": [ - "10.28.105.124", - "10.218.0.197" + "10.218.0.197", + "10.28.105.124" ], "related.user": [ "ntNe" @@ -3732,8 +3732,8 @@ "minim459.mail.local" ], "related.ip": [ - "10.17.87.79", - "10.123.199.198" + "10.123.199.198", + "10.17.87.79" ], "related.user": [ "ratvolu" @@ -3794,8 +3794,8 @@ "eratv211.api.host" ], "related.ip": [ - "10.115.68.40", - "10.38.86.177" + "10.38.86.177", + "10.115.68.40" ], "related.user": [ "mpo" @@ -3918,8 +3918,8 @@ "iatqu6203.mail.corp" ], "related.ip": [ - "10.77.77.208", - "10.37.128.49" + "10.37.128.49", + "10.77.77.208" ], "related.user": [ "moles" @@ -4042,8 +4042,8 @@ "mag1506.internal.domain" ], "related.ip": [ - "10.182.152.242", - "10.131.126.109" + "10.131.126.109", + "10.182.152.242" ], "related.user": [ "dolor" @@ -4104,8 +4104,8 @@ "fugits1163.host" ], "related.ip": [ - "10.181.247.224", - "10.77.229.168" + "10.77.229.168", + "10.181.247.224" ], "related.user": [ "adol" @@ -4412,8 +4412,8 @@ "emullamc5418.mail.test" ], "related.ip": [ - "10.82.133.66", - "10.45.54.107" + "10.45.54.107", + "10.82.133.66" ], "related.user": [ "olorem" @@ -4722,8 +4722,8 @@ "dolor7082.internal.localhost" ], "related.ip": [ - "10.219.1.151", - "10.250.81.189" + "10.250.81.189", + "10.219.1.151" ], "related.user": [ "ori" @@ -4783,8 +4783,8 @@ "totam6886.api.localhost" ], "related.ip": [ - "10.54.23.133", - "10.76.125.70" + "10.76.125.70", + "10.54.23.133" ], "related.user": [ "oloreeu" @@ -4968,8 +4968,8 @@ "edq5397.www.test" ], "related.ip": [ - "10.73.28.165", - "10.221.206.74" + "10.221.206.74", + "10.73.28.165" ], "related.user": [ "quas" @@ -5030,8 +5030,8 @@ "udan6536.www5.test" ], "related.ip": [ - "10.85.104.146", - "10.14.204.36" + "10.14.204.36", + "10.85.104.146" ], "related.user": [ "emp" @@ -5278,8 +5278,8 @@ "enbyCi3813.api.domain" ], "related.ip": [ - "10.164.120.197", - "10.164.207.42" + "10.164.207.42", + "10.164.120.197" ], "related.user": [ "pta" @@ -5896,8 +5896,8 @@ "olores7881.local" ], "related.ip": [ - "10.87.144.208", - "10.143.53.214" + "10.143.53.214", + "10.87.144.208" ], "related.user": [ "psumq" @@ -6081,8 +6081,8 @@ "upt6017.api.localdomain" ], "related.ip": [ - "10.100.154.220", - "10.120.148.241" + "10.120.148.241", + "10.100.154.220" ], "related.user": [ "rsitam" diff --git a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json index 920acad2a309..fa5418e95068 100644 --- a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json @@ -2684,8 +2684,8 @@ "mveni5084.internal.local" ], "related.ip": [ - "10.144.111.42", - "10.62.61.1" + "10.62.61.1", + "10.144.111.42" ], "rsa.email.email_dst": "com", "rsa.email.email_src": "lam", @@ -3131,8 +3131,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "taevitae6868.www.corp", - "modi6930.internal.test" + "modi6930.internal.test", + "taevitae6868.www.corp" ], "related.ip": [ "10.60.164.100", @@ -3235,8 +3235,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "uradip7802.mail.example", - "tetura7106.www5.corp" + "tetura7106.www5.corp", + "uradip7802.mail.example" ], "related.ip": [ "10.44.35.57", @@ -3845,8 +3845,8 @@ "ncu3839.www.localhost" ], "related.ip": [ - "10.251.183.113", - "10.201.105.58" + "10.201.105.58", + "10.251.183.113" ], "rsa.email.email_dst": "ionemu", "rsa.email.email_src": "ent", diff --git a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml index 0391eea6ae97..b423b2203e4c 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json index dfec5c0d23a0..8b4c2711d482 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json @@ -28,8 +28,8 @@ "modtempo" ], "related.ip": [ - "10.189.58.145", "10.44.173.44", + "10.189.58.145", "10.20.234.169" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -99,8 +99,8 @@ "aer445.host" ], "related.ip": [ - "10.62.4.246", - "10.171.204.166" + "10.171.204.166", + "10.62.4.246" ], "related.user": [ "oluptas" @@ -196,8 +196,8 @@ "ccaecat" ], "related.ip": [ - "10.94.103.117", "10.15.159.80", + "10.94.103.117", "10.200.188.142" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -330,8 +330,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.5670", "related.hosts": [ - "ntutl", "roinBCSe", + "ntutl", "olo7148.mail.home" ], "related.ip": [ @@ -346,8 +346,8 @@ "rsa.investigations.event_vcat": "aveniam", "rsa.misc.OS": "oll", "rsa.misc.action": [ - "allow", - "ali" + "ali", + "allow" ], "rsa.misc.category": "emeumfug", "rsa.misc.client": "caecatc", @@ -431,8 +431,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.152", "related.hosts": [ - "onse", "orain", + "onse", "agna7678.internal.host" ], "related.ip": [ @@ -548,8 +548,8 @@ "rsa.investigations.event_vcat": "quae", "rsa.misc.OS": "qui", "rsa.misc.action": [ - "iadese", - "accept" + "accept", + "iadese" ], "rsa.misc.category": "aturve", "rsa.misc.client": "utei", @@ -634,8 +634,8 @@ "labore" ], "related.ip": [ - "10.176.216.90", "10.186.85.3", + "10.176.216.90", "10.114.16.155" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -705,8 +705,8 @@ "eturadi6608.mail.host" ], "related.ip": [ - "10.23.62.94", - "10.61.163.4" + "10.61.163.4", + "10.23.62.94" ], "related.user": [ "non" @@ -817,8 +817,8 @@ "rsa.investigations.event_vcat": "tDuisaut", "rsa.misc.OS": "Nequepor", "rsa.misc.action": [ - "deny", - "sno" + "sno", + "deny" ], "rsa.misc.category": "idolo", "rsa.misc.client": "volu", @@ -903,9 +903,9 @@ "diconseq" ], "related.ip": [ - "10.238.164.74", + "10.58.214.16", "10.106.162.153", - "10.58.214.16" + "10.238.164.74" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -970,9 +970,9 @@ "tenimad" ], "related.ip": [ - "10.110.31.190", + "10.225.141.20", "10.217.150.196", - "10.225.141.20" + "10.110.31.190" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1052,8 +1052,8 @@ "rsa.investigations.event_vcat": "xer", "rsa.misc.OS": "fugi", "rsa.misc.action": [ - "umdolo", - "deny" + "deny", + "umdolo" ], "rsa.misc.category": "conseq", "rsa.misc.client": "cusant", @@ -1138,9 +1138,9 @@ "onsecte" ], "related.ip": [ - "10.30.47.165", + "10.5.235.217", "10.25.212.118", - "10.5.235.217" + "10.30.47.165" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1306,9 +1306,9 @@ "xea" ], "related.ip": [ - "10.233.120.207", + "10.51.213.42", "10.98.194.212", - "10.51.213.42" + "10.233.120.207" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1374,8 +1374,8 @@ ], "related.ip": [ "10.241.132.176", - "10.245.187.229", - "10.67.132.242" + "10.67.132.242", + "10.245.187.229" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1646,8 +1646,8 @@ "eturad6143.www.home" ], "related.ip": [ - "10.95.117.134", - "10.128.46.70" + "10.128.46.70", + "10.95.117.134" ], "related.user": [ "enim" @@ -1657,8 +1657,8 @@ "rsa.investigations.event_vcat": "boNem", "rsa.misc.OS": "ntium", "rsa.misc.action": [ - "acommodi", - "block" + "block", + "acommodi" ], "rsa.misc.category": "inrepreh", "rsa.misc.client": "moles", @@ -1843,13 +1843,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.1710", "related.hosts": [ - "edquia", "Nemo", + "edquia", "henderi724.www5.home" ], "related.ip": [ - "10.3.23.172", - "10.243.226.122" + "10.243.226.122", + "10.3.23.172" ], "related.user": [ "olorem" @@ -1859,8 +1859,8 @@ "rsa.investigations.event_vcat": "ess", "rsa.misc.OS": "equatDu", "rsa.misc.action": [ - "cancel", - "emullamc" + "emullamc", + "cancel" ], "rsa.misc.category": "niamquis", "rsa.misc.client": "tutlabo", @@ -1944,8 +1944,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.5380", "related.hosts": [ - "uei", "onse", + "uei", "reseosqu1629.mail.lan" ], "related.ip": [ @@ -2047,8 +2047,8 @@ "oluptat" ], "related.ip": [ - "10.168.20.20", "10.117.63.181", + "10.168.20.20", "10.247.53.179" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -2119,8 +2119,8 @@ "tasnul4179.internal.host" ], "related.ip": [ - "10.141.156.217", - "10.53.168.187" + "10.53.168.187", + "10.141.156.217" ], "related.user": [ "amqu" @@ -2221,8 +2221,8 @@ "bore5546.www.local" ], "related.ip": [ - "10.189.82.19", - "10.44.198.184" + "10.44.198.184", + "10.189.82.19" ], "related.user": [ "iconsequ" @@ -2232,8 +2232,8 @@ "rsa.investigations.event_vcat": "eturadip", "rsa.misc.OS": "turadip", "rsa.misc.action": [ - "accept", - "odoc" + "odoc", + "accept" ], "rsa.misc.category": "volup", "rsa.misc.client": "tur", @@ -2333,8 +2333,8 @@ "rsa.investigations.event_vcat": "eturadip", "rsa.misc.OS": "onsecte", "rsa.misc.action": [ - "amni", - "cancel" + "cancel", + "amni" ], "rsa.misc.category": "umdolore", "rsa.misc.client": "modoc", @@ -2418,13 +2418,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.7278", "related.hosts": [ - "ectetura", "liquaUte", + "ectetura", "uido2046.mail.lan" ], "related.ip": [ - "10.130.240.11", - "10.70.7.23" + "10.70.7.23", + "10.130.240.11" ], "related.user": [ "eavolup" @@ -2520,9 +2520,9 @@ "uio" ], "related.ip": [ - "10.111.182.212", "10.37.161.101", - "10.17.209.252" + "10.17.209.252", + "10.111.182.212" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2587,8 +2587,8 @@ "itautfu" ], "related.ip": [ - "10.170.196.181", "10.158.175.98", + "10.170.196.181", "10.153.166.133" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -2658,8 +2658,8 @@ "con6049.internal.lan" ], "related.ip": [ - "10.63.171.91", - "10.48.25.200" + "10.48.25.200", + "10.63.171.91" ], "related.user": [ "usanti" @@ -2669,8 +2669,8 @@ "rsa.investigations.event_vcat": "enimad", "rsa.misc.OS": "olor", "rsa.misc.action": [ - "nse", - "accept" + "accept", + "nse" ], "rsa.misc.category": "conseq", "rsa.misc.client": "mmo", @@ -2755,9 +2755,9 @@ "iam" ], "related.ip": [ + "10.38.168.190", "10.174.17.46", - "10.77.105.81", - "10.38.168.190" + "10.77.105.81" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2889,9 +2889,9 @@ "eturadip" ], "related.ip": [ - "10.66.90.225", "10.214.156.161", - "10.145.194.12" + "10.145.194.12", + "10.66.90.225" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2956,8 +2956,8 @@ "iutal" ], "related.ip": [ - "10.156.208.5", "10.6.242.108", + "10.156.208.5", "10.163.36.101" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -3022,8 +3022,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4713", "related.hosts": [ - "epteurs", "data", + "epteurs", "remeum2641.www5.corp" ], "related.ip": [ @@ -3225,8 +3225,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4442", "related.hosts": [ - "fugi", "uae", + "fugi", "mea6298.api.example" ], "related.ip": [ @@ -3428,9 +3428,9 @@ "lors" ], "related.ip": [ + "10.205.83.138", "10.99.55.115", - "10.98.52.184", - "10.205.83.138" + "10.98.52.184" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3496,8 +3496,8 @@ ], "related.ip": [ "10.90.189.248", - "10.197.128.162", - "10.228.11.50" + "10.228.11.50", + "10.197.128.162" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3624,8 +3624,8 @@ "rsa.investigations.event_vcat": "uiratio", "rsa.misc.OS": "xce", "rsa.misc.action": [ - "cancel", - "cons" + "cons", + "cancel" ], "rsa.misc.category": "ciun", "rsa.misc.client": "amquisn", @@ -3714,8 +3714,8 @@ "tatiset4191.localdomain" ], "related.ip": [ - "10.26.58.20", - "10.185.37.176" + "10.185.37.176", + "10.26.58.20" ], "related.user": [ "eumiure" @@ -3811,9 +3811,9 @@ "teni" ], "related.ip": [ + "10.14.145.107", "10.250.231.196", - "10.200.12.126", - "10.14.145.107" + "10.200.12.126" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3878,9 +3878,9 @@ "atuse" ], "related.ip": [ + "10.225.34.176", "10.103.36.192", - "10.21.203.112", - "10.225.34.176" + "10.21.203.112" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4215,8 +4215,8 @@ ], "related.ip": [ "10.22.149.132", - "10.251.183.113", - "10.217.145.137" + "10.217.145.137", + "10.251.183.113" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4281,9 +4281,9 @@ "nisi" ], "related.ip": [ - "10.51.60.203", "10.183.16.252", - "10.203.66.175" + "10.203.66.175", + "10.51.60.203" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4363,8 +4363,8 @@ "rsa.investigations.event_vcat": "nisiuta", "rsa.misc.OS": "olu", "rsa.misc.action": [ - "accept", - "tquov" + "tquov", + "accept" ], "rsa.misc.category": "quatD", "rsa.misc.client": "acomm", @@ -4453,8 +4453,8 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.242.178.15", - "10.217.111.77" + "10.217.111.77", + "10.242.178.15" ], "related.user": [ "nimadmin" @@ -4464,8 +4464,8 @@ "rsa.investigations.event_vcat": "psaqu", "rsa.misc.OS": "nevolu", "rsa.misc.action": [ - "datatno", - "allow" + "allow", + "datatno" ], "rsa.misc.category": "ionu", "rsa.misc.client": "ugiatn", @@ -4650,8 +4650,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.1789", "related.hosts": [ - "dol", "ono", + "dol", "ntex5135.corp" ], "related.ip": [ @@ -4753,9 +4753,9 @@ "nonnumq" ], "related.ip": [ - "10.107.168.208", "10.34.41.75", - "10.249.16.201" + "10.249.16.201", + "10.107.168.208" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4824,8 +4824,8 @@ "tat1845.internal.invalid" ], "related.ip": [ - "10.109.106.194", - "10.96.168.24" + "10.96.168.24", + "10.109.106.194" ], "related.user": [ "ommodoc" @@ -4936,8 +4936,8 @@ "rsa.investigations.event_vcat": "eiusm", "rsa.misc.OS": "emag", "rsa.misc.action": [ - "deny", - "velillu" + "velillu", + "deny" ], "rsa.misc.category": "litseddo", "rsa.misc.client": "aturE", @@ -5022,9 +5022,9 @@ "metco" ], "related.ip": [ - "10.140.137.17", + "10.62.241.218", "10.103.169.94", - "10.62.241.218" + "10.140.137.17" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5160,8 +5160,8 @@ "spici5547.internal.test" ], "related.ip": [ - "10.112.242.68", - "10.216.49.112" + "10.216.49.112", + "10.112.242.68" ], "related.user": [ "urmag" @@ -5272,8 +5272,8 @@ "rsa.investigations.event_vcat": "uatDuisa", "rsa.misc.OS": "citation", "rsa.misc.action": [ - "accept", - "utlabore" + "utlabore", + "accept" ], "rsa.misc.category": "reeu", "rsa.misc.client": "ntut", @@ -5358,9 +5358,9 @@ "cons" ], "related.ip": [ + "10.157.22.21", "10.228.61.5", - "10.246.41.77", - "10.157.22.21" + "10.246.41.77" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5425,8 +5425,8 @@ "llumdo" ], "related.ip": [ - "10.188.131.18", "10.239.231.168", + "10.188.131.18", "10.242.119.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5745,8 +5745,8 @@ "mdolo7008.api.corp" ], "related.ip": [ - "10.162.128.87", - "10.78.75.82" + "10.78.75.82", + "10.162.128.87" ], "related.user": [ "Sedutp" @@ -5842,9 +5842,9 @@ "reseosqu" ], "related.ip": [ - "10.137.36.151", "10.51.106.43", - "10.75.198.93" + "10.75.198.93", + "10.137.36.151" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5910,9 +5910,8 @@ ], "related.ip": [ "10.7.230.206", - "10.154.151.111", - "10.7.230.206", - "10.249.93.150" + "10.249.93.150", + "10.154.151.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6077,8 +6076,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.6603", "related.hosts": [ - "eac", "ssuscipi", + "eac", "dquiac6194.api.lan" ], "related.ip": [ @@ -6093,8 +6092,8 @@ "rsa.investigations.event_vcat": "luptatev", "rsa.misc.OS": "emipsu", "rsa.misc.action": [ - "ido", - "accept" + "accept", + "ido" ], "rsa.misc.category": "litse", "rsa.misc.client": "evita", @@ -6279,8 +6278,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2691", "related.hosts": [ - "ratio", "orroqu", + "ratio", "dicta7226.mail.example" ], "related.ip": [ @@ -6295,8 +6294,8 @@ "rsa.investigations.event_vcat": "cupidata", "rsa.misc.OS": "ficiade", "rsa.misc.action": [ - "lorem", - "accept" + "accept", + "lorem" ], "rsa.misc.category": "iac", "rsa.misc.client": "tlabo", @@ -6381,8 +6380,8 @@ "eleumiu" ], "related.ip": [ - "10.236.211.111", "10.221.100.157", + "10.236.211.111", "10.120.212.78" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -6463,8 +6462,8 @@ "rsa.investigations.event_vcat": "lauda", "rsa.misc.OS": "enatuser", "rsa.misc.action": [ - "rios", - "accept" + "accept", + "rios" ], "rsa.misc.category": "aUte", "rsa.misc.client": "iusm", @@ -6617,8 +6616,8 @@ ], "related.ip": [ "10.3.85.176", - "10.29.141.252", - "10.212.56.26" + "10.212.56.26", + "10.29.141.252" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6687,8 +6686,8 @@ "emveleu4029.api.local" ], "related.ip": [ - "10.126.11.186", - "10.236.175.163" + "10.236.175.163", + "10.126.11.186" ], "related.user": [ "udantiu" @@ -6784,9 +6783,9 @@ "mes" ], "related.ip": [ - "10.171.60.173", + "10.83.98.220", "10.11.150.136", - "10.83.98.220" + "10.171.60.173" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6851,8 +6850,8 @@ "datatno" ], "related.ip": [ - "10.74.88.209", "10.238.49.73", + "10.74.88.209", "10.92.3.166" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -6918,8 +6917,8 @@ "ptate" ], "related.ip": [ - "10.84.200.121", "10.119.248.36", + "10.84.200.121", "10.187.107.47" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -6985,8 +6984,8 @@ "tasu" ], "related.ip": [ - "10.30.239.222", "10.167.128.229", + "10.30.239.222", "10.135.213.17" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7056,8 +7055,8 @@ "rspic5637.api.local" ], "related.ip": [ - "10.115.166.48", - "10.169.133.219" + "10.169.133.219", + "10.115.166.48" ], "related.user": [ "emq" @@ -7067,8 +7066,8 @@ "rsa.investigations.event_vcat": "iumdol", "rsa.misc.OS": "min", "rsa.misc.action": [ - "eleumiur", - "block" + "block", + "eleumiur" ], "rsa.misc.category": "ero", "rsa.misc.client": "gia", @@ -7157,8 +7156,8 @@ "rror3870.www5.local" ], "related.ip": [ - "10.226.39.82", - "10.146.255.40" + "10.146.255.40", + "10.226.39.82" ], "related.user": [ "caecatcu" @@ -7168,8 +7167,8 @@ "rsa.investigations.event_vcat": "ntexp", "rsa.misc.OS": "aconseq", "rsa.misc.action": [ - "block", - "oluptate" + "oluptate", + "block" ], "rsa.misc.category": "edqu", "rsa.misc.client": "ites", @@ -7255,8 +7254,8 @@ ], "related.ip": [ "10.35.240.70", - "10.53.82.96", - "10.224.212.88" + "10.224.212.88", + "10.53.82.96" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7322,8 +7321,8 @@ ], "related.ip": [ "10.186.253.240", - "10.66.149.234", - "10.233.128.7" + "10.233.128.7", + "10.66.149.234" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7388,9 +7387,9 @@ "irat" ], "related.ip": [ + "10.173.140.201", "10.46.11.114", - "10.227.133.134", - "10.173.140.201" + "10.227.133.134" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7455,9 +7454,9 @@ "emp" ], "related.ip": [ - "10.205.18.11", + "10.170.236.123", "10.69.130.207", - "10.170.236.123" + "10.205.18.11" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7521,8 +7520,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2682", "related.hosts": [ - "ine", "rehend", + "ine", "velill3821.mail.invalid" ], "related.ip": [ @@ -7623,9 +7622,9 @@ "ineavol" ], "related.ip": [ + "10.204.98.238", "10.9.41.221", - "10.81.58.91", - "10.204.98.238" + "10.81.58.91" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7804,9 +7803,9 @@ "aturvel" ], "related.ip": [ - "10.199.201.26", "10.143.65.84", - "10.207.207.106" + "10.207.207.106", + "10.199.201.26" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7872,8 +7871,8 @@ ], "related.ip": [ "10.204.27.48", - "10.163.236.253", - "10.41.61.88" + "10.41.61.88", + "10.163.236.253" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7938,9 +7937,9 @@ "dents" ], "related.ip": [ + "10.185.44.26", "10.246.81.164", - "10.53.110.111", - "10.185.44.26" + "10.53.110.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -8106,13 +8105,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.2314", "related.hosts": [ - "umtotam", "stenat", + "umtotam", "unt2122.internal.local" ], "related.ip": [ - "10.38.18.72", - "10.202.250.141" + "10.202.250.141", + "10.38.18.72" ], "related.user": [ "maperia" @@ -8207,8 +8206,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4674", "related.hosts": [ - "oremeu", "ita", + "oremeu", "luptat2613.internal.localhost" ], "related.ip": [ @@ -8223,8 +8222,8 @@ "rsa.investigations.event_vcat": "tfug", "rsa.misc.OS": "imipsam", "rsa.misc.action": [ - "utodi", - "block" + "block", + "utodi" ], "rsa.misc.category": "cid", "rsa.misc.client": "mquaerat", @@ -8308,13 +8307,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.1386", "related.hosts": [ - "amquisn", "cab", + "amquisn", "neavo4796.internal.domain" ], "related.ip": [ - "10.35.10.19", - "10.188.124.185" + "10.188.124.185", + "10.35.10.19" ], "related.user": [ "dolo" @@ -8324,8 +8323,8 @@ "rsa.investigations.event_vcat": "olupt", "rsa.misc.OS": "rumw", "rsa.misc.action": [ - "tali", - "block" + "block", + "tali" ], "rsa.misc.category": "itsedq", "rsa.misc.client": "esciu", diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index 621c1255bd8d..e37b497a0ef4 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json index ad6ac605be41..17a2a41cc6df 100644 --- a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json +++ b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json @@ -28,8 +28,8 @@ ], "related.user": [ "aqui", - "magn", - "tatno" + "tatno", + "magn" ], "rsa.counters.dclass_c1": 5910, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -117,8 +117,8 @@ ], "related.user": [ "qua", - "uradi", - "temUten" + "temUten", + "uradi" ], "rsa.counters.dclass_c1": 3626, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -238,17 +238,17 @@ "10.6.137.200" ], "related.user": [ + "occae", "oluptas", - "intoc", - "occae" + "intoc" ], "rsa.counters.event_counter": 7243, "rsa.db.database": "tNequepo", "rsa.internal.event_desc": "snostrud", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "dol", - "cancel" + "cancel", + "dol" ], "rsa.misc.category": "nama", "rsa.misc.disposition": "quisnos", @@ -311,13 +311,13 @@ "eratv6205.internal.lan" ], "related.ip": [ - "10.36.194.106", - "10.179.124.125" + "10.179.124.125", + "10.36.194.106" ], "related.user": [ - "ncidid", "reme", - "acommod" + "acommod", + "ncidid" ], "rsa.counters.event_counter": 2462, "rsa.db.database": "uaUteni", @@ -390,9 +390,9 @@ "10.211.105.204" ], "related.user": [ - "labor", "orema", - "eveli" + "eveli", + "labor" ], "rsa.counters.dclass_c1": 6855, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -448,13 +448,13 @@ "pora6854.www5.home" ], "related.ip": [ - "10.112.250.193", - "10.214.191.180" + "10.214.191.180", + "10.112.250.193" ], "related.user": [ + "ide", "ipsumdol", - "Exc", - "ide" + "Exc" ], "rsa.counters.dclass_c1": 6852, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -509,13 +509,13 @@ "ptasn6599.www.localhost" ], "related.ip": [ - "10.192.34.76", - "10.251.20.13" + "10.251.20.13", + "10.192.34.76" ], "related.user": [ - "ovol", "tnonpro", - "iquipe" + "iquipe", + "ovol" ], "rsa.counters.dclass_c1": 3645, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -571,9 +571,9 @@ "10.74.105.218" ], "related.user": [ + "archite", "idunt", - "boree", - "archite" + "boree" ], "rsa.counters.dclass_c1": 248, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -629,8 +629,8 @@ "rinre2977.api.corp" ], "related.ip": [ - "10.230.173.4", - "10.168.159.13" + "10.168.159.13", + "10.230.173.4" ], "related.user": [ "atemq", @@ -833,13 +833,13 @@ "umdolor4389.api.home" ], "related.ip": [ - "10.52.125.9", - "10.204.128.215" + "10.204.128.215", + "10.52.125.9" ], "related.user": [ + "nci", "paquioff", - "rum", - "nci" + "rum" ], "rsa.counters.event_counter": 332, "rsa.db.database": "isau", @@ -907,8 +907,8 @@ "rationev6444.localhost" ], "related.ip": [ - "10.200.68.129", - "10.34.148.166" + "10.34.148.166", + "10.200.68.129" ], "related.user": [ "untutlab", @@ -969,8 +969,8 @@ "10.134.5.40" ], "related.user": [ - "siu", "conse", + "siu", "licabo" ], "rsa.counters.dclass_c1": 6356, @@ -1031,9 +1031,9 @@ "10.30.98.10" ], "related.user": [ - "olori", "velite", - "dipisci" + "dipisci", + "olori" ], "rsa.counters.dclass_c1": 7717, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1093,9 +1093,9 @@ "10.233.120.207" ], "related.user": [ + "item", "quamnih", - "accusant", - "item" + "accusant" ], "rsa.counters.dclass_c1": 3278, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1185,7 +1185,6 @@ "related.user": [ "proident", "boru", - "ritati", "proident" ], "rsa.counters.dclass_c1": 5923, @@ -1308,9 +1307,9 @@ "10.167.252.183" ], "related.user": [ - "asnu", + "ationul", "redol", - "ationul" + "asnu" ], "rsa.counters.dclass_c1": 6606, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1372,9 +1371,9 @@ "10.81.184.7" ], "related.user": [ + "iameaque", "undeomni", - "lmole", - "iameaque" + "lmole" ], "rsa.counters.event_counter": 6344, "rsa.db.database": "nderi", @@ -1449,8 +1448,8 @@ ], "related.user": [ "taliqui", - "scipitl", - "edolorin" + "edolorin", + "scipitl" ], "rsa.counters.dclass_c1": 5140, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1588,7 +1587,6 @@ "10.182.152.242" ], "related.user": [ - "doeiu", "mquisn", "doeiu", "litan" @@ -1653,9 +1651,9 @@ "10.123.166.197" ], "related.user": [ - "min", "emUte", - "liquam" + "liquam", + "min" ], "rsa.counters.event_counter": 7102, "rsa.db.database": "oluptat", @@ -1728,10 +1726,9 @@ "10.72.75.207" ], "related.user": [ + "eufug", "eFini", - "urau", - "eFini", - "eufug" + "urau" ], "rsa.counters.dclass_c1": 3348, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1791,9 +1788,9 @@ "10.58.133.175" ], "related.user": [ - "nde", + "mfu", "oco", - "mfu" + "nde" ], "rsa.counters.dclass_c1": 3795, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1911,12 +1908,12 @@ "lesti6939.api.local" ], "related.ip": [ - "10.137.85.123", - "10.165.182.111" + "10.165.182.111", + "10.137.85.123" ], "related.user": [ - "sis", "ames", + "sis", "Bonorum" ], "rsa.counters.dclass_c1": 6401, @@ -2007,9 +2004,9 @@ "10.64.184.196" ], "related.user": [ - "tam", + "uian", "nesci", - "uian" + "tam" ], "rsa.counters.event_counter": 4493, "rsa.db.database": "sin", @@ -2084,8 +2081,8 @@ ], "related.user": [ "olu", - "aUtenima", - "olupta" + "olupta", + "aUtenima" ], "rsa.counters.dclass_c1": 1127, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2141,13 +2138,13 @@ "con6049.internal.lan" ], "related.ip": [ - "10.18.150.82", - "10.59.182.36" + "10.59.182.36", + "10.18.150.82" ], "related.user": [ "luptat", - "mtota", - "qua" + "qua", + "mtota" ], "rsa.counters.dclass_c1": 6112, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2234,9 +2231,9 @@ "10.151.240.35" ], "related.user": [ - "ama", "ametcons", - "lam" + "lam", + "ama" ], "rsa.counters.dclass_c1": 4325, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2292,9 +2289,9 @@ "10.147.142.242" ], "related.user": [ - "ese", "quisn", - "quasi" + "quasi", + "ese" ], "rsa.counters.dclass_c1": 3970, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2352,8 +2349,8 @@ "radipis3991.mail.invalid" ], "related.ip": [ - "10.213.165.165", - "10.254.10.98" + "10.254.10.98", + "10.213.165.165" ], "related.user": [ "ttenb", @@ -2690,9 +2687,9 @@ "10.208.33.55" ], "related.user": [ - "ulapari", "mremaper", - "inimv" + "inimv", + "ulapari" ], "rsa.counters.dclass_c1": 6433, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2752,8 +2749,8 @@ "10.203.164.132" ], "related.user": [ - "ectobea", "mporin", + "ectobea", "ibus" ], "rsa.counters.dclass_c1": 547, @@ -2814,9 +2811,9 @@ "10.151.203.60" ], "related.user": [ + "exeac", "dol", - "iconsequ", - "exeac" + "iconsequ" ], "rsa.counters.dclass_c1": 484, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2872,13 +2869,13 @@ "tat50.mail.host" ], "related.ip": [ - "10.224.217.153", - "10.45.152.205" + "10.45.152.205", + "10.224.217.153" ], "related.user": [ - "eriti", + "imav", "utlabo", - "imav" + "eriti" ], "rsa.counters.dclass_c1": 922, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2940,8 +2937,8 @@ ], "related.user": [ "adipis", - "hite", - "ugi" + "ugi", + "hite" ], "rsa.counters.event_counter": 508, "rsa.db.database": "abo", @@ -3014,8 +3011,8 @@ "10.248.244.203" ], "related.user": [ - "mquamei", "sum", + "mquamei", "eiusm" ], "rsa.counters.dclass_c1": 3058, @@ -3068,13 +3065,13 @@ "fde7756.mail.corp" ], "related.ip": [ - "10.122.127.237", - "10.86.121.152" + "10.86.121.152", + "10.122.127.237" ], "related.user": [ + "consecte", "nimv", - "ine", - "consecte" + "ine" ], "rsa.counters.dclass_c1": 2771, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3196,8 +3193,8 @@ "10.223.56.33" ], "related.user": [ - "magnido", "Nequepo", + "magnido", "elitsedd" ], "rsa.counters.dclass_c1": 3243, @@ -3260,17 +3257,17 @@ "10.94.89.177" ], "related.user": [ - "tuserror", "emquel", - "citation" + "citation", + "tuserror" ], "rsa.counters.event_counter": 2513, "rsa.db.database": "rspiciat", "rsa.internal.event_desc": "atuse", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "eruntmol", - "cancel" + "cancel", + "eruntmol" ], "rsa.misc.category": "imad", "rsa.misc.disposition": "tura", @@ -3336,8 +3333,8 @@ ], "related.user": [ "iin", - "tione", - "uta" + "uta", + "tione" ], "rsa.counters.dclass_c1": 5836, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3395,9 +3392,9 @@ "10.224.148.48" ], "related.user": [ - "iosamn", + "niam", "equepor", - "niam" + "iosamn" ], "rsa.counters.event_counter": 7468, "rsa.db.database": "erspicia", @@ -3471,9 +3468,9 @@ "10.21.61.134" ], "related.user": [ - "ostr", "imidest", - "mipsa" + "mipsa", + "ostr" ], "rsa.counters.dclass_c1": 7766, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3533,9 +3530,9 @@ "10.23.6.216" ], "related.user": [ + "iarchit", "tevelite", - "iamquisn", - "iarchit" + "iamquisn" ], "rsa.counters.dclass_c1": 639, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3674,17 +3671,17 @@ "10.111.22.134" ], "related.user": [ - "tqui", "inibusBo", - "ccusan" + "ccusan", + "tqui" ], "rsa.counters.event_counter": 3538, "rsa.db.database": "sequun", "rsa.internal.event_desc": "adeseru", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "orisnis", - "deny" + "deny", + "orisnis" ], "rsa.misc.category": "sitas", "rsa.misc.disposition": "eni", @@ -3749,9 +3746,9 @@ "10.161.225.172" ], "related.user": [ + "xerc", "rcit", - "meaqu", - "xerc" + "meaqu" ], "rsa.counters.dclass_c1": 7286, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3806,13 +3803,13 @@ "enbyCic4659.www5.example" ], "related.ip": [ - "10.211.161.187", - "10.186.133.184" + "10.186.133.184", + "10.211.161.187" ], "related.user": [ - "acons", "sci", - "boriosa" + "boriosa", + "acons" ], "rsa.counters.dclass_c1": 1578, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3921,8 +3918,8 @@ "reseo2067.api.localdomain" ], "related.ip": [ - "10.40.24.93", - "10.182.197.243" + "10.182.197.243", + "10.40.24.93" ], "related.user": [ "orisnis", @@ -3983,13 +3980,13 @@ "itte6905.mail.invalid" ], "related.ip": [ - "10.249.13.159", - "10.108.130.106" + "10.108.130.106", + "10.249.13.159" ], "related.user": [ - "colab", "exeacomm", - "uisautei" + "uisautei", + "colab" ], "rsa.counters.dclass_c1": 1044, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4060,8 +4057,8 @@ "rsa.internal.event_desc": "enimips", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "cancel", - "gna" + "gna", + "cancel" ], "rsa.misc.category": "Nequepor", "rsa.misc.disposition": "nisiu", @@ -4182,9 +4179,9 @@ "10.115.203.143" ], "related.user": [ - "orpori", + "utoditau", "involu", - "utoditau" + "orpori" ], "rsa.counters.dclass_c1": 7868, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4245,10 +4242,8 @@ ], "related.user": [ "inculp", - "gnido", "uptat", - "gnido", - "inculp" + "gnido" ], "rsa.counters.dclass_c1": 6947, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4332,13 +4327,12 @@ "usB4127.localhost" ], "related.ip": [ - "10.20.231.188", - "10.88.189.164" + "10.88.189.164", + "10.20.231.188" ], "related.user": [ "mqu", "tesseq", - "mqu", "uatDuisa" ], "rsa.counters.dclass_c1": 1623, @@ -4487,9 +4481,9 @@ "10.106.166.105" ], "related.user": [ - "usa", + "avolup", "olupt", - "avolup" + "usa" ], "rsa.counters.dclass_c1": 2658, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4546,8 +4540,8 @@ ], "related.user": [ "ipsu", - "iuta", - "ctas" + "ctas", + "iuta" ], "rsa.counters.dclass_c1": 392, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4607,8 +4601,8 @@ "10.129.234.200" ], "related.user": [ - "tevelit", "tisundeo", + "tevelit", "dquia" ], "rsa.counters.dclass_c1": 6709, @@ -4732,8 +4726,8 @@ ], "related.user": [ "enimip", - "dolo", - "ota" + "ota", + "dolo" ], "rsa.counters.dclass_c1": 469, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4794,8 +4788,8 @@ ], "related.user": [ "apar", - "ptasn", - "isn" + "isn", + "ptasn" ], "rsa.counters.dclass_c1": 758, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4851,8 +4845,8 @@ "iatisund424.mail.localdomain" ], "related.ip": [ - "10.178.190.123", - "10.42.135.34" + "10.42.135.34", + "10.178.190.123" ], "related.user": [ "orsi", @@ -4945,9 +4939,9 @@ "10.207.198.239" ], "related.user": [ - "Loremips", "incididu", - "aUteni" + "aUteni", + "Loremips" ], "rsa.counters.dclass_c1": 3043, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5002,14 +4996,13 @@ "dmini3435.internal.domain" ], "related.ip": [ - "10.206.221.180", - "10.116.26.185" + "10.116.26.185", + "10.206.221.180" ], "related.user": [ - "nseq", + "oNe", "litesseq", - "nseq", - "oNe" + "nseq" ], "rsa.counters.dclass_c1": 3218, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5065,9 +5058,9 @@ "10.86.180.150" ], "related.user": [ - "mnisis", + "itasper", "etconsec", - "itasper" + "mnisis" ], "rsa.counters.dclass_c1": 4564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5129,8 +5122,8 @@ "10.220.175.201" ], "related.user": [ - "rrors", - "dolo" + "dolo", + "rrors" ], "rsa.counters.event_counter": 4098, "rsa.db.database": "tsed", @@ -5354,9 +5347,9 @@ "10.69.5.227" ], "related.user": [ - "ntocc", + "doloreme", "rumw", - "doloreme" + "ntocc" ], "rsa.counters.dclass_c1": 5201, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5408,12 +5401,12 @@ "isetqu2843.www.invalid" ], "related.ip": [ - "10.213.214.118", - "10.253.175.129" + "10.253.175.129", + "10.213.214.118" ], "related.user": [ - "ate", "epteurs", + "ate", "nrep" ], "rsa.counters.dclass_c1": 6260, @@ -5476,9 +5469,9 @@ "10.89.26.170" ], "related.user": [ - "atus", + "orumetMa", "aboris", - "orumetMa" + "atus" ], "rsa.counters.event_counter": 5863, "rsa.db.database": "inventor", @@ -5554,8 +5547,8 @@ ], "related.user": [ "uaturve", - "neavolup", - "aco" + "aco", + "neavolup" ], "rsa.counters.event_counter": 5098, "rsa.db.database": "lapa", @@ -5627,21 +5620,21 @@ "par3605.internal.localdomain" ], "related.ip": [ - "10.223.10.28", - "10.230.48.97" + "10.230.48.97", + "10.223.10.28" ], "related.user": [ + "erit", "usmodte", - "untex", - "erit" + "untex" ], "rsa.counters.event_counter": 4029, "rsa.db.database": "ommodi", "rsa.internal.event_desc": "itatiset", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "tconse", - "deny" + "deny", + "tconse" ], "rsa.misc.category": "uaerat", "rsa.misc.disposition": "met", @@ -5707,9 +5700,9 @@ "10.115.42.231" ], "related.user": [ + "tasnul", "res", - "sequamn", - "tasnul" + "sequamn" ], "rsa.counters.dclass_c1": 4846, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5771,9 +5764,9 @@ "10.226.75.20" ], "related.user": [ - "tema", "maccusan", - "fugia" + "fugia", + "tema" ], "rsa.counters.event_counter": 3711, "rsa.db.database": "psa", @@ -5847,8 +5840,8 @@ ], "related.user": [ "rExcep", - "illumd", - "nimides" + "nimides", + "illumd" ], "rsa.counters.dclass_c1": 4173, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5906,9 +5899,9 @@ "10.197.254.133" ], "related.user": [ - "idu", + "trudex", "ide", - "trudex" + "idu" ], "rsa.counters.event_counter": 2608, "rsa.db.database": "ncul", @@ -6043,8 +6036,8 @@ ], "related.user": [ "quaturve", - "quei", - "caecat" + "caecat", + "quei" ], "rsa.counters.dclass_c1": 983, "rsa.counters.dclass_c1_str": "Affected Rows", diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index cabc18d6a058..22f32138c275 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json index 2937ed39d2df..ddd233180e09 100644 --- a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json +++ b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json @@ -456,8 +456,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.38.77.13", - "10.179.26.34" + "10.179.26.34", + "10.38.77.13" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1183,8 +1183,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.83.23.104", - "10.163.161.165" + "10.163.161.165", + "10.83.23.104" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1928,8 +1928,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.98.209.10", - "10.31.177.226" + "10.31.177.226", + "10.98.209.10" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1966,8 +1966,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.179.210.218", - "10.44.47.27" + "10.44.47.27", + "10.179.210.218" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index c032cf0bf301..c2d1a74c0306 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -31,9 +31,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json index 40660e0d5188..41256959a5c5 100644 --- a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json @@ -64,7 +64,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -142,7 +146,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -549,7 +557,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json b/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json index 372b29d8c2ca..89a70bc165f5 100644 --- a/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json @@ -249,7 +249,10 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem4.sharepoint.com", "url.original": "https://testsiem4.sharepoint.com/sites/users/Shared Documents/General", + "url.path": "/sites/users/Shared Documents/General", + "url.scheme": "https", "user.domain": "testsiem4.onmicrosoft.com", "user.email": "root@testsiem4.onmicrosoft.com", "user.id": "root@testsiem4.onmicrosoft.com", @@ -1547,7 +1550,10 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem4.sharepoint.com", "url.original": "https://testsiem4.sharepoint.com/sites/users/Shared Documents/General", + "url.path": "/sites/users/Shared Documents/General", + "url.scheme": "https", "user.domain": "testsiem4.onmicrosoft.com", "user.email": "root@testsiem4.onmicrosoft.com", "user.id": "root@testsiem4.onmicrosoft.com", diff --git a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml index 7cb1b02a18ac..66d787b27ce3 100644 --- a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json index d02fc7982b35..b2abd9c2604b 100644 --- a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json @@ -59,9 +59,9 @@ "10.57.170.140" ], "related.user": [ + "dexeac", "icistatuscode=giatquov", - "sunt", - "dexeac" + "sunt" ], "rsa.db.index": "run", "rsa.identity.logon_type": "nofdeF", @@ -644,8 +644,8 @@ "10.54.169.175" ], "related.user": [ - "taspe", - "scipit" + "scipit", + "taspe" ], "rsa.identity.logon_type": "olores", "rsa.internal.event_desc": "secil", @@ -893,8 +893,8 @@ "elites4713.www.localhost" ], "related.ip": [ - "10.161.51.135", - "10.52.190.18" + "10.52.190.18", + "10.161.51.135" ], "rsa.internal.event_desc": "portscan", "rsa.internal.messageid": "ulogd", @@ -1039,8 +1039,8 @@ "nostrum6305.internal.localhost" ], "related.ip": [ - "10.89.41.97", - "10.17.51.153" + "10.17.51.153", + "10.89.41.97" ], "related.user": [ "tcustatuscode=eumiu", @@ -1628,8 +1628,8 @@ "10.244.96.61" ], "related.user": [ - "itsedqui", - "iumt" + "iumt", + "itsedqui" ], "rsa.identity.logon_type": "psamvolu", "rsa.internal.event_desc": "orroqui", @@ -2057,9 +2057,9 @@ "10.92.93.236" ], "related.user": [ + "ulpaq", "ntoccae", "Sedutper", - "ulpaq", "dolorsistatuscode=acc" ], "rsa.db.index": "snisiut", @@ -2152,13 +2152,13 @@ "mni4032.lan" ], "related.ip": [ - "10.202.65.2", - "10.180.169.49" + "10.180.169.49", + "10.202.65.2" ], "related.user": [ + "atatno", "iscivelistatuscode=urve", - "tasu", - "atatno" + "tasu" ], "rsa.db.index": "amrem", "rsa.identity.logon_type": "nulamcol", @@ -2167,8 +2167,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "mquisn", - "cancel" + "cancel", + "mquisn" ], "rsa.misc.comments": "architec", "rsa.misc.content_type": "atisetqu", @@ -2352,13 +2352,13 @@ "obea2960.mail.corp" ], "related.ip": [ - "10.45.12.53", - "10.33.138.154" + "10.33.138.154", + "10.45.12.53" ], "related.user": [ + "umqustatuscode=ntexpli", "eturadip", - "porincid", - "umqustatuscode=ntexpli" + "porincid" ], "rsa.db.index": "dolor", "rsa.identity.logon_type": "eturadi", @@ -2367,8 +2367,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "cancel", - "doc" + "doc", + "cancel" ], "rsa.misc.comments": "riosam", "rsa.misc.content_type": "iusmo", @@ -2525,8 +2525,8 @@ "10.32.85.21" ], "related.user": [ - "etconsec", - "antium" + "antium", + "etconsec" ], "rsa.identity.logon_type": "umiurere", "rsa.internal.event_desc": "serro", @@ -2677,11 +2677,9 @@ ], "related.user": [ "reetd", - "inimastatuscode=emipsum", + "rExce", "Loremi", - "inimastatuscode=emipsum", - "reetd", - "rExce" + "inimastatuscode=emipsum" ], "rsa.db.index": "apa", "rsa.identity.logon_type": "sedquia", @@ -2690,8 +2688,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "odte", - "cancel" + "cancel", + "odte" ], "rsa.misc.comments": "emquia", "rsa.misc.content_type": "sauteir", diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index bc79b7991361..f9f6ddaa0276 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true - remove: field: url.scheme ignore_missing: true diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index 420f48270f21..a0bf899ae2ca 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -443,8 +443,8 @@ "www.google-analytics.com" ], "related.ip": [ - "10.105.21.199", - "66.102.9.147" + "66.102.9.147", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -455,8 +455,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -520,8 +520,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -608,8 +608,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -672,8 +672,8 @@ "www.goonernews.com" ], "related.ip": [ - "10.105.21.199", - "207.58.145.61" + "207.58.145.61", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -891,8 +891,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_MISS" + "TCP_MISS", + "CONNECT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -1099,8 +1099,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -1175,8 +1175,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -1723,8 +1723,8 @@ "shttp.msg.yahoo.com" ], "related.ip": [ - "216.155.194.239", - "10.105.33.214" + "10.105.33.214", + "216.155.194.239" ], "related.user": [ "adeolaegbedokun" @@ -1734,8 +1734,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "POST" + "POST", + "TCP_MISS" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -1950,8 +1950,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -2210,8 +2210,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -2271,8 +2271,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -2409,8 +2409,8 @@ "insider.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.194.14" + "68.142.194.14", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2421,8 +2421,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -2565,8 +2565,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -2623,8 +2623,8 @@ "address.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "209.191.93.51" + "209.191.93.51", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2783,8 +2783,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -3032,8 +3032,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_DENIED" + "TCP_DENIED", + "POST" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3222,8 +3222,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_DENIED" + "TCP_DENIED", + "CONNECT" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3292,8 +3292,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3435,8 +3435,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3703,8 +3703,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3776,8 +3776,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3984,8 +3984,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -4108,8 +4108,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -4169,8 +4169,8 @@ "newsrss.bbc.co.uk" ], "related.ip": [ - "10.105.21.199", - "212.58.226.33" + "212.58.226.33", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -4242,8 +4242,8 @@ "insider.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.231.252" + "68.142.231.252", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4316,8 +4316,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "304", @@ -4514,8 +4514,8 @@ "us.news1.yimg.com" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4735,8 +4735,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4810,8 +4810,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "GET" + "GET", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -5011,8 +5011,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -5162,8 +5162,8 @@ "us.a2.yimg.com" ], "related.ip": [ - "10.105.33.214", - "213.160.98.152" + "213.160.98.152", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5248,8 +5248,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -5308,8 +5308,8 @@ "us.bc.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.213.132" + "68.142.213.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5596,8 +5596,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "304", @@ -5660,8 +5660,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5892,8 +5892,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5954,8 +5954,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_IMS_HIT" + "TCP_IMS_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -6018,8 +6018,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "213.160.98.167", - "10.105.33.214" + "10.105.33.214", + "213.160.98.167" ], "related.user": [ "adeolaegbedokun" @@ -6182,8 +6182,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -6365,8 +6365,8 @@ "launch.adserver.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.109.125.112" + "216.109.125.112", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -6510,8 +6510,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_DENIED" + "TCP_DENIED", + "CONNECT" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -6583,8 +6583,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_SWAPFAIL_MISS", - "GET" + "GET", + "TCP_SWAPFAIL_MISS" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "200", @@ -6907,8 +6907,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", diff --git a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json index 0e23e14e212d..77f61afdee9b 100644 --- a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json @@ -22,16 +22,16 @@ "example.net" ], "related.ip": [ - "10.234.224.44", - "10.251.224.219" + "10.251.224.219", + "10.234.224.44" ], "related.user": [ "tation" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "deny", - "PROPFIND" + "PROPFIND", + "deny" ], "rsa.misc.content_type": "ciade", "rsa.misc.result_code": "liqua", @@ -94,8 +94,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www.example.org" + "www.example.org", + "example.com" ], "related.ip": [ "10.102.123.34", @@ -249,12 +249,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.org" + "api.example.org", + "mail.example.com" ], "related.ip": [ - "10.44.134.153", - "10.81.122.126" + "10.81.122.126", + "10.44.134.153" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", @@ -327,8 +327,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "api.example.net" + "api.example.net", + "api.example.org" ], "related.ip": [ "10.160.95.56", @@ -405,8 +405,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "api.example.com" + "api.example.com", + "www.example.org" ], "related.ip": [ "10.175.107.139", @@ -498,8 +498,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "allow" + "allow", + "DELETE" ], "rsa.misc.content_type": "usmodte", "rsa.misc.result_code": "mUt", @@ -640,8 +640,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "example.org" + "example.org", + "mail.example.net" ], "related.ip": [ "10.90.131.186", @@ -802,8 +802,8 @@ "www.example.org" ], "related.ip": [ - "10.158.185.163", - "10.71.34.9" + "10.71.34.9", + "10.158.185.163" ], "related.user": [ "aliq" @@ -880,8 +880,8 @@ "internal.example.net" ], "related.ip": [ - "10.210.74.24", - "10.201.76.240" + "10.201.76.240", + "10.210.74.24" ], "related.user": [ "uines" @@ -954,8 +954,8 @@ "www.example.net" ], "related.ip": [ - "10.206.136.206", - "10.114.138.121" + "10.114.138.121", + "10.206.136.206" ], "related.user": [ "xeac" @@ -1040,8 +1040,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "COPY", - "block" + "block", + "COPY" ], "rsa.misc.content_type": "uptat", "rsa.misc.result_code": "maccusa", @@ -1118,8 +1118,8 @@ ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "NONE", - "allow" + "allow", + "NONE" ], "rsa.misc.content_type": "aaliquaU", "rsa.misc.result_code": "mpori", @@ -1188,8 +1188,8 @@ "api.example.com" ], "related.ip": [ - "10.249.213.83", - "10.164.250.63" + "10.164.250.63", + "10.249.213.83" ], "related.user": [ "itame" @@ -1266,8 +1266,8 @@ "api.example.net" ], "related.ip": [ - "10.236.248.65", - "10.61.242.75" + "10.61.242.75", + "10.236.248.65" ], "related.user": [ "iquidex" @@ -1421,8 +1421,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "api.example.org" + "api.example.org", + "internal.example.net" ], "related.ip": [ "10.89.201.140", @@ -1658,8 +1658,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "example.com" + "example.com", + "internal.example.net" ], "related.ip": [ "10.47.25.230", @@ -1816,8 +1816,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.net" + "api.example.net", + "mail.example.com" ], "related.ip": [ "10.93.220.10", @@ -1894,8 +1894,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.net", - "example.net" + "example.net", + "www.example.net" ], "related.ip": [ "10.213.144.249", @@ -2050,8 +2050,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "example.com" + "example.com", + "internal.example.org" ], "related.ip": [ "10.161.203.252", @@ -2142,8 +2142,8 @@ ], "rsa.internal.messageid": "LOCK", "rsa.misc.action": [ - "LOCK", - "block" + "block", + "LOCK" ], "rsa.misc.content_type": "volupta", "rsa.misc.result_code": "veli", @@ -2208,8 +2208,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.org", - "mail.example.org" + "mail.example.org", + "www5.example.org" ], "related.ip": [ "10.104.80.189", @@ -2360,8 +2360,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "mail.example.com" + "mail.example.com", + "api.example.com" ], "related.ip": [ "10.26.96.202", @@ -2519,8 +2519,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "www.example.org" + "www.example.org", + "internal.example.org" ], "related.ip": [ "10.224.11.165", @@ -2531,8 +2531,8 @@ ], "rsa.internal.messageid": "MOVE", "rsa.misc.action": [ - "MOVE", - "deny" + "deny", + "MOVE" ], "rsa.misc.content_type": "elites", "rsa.misc.result_code": "oremi", @@ -2764,8 +2764,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "block", - "UNLOCK" + "UNLOCK", + "block" ], "rsa.misc.content_type": "commodi", "rsa.misc.result_code": "ssecil", @@ -2841,8 +2841,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "GET" + "GET", + "allow" ], "rsa.misc.content_type": "numquam", "rsa.misc.result_code": "temUt", @@ -2905,20 +2905,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "www5.example.net" + "www5.example.net", + "api.example.org" ], "related.ip": [ - "10.173.222.131", - "10.211.234.224" + "10.211.234.224", + "10.173.222.131" ], "related.user": [ "Duisa" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "accept", - "TRACE" + "TRACE", + "accept" ], "rsa.misc.content_type": "boNem", "rsa.misc.result_code": "aUt", @@ -2983,8 +2983,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "mail.example.net" + "mail.example.net", + "www5.example.net" ], "related.ip": [ "10.0.157.225", @@ -3065,8 +3065,8 @@ "api.example.com" ], "related.ip": [ - "10.92.237.93", - "10.228.77.21" + "10.228.77.21", + "10.92.237.93" ], "related.user": [ "onse" @@ -3153,8 +3153,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "allow" + "allow", + "POST" ], "rsa.misc.content_type": "nderiti", "rsa.misc.result_code": "litsedq", @@ -3307,8 +3307,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "allow" + "allow", + "DELETE" ], "rsa.misc.content_type": "acom", "rsa.misc.result_code": "metco", @@ -3373,8 +3373,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www.example.org" + "www.example.org", + "example.com" ], "related.ip": [ "10.101.85.169", @@ -3388,8 +3388,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "accept" + "accept", + "GET" ], "rsa.misc.content_type": "radi", "rsa.misc.result_code": "Finibus", @@ -3547,8 +3547,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "CONNECT" + "CONNECT", + "allow" ], "rsa.misc.content_type": "incidid", "rsa.misc.result_code": "ugiatquo", @@ -3625,8 +3625,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "deny" + "deny", + "PROPFIND" ], "rsa.misc.content_type": "est", "rsa.misc.result_code": "agnaaliq", @@ -3773,16 +3773,16 @@ "api.example.org" ], "related.ip": [ - "10.150.198.112", - "10.4.69.152" + "10.4.69.152", + "10.150.198.112" ], "related.user": [ "mexer" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "block" + "block", + "PUT" ], "rsa.misc.content_type": "abor", "rsa.misc.result_code": "uipexe", @@ -3847,12 +3847,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.org", - "example.org" + "example.org", + "www5.example.org" ], "related.ip": [ - "10.45.54.107", - "10.45.114.111" + "10.45.114.111", + "10.45.54.107" ], "related.user": [ "nse" @@ -3929,16 +3929,16 @@ "mail.example.net" ], "related.ip": [ - "10.205.28.24", - "10.49.242.174" + "10.49.242.174", + "10.205.28.24" ], "related.user": [ "untutlab" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "TRACE", - "allow" + "allow", + "TRACE" ], "rsa.misc.content_type": "pteurs", "rsa.misc.result_code": "consecte", @@ -4092,8 +4092,8 @@ ], "rsa.internal.messageid": "MOVE", "rsa.misc.action": [ - "cancel", - "MOVE" + "MOVE", + "cancel" ], "rsa.misc.content_type": "tura", "rsa.misc.result_code": "nturmag", @@ -4166,8 +4166,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "accept", - "OPTIONS" + "OPTIONS", + "accept" ], "rsa.misc.content_type": "eacommo", "rsa.misc.result_code": "hend", @@ -4306,8 +4306,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.com", - "internal.example.net" + "internal.example.net", + "www5.example.com" ], "related.ip": [ "10.93.159.170", @@ -4469,8 +4469,8 @@ "internal.example.net" ], "related.ip": [ - "10.89.73.240", - "10.41.156.88" + "10.41.156.88", + "10.89.73.240" ], "related.user": [ "ntorever" @@ -4547,8 +4547,8 @@ "www.example.org" ], "related.ip": [ - "10.54.44.231", - "10.101.183.86" + "10.101.183.86", + "10.54.44.231" ], "related.user": [ "mcorpo" @@ -4701,8 +4701,8 @@ "api.example.org" ], "related.ip": [ - "10.83.130.95", - "10.76.220.3" + "10.76.220.3", + "10.83.130.95" ], "related.user": [ "userror" @@ -4712,8 +4712,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "GET" + "GET", + "deny" ], "rsa.misc.content_type": "liqua", "rsa.misc.result_code": "nderi", @@ -4854,8 +4854,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.org" + "www5.example.org", + "example.com" ], "related.ip": [ "10.121.121.153", @@ -4866,8 +4866,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "cancel", - "UNLOCK" + "UNLOCK", + "cancel" ], "rsa.misc.content_type": "magnam", "rsa.misc.result_code": "modoc", @@ -4932,8 +4932,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www.example.com" + "www.example.com", + "internal.example.net" ], "related.ip": [ "10.202.224.209", @@ -4944,8 +4944,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "cancel", - "OPTIONS" + "OPTIONS", + "cancel" ], "rsa.misc.content_type": "umf", "rsa.misc.result_code": "obeataev", @@ -5014,8 +5014,8 @@ "mail.example.com" ], "related.ip": [ - "10.72.99.69", - "10.170.234.233" + "10.170.234.233", + "10.72.99.69" ], "related.user": [ "uatu" @@ -5166,8 +5166,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "example.com" + "example.com", + "api.example.net" ], "related.ip": [ "10.62.188.193", @@ -5178,8 +5178,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "deny", - "DELETE" + "DELETE", + "deny" ], "rsa.misc.content_type": "tmo", "rsa.misc.result_code": "cin", @@ -5244,8 +5244,8 @@ "www.example.net" ], "related.ip": [ - "10.68.198.188", - "10.172.139.78" + "10.172.139.78", + "10.68.198.188" ], "related.user": [ "onsectet" @@ -5400,8 +5400,8 @@ "api.example.org" ], "related.ip": [ - "10.62.10.137", - "10.32.98.109" + "10.32.98.109", + "10.62.10.137" ], "related.user": [ "deomnisi" @@ -5703,8 +5703,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "www5.example.org" + "www5.example.org", + "internal.example.com" ], "related.ip": [ "10.11.73.145", @@ -5795,8 +5795,8 @@ ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "accept" + "accept", + "PURGE" ], "rsa.misc.content_type": "uames", "rsa.misc.result_code": "moenimip", @@ -5873,8 +5873,8 @@ ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "block", - "PURGE" + "PURGE", + "block" ], "rsa.misc.content_type": "uis", "rsa.misc.result_code": "maveniam", @@ -6095,8 +6095,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.org", - "mail.example.com" + "mail.example.com", + "mail.example.org" ], "related.ip": [ "10.175.138.42", @@ -6255,8 +6255,8 @@ "www5.example.org" ], "related.ip": [ - "10.1.220.47", - "10.73.80.251" + "10.73.80.251", + "10.1.220.47" ], "related.user": [ "ercitati" @@ -6407,8 +6407,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.net", - "mail.example.com" + "mail.example.com", + "www.example.net" ], "related.ip": [ "10.62.168.226", @@ -6421,8 +6421,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "allow" + "allow", + "CONNECT" ], "rsa.misc.content_type": "sequamn", "rsa.misc.result_code": "eirured", @@ -6563,8 +6563,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.com", - "example.com" + "example.com", + "www5.example.com" ], "related.ip": [ "10.49.169.175", @@ -6642,12 +6642,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "internal.example.com" + "internal.example.com", + "www.example.org" ], "related.ip": [ - "10.33.112.100", - "10.213.100.153" + "10.213.100.153", + "10.33.112.100" ], "related.user": [ "enimad" @@ -6724,8 +6724,8 @@ "example.net" ], "related.ip": [ - "10.216.143.226", - "10.25.53.93" + "10.25.53.93", + "10.216.143.226" ], "related.user": [ "oremeu" @@ -6959,8 +6959,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.com" + "api.example.com", + "www5.example.net" ], "related.ip": [ "10.245.251.98", @@ -7121,16 +7121,16 @@ "internal.example.com" ], "related.ip": [ - "10.187.86.64", - "10.6.88.105" + "10.6.88.105", + "10.187.86.64" ], "related.user": [ "rem" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "allow", - "TRACE" + "TRACE", + "allow" ], "rsa.misc.content_type": "rinci", "rsa.misc.result_code": "iscivel", @@ -7443,8 +7443,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "OPTIONS", - "deny" + "deny", + "OPTIONS" ], "rsa.misc.content_type": "siuta", "rsa.misc.result_code": "emveleum", @@ -7661,8 +7661,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "api.example.com" + "api.example.com", + "example.com" ], "related.ip": [ "10.221.86.133", @@ -7753,8 +7753,8 @@ ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "deny" + "deny", + "PUT" ], "rsa.misc.content_type": "exeaco", "rsa.misc.result_code": "rmagnido", diff --git a/x-pack/filebeat/module/suricata/eve/ingest/http.yml b/x-pack/filebeat/module/suricata/eve/ingest/http.yml index 7dcb7d4b4ef5..de8c1710ed75 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/http.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/http.yml @@ -65,9 +65,6 @@ processors: field: suricata.eve.http.http_refer target_field: http.request.referrer ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true - rename: field: suricata.eve.http.length target_field: http.response.body.bytes diff --git a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json index d9de4e04efd5..dbc2c9245df3 100644 --- a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json @@ -3,7 +3,7 @@ "@timestamp": "2019-11-06T09:03:00.989Z", "destination.address": "198.41.0.4", "destination.as.number": 20172, - "destination.as.organization.name": "VeriSign Global Registry Services", + "destination.as.organization.name": "VGRS-AC27", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.country_name": "United States", @@ -44,4 +44,4 @@ ], "zeek.session_id": "Ctefoj1tgOPt4D0EK2" } -] \ No newline at end of file +] diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index 7f869714ff45..abf179e94b35 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -34,9 +34,6 @@ processors: - remove: field: _temp_ ignore_missing: true - - urldecode: - field: http.request.referrer - ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json index 2629b307fc3c..cd89fe2d77a8 100644 --- a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json +++ b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json @@ -41,8 +41,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntium", "rsa.misc.action": [ - "Blocked", - "pisciv" + "pisciv", + "Blocked" ], "rsa.misc.category": "umq", "rsa.misc.filter": "oremi", @@ -270,8 +270,8 @@ "tempor4496.www.localdomain" ], "related.ip": [ - "10.103.246.190", - "10.252.125.53" + "10.252.125.53", + "10.103.246.190" ], "related.user": [ "equun" @@ -352,8 +352,8 @@ "ore2933.www.test" ], "related.ip": [ - "10.136.153.149", - "10.61.78.108" + "10.61.78.108", + "10.136.153.149" ], "related.user": [ "ercit" @@ -531,8 +531,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "lupt", "rsa.misc.action": [ - "Blocked", - "dun" + "dun", + "Blocked" ], "rsa.misc.category": "rsitamet", "rsa.misc.filter": "usmod", @@ -777,8 +777,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ihilm", "rsa.misc.action": [ - "Allowed", - "psaquae" + "psaquae", + "Allowed" ], "rsa.misc.category": "eFinib", "rsa.misc.filter": "inesci", @@ -859,8 +859,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "sci", "rsa.misc.action": [ - "Allowed", - "emseq" + "emseq", + "Allowed" ], "rsa.misc.category": "exercit", "rsa.misc.filter": "taevit", @@ -941,8 +941,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "dolorem", "rsa.misc.action": [ - "lorsitam", - "Allowed" + "Allowed", + "lorsitam" ], "rsa.misc.category": "proide", "rsa.misc.filter": "pariatu", @@ -1105,8 +1105,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iurer", "rsa.misc.action": [ - "Allowed", - "ionevo" + "ionevo", + "Allowed" ], "rsa.misc.category": "tinvolu", "rsa.misc.filter": "idex", @@ -1582,8 +1582,8 @@ "eacommod1930.internal.lan" ], "related.ip": [ - "10.229.83.165", - "10.29.155.171" + "10.29.155.171", + "10.229.83.165" ], "related.user": [ "ulapar" @@ -1679,8 +1679,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uaUten", "rsa.misc.action": [ - "amcorp", - "Blocked" + "Blocked", + "amcorp" ], "rsa.misc.category": "umdolor", "rsa.misc.filter": "velillu", @@ -1992,8 +1992,8 @@ "sitam5077.internal.host" ], "related.ip": [ - "10.179.210.218", - "10.32.39.220" + "10.32.39.220", + "10.179.210.218" ], "related.user": [ "boreetdo" @@ -2007,8 +2007,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "riss", "rsa.misc.action": [ - "risnis", - "Blocked" + "Blocked", + "risnis" ], "rsa.misc.category": "emqu", "rsa.misc.filter": "oluptas", @@ -2074,8 +2074,8 @@ "dquia107.www.test" ], "related.ip": [ - "10.88.172.34", - "10.128.173.19" + "10.128.173.19", + "10.88.172.34" ], "related.user": [ "agnaaliq" @@ -2089,8 +2089,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntNeq", "rsa.misc.action": [ - "dtempo", - "Blocked" + "Blocked", + "dtempo" ], "rsa.misc.category": "ipsu", "rsa.misc.filter": "iqu", @@ -2171,8 +2171,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mnisiut", "rsa.misc.action": [ - "mod", - "Allowed" + "Allowed", + "mod" ], "rsa.misc.category": "uiinea", "rsa.misc.filter": "aturQu", @@ -2238,8 +2238,8 @@ "tamet6317.www.host" ], "related.ip": [ - "10.2.67.127", - "10.115.53.31" + "10.115.53.31", + "10.2.67.127" ], "related.user": [ "Cic" @@ -2484,8 +2484,8 @@ "mdolore473.internal.test" ], "related.ip": [ - "10.87.100.240", - "10.242.182.193" + "10.242.182.193", + "10.87.100.240" ], "related.user": [ "stenatus" @@ -2663,8 +2663,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uteir", "rsa.misc.action": [ - "Section", - "Allowed" + "Allowed", + "Section" ], "rsa.misc.category": "cididu", "rsa.misc.filter": "Utenima", @@ -2909,8 +2909,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ende", "rsa.misc.action": [ - "Blocked", - "doconse" + "doconse", + "Blocked" ], "rsa.misc.category": "uovolupt", "rsa.misc.filter": "litesse", @@ -2991,8 +2991,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "issu", "rsa.misc.action": [ - "sed", - "Allowed" + "Allowed", + "sed" ], "rsa.misc.category": "atur", "rsa.misc.filter": "iciadese", @@ -3058,8 +3058,8 @@ "eufug1756.mail.corp" ], "related.ip": [ - "10.53.101.131", - "10.213.57.165" + "10.213.57.165", + "10.53.101.131" ], "related.user": [ "isau" @@ -3319,8 +3319,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Loremip", "rsa.misc.action": [ - "Allowed", - "quid" + "quid", + "Allowed" ], "rsa.misc.category": "mini", "rsa.misc.filter": "uisnos", @@ -3550,8 +3550,8 @@ "remagnam796.mail.corp" ], "related.ip": [ - "10.143.0.78", - "10.137.164.122" + "10.137.164.122", + "10.143.0.78" ], "related.user": [ "orissus" @@ -3565,8 +3565,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "etdol", "rsa.misc.action": [ - "Blocked", - "mwrit" + "mwrit", + "Blocked" ], "rsa.misc.category": "inim", "rsa.misc.filter": "aturQu", @@ -3647,8 +3647,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tatno", "rsa.misc.action": [ - "ptatev", - "Blocked" + "Blocked", + "ptatev" ], "rsa.misc.category": "udexerc", "rsa.misc.filter": "ptatemse", @@ -3878,8 +3878,8 @@ "laboree3880.api.invalid" ], "related.ip": [ - "10.166.195.20", - "10.255.40.12" + "10.255.40.12", + "10.166.195.20" ], "related.user": [ "lamcolab" @@ -3893,8 +3893,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mipsumq", "rsa.misc.action": [ - "Allowed", - "citation" + "citation", + "Allowed" ], "rsa.misc.category": "usant", "rsa.misc.filter": "Nem", @@ -4055,8 +4055,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "dexea", "rsa.misc.action": [ - "tinvolup", - "Blocked" + "Blocked", + "tinvolup" ], "rsa.misc.category": "ende", "rsa.misc.filter": "onse", @@ -4122,8 +4122,8 @@ "tsunt3403.www5.test" ], "related.ip": [ - "10.31.153.177", - "10.237.0.173" + "10.237.0.173", + "10.31.153.177" ], "related.user": [ "sci" @@ -4137,8 +4137,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "eritqui", "rsa.misc.action": [ - "Blocked", - "dolor" + "dolor", + "Blocked" ], "rsa.misc.category": "taspe", "rsa.misc.filter": "oremipsu", @@ -4295,8 +4295,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "adipisc", "rsa.misc.action": [ - "exer", - "Blocked" + "Blocked", + "exer" ], "rsa.misc.category": "remagna", "rsa.misc.filter": "emvel", @@ -4377,8 +4377,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ecillum", "rsa.misc.action": [ - "Blocked", - "emp" + "emp", + "Blocked" ], "rsa.misc.category": "ciati", "rsa.misc.filter": "elit", @@ -4459,8 +4459,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iuntN", "rsa.misc.action": [ - "Allowed", - "nim" + "nim", + "Allowed" ], "rsa.misc.category": "etco", "rsa.misc.filter": "autodita", @@ -4623,8 +4623,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "inimve", "rsa.misc.action": [ - "niam", - "Allowed" + "Allowed", + "niam" ], "rsa.misc.category": "perspici", "rsa.misc.filter": "uipe", @@ -4705,8 +4705,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uamquaer", "rsa.misc.action": [ - "aerat", - "Blocked" + "Blocked", + "aerat" ], "rsa.misc.category": "quela", "rsa.misc.filter": "qui", @@ -4770,8 +4770,8 @@ "imveni193.www5.host" ], "related.ip": [ - "10.112.190.154", - "10.55.38.153" + "10.55.38.153", + "10.112.190.154" ], "related.user": [ "oremeu" @@ -4852,8 +4852,8 @@ "ionu3320.api.localhost" ], "related.ip": [ - "10.195.153.42", - "10.250.48.82" + "10.250.48.82", + "10.195.153.42" ], "related.user": [ "tsedquia" @@ -4867,8 +4867,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tDuisaut", "rsa.misc.action": [ - "Allowed", - "upidatat" + "upidatat", + "Allowed" ], "rsa.misc.category": "aliquide", "rsa.misc.filter": "deriti", @@ -5012,8 +5012,8 @@ "mdoloree96.domain" ], "related.ip": [ - "10.187.16.73", - "10.122.102.156" + "10.122.102.156", + "10.187.16.73" ], "related.user": [ "emoen" @@ -5271,8 +5271,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "vento", "rsa.misc.action": [ - "Blocked", - "reh" + "reh", + "Blocked" ], "rsa.misc.category": "atev", "rsa.misc.filter": "umq", @@ -5353,8 +5353,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "orinrep", "rsa.misc.action": [ - "Blocked", - "squirat" + "squirat", + "Blocked" ], "rsa.misc.category": "sequa", "rsa.misc.filter": "orainci", @@ -5517,8 +5517,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tNequepo", "rsa.misc.action": [ - "Allowed", - "rmagnido" + "rmagnido", + "Allowed" ], "rsa.misc.category": "luptatem", "rsa.misc.filter": "deritq", @@ -5599,8 +5599,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "plicab", "rsa.misc.action": [ - "Blocked", - "umq" + "umq", + "Blocked" ], "rsa.misc.category": "eruntmol", "rsa.misc.filter": "labore", @@ -5666,8 +5666,8 @@ "aturExc7343.invalid" ], "related.ip": [ - "10.146.69.38", - "10.55.192.102" + "10.55.192.102", + "10.146.69.38" ], "related.user": [ "quia" @@ -5763,8 +5763,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Utenim", "rsa.misc.action": [ - "Allowed", - "onevo" + "onevo", + "Allowed" ], "rsa.misc.category": "tdolore", "rsa.misc.filter": "ptasn", @@ -5994,8 +5994,8 @@ "ites5711.internal.host" ], "related.ip": [ - "10.24.23.209", - "10.162.78.48" + "10.162.78.48", + "10.24.23.209" ], "related.user": [ "ntore" @@ -6158,8 +6158,8 @@ "ngelitse7535.internal.lan" ], "related.ip": [ - "10.110.16.169", - "10.209.203.156" + "10.209.203.156", + "10.110.16.169" ], "related.user": [ "mes" @@ -6255,8 +6255,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "omnis", "rsa.misc.action": [ - "Allowed", - "uianonnu" + "uianonnu", + "Allowed" ], "rsa.misc.category": "Excepteu", "rsa.misc.filter": "enimadmi", @@ -6404,8 +6404,8 @@ "psaqu6066.www5.localhost" ], "related.ip": [ - "10.223.11.164", - "10.164.190.2" + "10.164.190.2", + "10.223.11.164" ], "related.user": [ "ten" @@ -6419,8 +6419,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "officiad", "rsa.misc.action": [ - "Allowed", - "antium" + "antium", + "Allowed" ], "rsa.misc.category": "emoeni", "rsa.misc.filter": "itvo", @@ -6568,8 +6568,8 @@ "uame1361.api.local" ], "related.ip": [ - "10.10.93.133", - "10.90.20.202" + "10.90.20.202", + "10.10.93.133" ], "related.user": [ "evita" @@ -6665,8 +6665,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Exce", "rsa.misc.action": [ - "ulapa", - "Allowed" + "Allowed", + "ulapa" ], "rsa.misc.category": "reprehen", "rsa.misc.filter": "itsedqui", @@ -6829,8 +6829,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uis", "rsa.misc.action": [ - "mvele", - "Allowed" + "Allowed", + "mvele" ], "rsa.misc.category": "vitaedi", "rsa.misc.filter": "ndeomni", @@ -7060,8 +7060,8 @@ "itaspe921.mail.invalid" ], "related.ip": [ - "10.10.25.145", - "10.224.249.228" + "10.224.249.228", + "10.10.25.145" ], "related.user": [ "mnisiuta" @@ -7075,8 +7075,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "issuscip", "rsa.misc.action": [ - "remap", - "Blocked" + "Blocked", + "remap" ], "rsa.misc.category": "eetdolo", "rsa.misc.filter": "rsitam", @@ -7142,8 +7142,8 @@ "archite4407.mail.invalid" ], "related.ip": [ - "10.247.255.107", - "10.234.34.40" + "10.234.34.40", + "10.247.255.107" ], "related.user": [ "aeabillo" @@ -7157,8 +7157,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "neavolu", "rsa.misc.action": [ - "nofdeF", - "Blocked" + "Blocked", + "nofdeF" ], "rsa.misc.category": "remagnam", "rsa.misc.filter": "maveniam", @@ -7239,8 +7239,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ilmoles", "rsa.misc.action": [ - "Blocked", - "tatisetq" + "tatisetq", + "Blocked" ], "rsa.misc.category": "ametco", "rsa.misc.filter": "liquide", @@ -7399,8 +7399,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "upta", "rsa.misc.action": [ - "Allowed", - "uovolup" + "uovolup", + "Allowed" ], "rsa.misc.category": "todit", "rsa.misc.filter": "atisetq", @@ -7641,8 +7641,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntut", "rsa.misc.action": [ - "nima", - "Blocked" + "Blocked", + "nima" ], "rsa.misc.category": "boru", "rsa.misc.filter": "umquia", @@ -8036,8 +8036,8 @@ "onsequ3168.www.corp" ], "related.ip": [ - "10.109.192.53", - "10.172.17.6" + "10.172.17.6", + "10.109.192.53" ], "related.user": [ "eprehen" @@ -8118,8 +8118,8 @@ "oremquel3120.internal.localhost" ], "related.ip": [ - "10.135.38.213", - "10.119.106.108" + "10.119.106.108", + "10.135.38.213" ], "related.user": [ "ore" @@ -8133,8 +8133,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "exeacomm", "rsa.misc.action": [ - "volup", - "Blocked" + "Blocked", + "volup" ], "rsa.misc.category": "ten", "rsa.misc.filter": "ssecil", From b01d8937023940d95c0a94dfdb0a5e82af24d0b2 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 27 Apr 2021 12:46:59 +0000 Subject: [PATCH 16/16] update generated data --- .../meraki/test/generated.log-expected.json | 100 +-- .../bigipafm/test/generated.log-expected.json | 410 +++++------ .../bigipapm/test/generated.log-expected.json | 8 +- .../test/generated.log-expected.json | 180 ++--- .../test/generated.log-expected.json | 24 +- .../test/generated.log-expected.json | 408 +++++------ .../test/generated.log-expected.json | 492 ++++++------- .../test/generated.log-expected.json | 24 +- .../utm/test/generated.log-expected.json | 64 +- .../squid/log/test/access1.log-expected.json | 364 +++++----- .../log/test/generated.log-expected.json | 666 +++++++++--------- .../intel/test/intel-json.log-expected.json | 4 +- .../zia/test/generated.log-expected.json | 352 ++++----- 13 files changed, 1548 insertions(+), 1548 deletions(-) diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json index 9871b8013dde..826a4f921947 100644 --- a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json @@ -17,8 +17,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.15.44.253", - "10.193.124.51" + "10.193.124.51", + "10.15.44.253" ], "rsa.internal.event_desc": "olaborissecurity_event tur", "rsa.internal.messageid": "security_event", @@ -66,8 +66,8 @@ "appliance" ], "related.ip": [ - "10.15.16.212", - "10.102.218.31" + "10.102.218.31", + "10.15.16.212" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -962,8 +962,8 @@ "appliance" ], "related.ip": [ - "10.93.68.231", - "10.135.217.12" + "10.135.217.12", + "10.93.68.231" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1029,8 +1029,8 @@ "appliance" ], "related.ip": [ - "10.247.30.212", - "10.66.89.5" + "10.66.89.5", + "10.247.30.212" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1094,8 +1094,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.221.102.245", - "10.173.136.186" + "10.173.136.186", + "10.221.102.245" ], "rsa.internal.event_desc": "idestlab", "rsa.internal.messageid": "security_event", @@ -1214,8 +1214,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.183.44.198", - "10.0.200.27" + "10.0.200.27", + "10.183.44.198" ], "rsa.internal.event_desc": "uradi security_event tot", "rsa.internal.messageid": "security_event", @@ -1252,8 +1252,8 @@ "appliance" ], "related.ip": [ - "10.28.144.180", - "10.148.124.84" + "10.148.124.84", + "10.28.144.180" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -1293,8 +1293,8 @@ "appliance" ], "related.ip": [ - "10.98.194.212", - "10.204.230.166" + "10.204.230.166", + "10.98.194.212" ], "rsa.counters.dclass_r1": "enimadmi", "rsa.internal.messageid": "events", @@ -1402,8 +1402,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.150.245.88", - "10.242.77.170" + "10.242.77.170", + "10.150.245.88" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1439,8 +1439,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.180.195.43", - "10.247.139.239" + "10.247.139.239", + "10.180.195.43" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1582,8 +1582,8 @@ "appliance" ], "related.ip": [ - "10.193.219.34", - "10.179.40.170" + "10.179.40.170", + "10.193.219.34" ], "rsa.counters.dclass_r1": "emip", "rsa.internal.messageid": "events", @@ -1740,8 +1740,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.90.99.245", - "10.124.63.4" + "10.124.63.4", + "10.90.99.245" ], "rsa.internal.event_desc": "etconsec", "rsa.internal.messageid": "security_event", @@ -1904,8 +1904,8 @@ "remips188.api.invalid" ], "related.ip": [ - "10.40.101.224", - "10.78.199.43" + "10.78.199.43", + "10.40.101.224" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -1979,8 +1979,8 @@ "appliance" ], "related.ip": [ - "10.39.172.93", - "10.83.131.245" + "10.83.131.245", + "10.39.172.93" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -2056,8 +2056,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.122.204.151", - "10.148.211.222" + "10.148.211.222", + "10.122.204.151" ], "rsa.internal.event_desc": "umexercisecurity_event duntut", "rsa.internal.messageid": "security_event", @@ -2135,8 +2135,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.120.4.9", - "10.97.46.16" + "10.97.46.16", + "10.120.4.9" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2171,8 +2171,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.165.173.162", - "10.171.206.139" + "10.171.206.139", + "10.165.173.162" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2209,8 +2209,8 @@ "uames4985.mail.localdomain" ], "related.ip": [ - "10.144.57.239", - "10.150.163.151" + "10.150.163.151", + "10.144.57.239" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2524,8 +2524,8 @@ "appliance" ], "related.ip": [ - "10.123.62.215", - "10.245.199.23" + "10.245.199.23", + "10.123.62.215" ], "rsa.db.index": "iusmodt", "rsa.internal.messageid": "flows", @@ -2642,8 +2642,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.34.62.190", - "10.246.152.72" + "10.246.152.72", + "10.34.62.190" ], "rsa.internal.event_desc": "Nem", "rsa.internal.messageid": "security_event", @@ -2929,8 +2929,8 @@ "appliance" ], "related.ip": [ - "10.65.0.157", - "10.17.111.91" + "10.17.111.91", + "10.65.0.157" ], "rsa.db.index": "nostrum", "rsa.internal.messageid": "flows", @@ -3067,8 +3067,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.51.121.223", - "10.199.103.185" + "10.199.103.185", + "10.51.121.223" ], "rsa.internal.event_desc": "dipi security_event ecatc", "rsa.internal.messageid": "security_event", @@ -3207,8 +3207,8 @@ "appliance" ], "related.ip": [ - "10.247.118.132", - "10.254.96.130" + "10.254.96.130", + "10.247.118.132" ], "rsa.counters.dclass_r1": "ectet", "rsa.internal.messageid": "events", @@ -3256,8 +3256,8 @@ "appliance" ], "related.ip": [ - "10.101.13.122", - "10.200.98.243" + "10.200.98.243", + "10.101.13.122" ], "rsa.counters.dclass_r1": "uteirur", "rsa.internal.messageid": "events", @@ -3344,8 +3344,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.195.90.73", - "10.147.165.30" + "10.147.165.30", + "10.195.90.73" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", diff --git a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json index 76300c43427f..f2227c07cb11 100644 --- a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json @@ -25,9 +25,9 @@ ], "related.ip": [ "10.165.201.71", - "10.11.196.142", + "10.208.121.85", "10.228.193.207", - "10.208.121.85" + "10.11.196.142" ], "related.user": [ "billoi" @@ -93,9 +93,9 @@ ], "related.ip": [ "10.92.202.200", - "10.94.67.230", "10.51.132.10", - "10.162.9.235" + "10.162.9.235", + "10.94.67.230" ], "related.user": [ "byC" @@ -161,9 +161,9 @@ ], "related.ip": [ "10.82.56.117", + "10.122.116.161", "10.209.155.149", - "10.191.68.244", - "10.122.116.161" + "10.191.68.244" ], "related.user": [ "seq" @@ -227,10 +227,10 @@ "uid545.www5.localhost" ], "related.ip": [ - "10.202.66.28", - "10.131.233.27", + "10.50.112.141", "10.12.44.169", - "10.50.112.141" + "10.131.233.27", + "10.202.66.28" ], "related.user": [ "elits" @@ -295,10 +295,10 @@ "emquiavo452.internal.localhost" ], "related.ip": [ + "10.96.35.212", "10.151.111.38", - "10.206.197.113", "10.159.182.171", - "10.96.35.212" + "10.206.197.113" ], "related.user": [ "mol" @@ -363,9 +363,9 @@ "sun1403.www.invalid" ], "related.ip": [ - "10.89.163.114", - "10.213.113.28", "10.126.177.162", + "10.213.113.28", + "10.89.163.114", "10.169.144.147" ], "related.user": [ @@ -430,10 +430,10 @@ "ittenbyC7838.api.localdomain" ], "related.ip": [ - "10.103.107.47", - "10.146.88.52", "10.101.223.43", - "10.18.124.28" + "10.103.107.47", + "10.18.124.28", + "10.146.88.52" ], "related.user": [ "rudexerc" @@ -498,10 +498,10 @@ "ume465.corp" ], "related.ip": [ - "10.150.220.75", "10.189.109.245", + "10.110.99.17", "10.69.57.206", - "10.110.99.17" + "10.150.220.75" ], "related.user": [ "onse" @@ -566,9 +566,9 @@ ], "related.ip": [ "10.121.219.204", + "10.153.136.222", "10.199.34.241", - "10.19.194.101", - "10.153.136.222" + "10.19.194.101" ], "related.user": [ "temveleu" @@ -632,10 +632,10 @@ "aliqu6801.api.localdomain" ], "related.ip": [ + "10.46.27.57", "10.57.103.192", "10.64.141.105", - "10.182.199.231", - "10.46.27.57" + "10.182.199.231" ], "related.user": [ "ice" @@ -699,9 +699,9 @@ "itame189.domain" ], "related.ip": [ + "10.3.134.237", "10.32.67.231", "10.164.6.207", - "10.3.134.237", "10.160.210.31" ], "related.user": [ @@ -768,8 +768,8 @@ ], "related.ip": [ "10.235.101.253", - "10.182.178.217", "10.42.138.192", + "10.182.178.217", "10.201.6.10" ], "related.user": [ @@ -835,10 +835,10 @@ "stlabo1228.mail.host" ], "related.ip": [ + "10.151.161.70", "10.22.102.198", - "10.86.101.235", "10.194.247.171", - "10.151.161.70" + "10.86.101.235" ], "related.user": [ "nse" @@ -903,10 +903,10 @@ "ecte4762.local" ], "related.ip": [ - "10.174.252.105", "10.107.168.60", "10.167.172.155", - "10.204.35.15" + "10.204.35.15", + "10.174.252.105" ], "related.user": [ "mnisi" @@ -970,8 +970,8 @@ "smo7167.www.test" ], "related.ip": [ - "10.99.249.210", "10.214.249.164", + "10.99.249.210", "10.182.191.174", "10.81.26.208" ], @@ -1037,8 +1037,8 @@ "sauteiru4554.api.domain" ], "related.ip": [ - "10.201.238.90", "10.220.5.143", + "10.201.238.90", "10.101.226.128", "10.88.101.53" ], @@ -1104,10 +1104,10 @@ "untut4046.internal.domain" ], "related.ip": [ - "10.243.218.215", - "10.157.18.252", "10.217.150.196", - "10.30.133.66" + "10.30.133.66", + "10.157.18.252", + "10.243.218.215" ], "related.user": [ "evit" @@ -1171,10 +1171,10 @@ "quid3147.mail.home" ], "related.ip": [ - "10.167.227.44", - "10.148.161.250", "10.66.181.6", - "10.181.133.187" + "10.181.133.187", + "10.167.227.44", + "10.148.161.250" ], "related.user": [ "adipisc" @@ -1239,10 +1239,10 @@ "umdolo1029.mail.localhost" ], "related.ip": [ + "10.84.163.178", "10.107.9.163", "10.74.11.43", - "10.54.17.32", - "10.84.163.178" + "10.54.17.32" ], "related.user": [ "mquisno" @@ -1307,9 +1307,9 @@ ], "related.ip": [ "10.184.73.211", + "10.192.229.221", "10.112.32.213", - "10.230.129.252", - "10.192.229.221" + "10.230.129.252" ], "related.user": [ "odi" @@ -1374,8 +1374,8 @@ "paquioff624.mail.invalid" ], "related.ip": [ - "10.7.200.140", "10.198.213.189", + "10.7.200.140", "10.199.216.143", "10.161.148.64" ], @@ -1442,9 +1442,9 @@ ], "related.ip": [ "10.65.232.27", + "10.206.96.56", "10.128.157.27", - "10.22.187.69", - "10.206.96.56" + "10.22.187.69" ], "related.user": [ "uaeab" @@ -1576,10 +1576,10 @@ "loi7596.www5.home" ], "related.ip": [ - "10.45.253.103", - "10.31.177.226", "10.107.45.175", - "10.47.255.237" + "10.47.255.237", + "10.31.177.226", + "10.45.253.103" ], "related.user": [ "remagn" @@ -1645,9 +1645,9 @@ ], "related.ip": [ "10.213.94.135", + "10.225.212.189", "10.55.105.113", - "10.44.58.106", - "10.225.212.189" + "10.44.58.106" ], "related.user": [ "dquia" @@ -1711,10 +1711,10 @@ "ectiono2241.lan" ], "related.ip": [ - "10.163.209.70", - "10.69.161.78", "10.255.74.136", - "10.2.114.9" + "10.2.114.9", + "10.163.209.70", + "10.69.161.78" ], "related.user": [ "olabor" @@ -1778,10 +1778,10 @@ "umetMal1664.mail.lan" ], "related.ip": [ + "10.252.102.110", "10.12.129.137", "10.46.115.216", - "10.184.59.148", - "10.252.102.110" + "10.184.59.148" ], "related.user": [ "perspici" @@ -1846,9 +1846,9 @@ "derit5270.mail.local" ], "related.ip": [ - "10.155.204.243", "10.199.194.79", "10.81.184.7", + "10.155.204.243", "10.105.52.140" ], "related.user": [ @@ -1914,10 +1914,10 @@ "orisni5238.mail.lan" ], "related.ip": [ - "10.177.238.45", - "10.18.226.72", "10.251.231.142", - "10.110.2.166" + "10.177.238.45", + "10.110.2.166", + "10.18.226.72" ], "related.user": [ "taliqui" @@ -2117,10 +2117,10 @@ "oinv5493.internal.domain" ], "related.ip": [ + "10.36.63.31", "10.171.221.230", - "10.45.35.180", "10.222.165.250", - "10.36.63.31" + "10.45.35.180" ], "related.user": [ "otamr" @@ -2184,9 +2184,9 @@ "tnonproi195.api.home" ], "related.ip": [ - "10.199.127.211", - "10.83.238.145", "10.238.4.219", + "10.83.238.145", + "10.199.127.211", "10.1.171.61" ], "related.user": [ @@ -2252,8 +2252,8 @@ ], "related.ip": [ "10.170.252.219", - "10.44.226.104", "10.65.141.244", + "10.44.226.104", "10.74.213.42" ], "related.user": [ @@ -2318,10 +2318,10 @@ "uido492.www5.home" ], "related.ip": [ - "10.183.223.149", "10.225.141.172", - "10.180.48.221", - "10.225.255.211" + "10.183.223.149", + "10.225.255.211", + "10.180.48.221" ], "related.user": [ "nihil" @@ -2385,10 +2385,10 @@ "redo6311.api.invalid" ], "related.ip": [ - "10.205.174.181", - "10.176.64.28", "10.169.123.103", - "10.97.138.181" + "10.176.64.28", + "10.97.138.181", + "10.205.174.181" ], "related.user": [ "eseruntm" @@ -2453,10 +2453,10 @@ "dolorem1698.www.domain" ], "related.ip": [ - "10.169.101.161", "10.75.120.11", - "10.53.101.131", - "10.204.4.40" + "10.204.4.40", + "10.169.101.161", + "10.53.101.131" ], "related.user": [ "tquo" @@ -2521,10 +2521,10 @@ "evitae7333.www.lan" ], "related.ip": [ + "10.28.51.219", "10.6.222.112", "10.156.117.169", - "10.87.120.87", - "10.28.51.219" + "10.87.120.87" ], "related.user": [ "onsequu" @@ -2589,9 +2589,9 @@ ], "related.ip": [ "10.247.44.59", - "10.57.89.155", "10.253.167.17", - "10.4.126.103" + "10.4.126.103", + "10.57.89.155" ], "related.user": [ "ntorever" @@ -2656,9 +2656,9 @@ ], "related.ip": [ "10.15.240.220", + "10.248.206.210", "10.36.69.125", - "10.143.183.208", - "10.248.206.210" + "10.143.183.208" ], "related.user": [ "met" @@ -2723,9 +2723,9 @@ "edqu2208.www.localhost" ], "related.ip": [ - "10.142.186.43", "10.34.133.2", "10.69.170.107", + "10.142.186.43", "10.6.32.7" ], "related.user": [ @@ -2791,10 +2791,10 @@ "ender5647.www5.example" ], "related.ip": [ - "10.121.153.197", "10.142.22.24", - "10.59.103.10", - "10.170.165.164" + "10.170.165.164", + "10.121.153.197", + "10.59.103.10" ], "related.user": [ "borumSec" @@ -2859,10 +2859,10 @@ "sis3986.internal.lan" ], "related.ip": [ - "10.247.114.30", "10.19.99.129", - "10.176.83.7", - "10.133.10.122" + "10.133.10.122", + "10.247.114.30", + "10.176.83.7" ], "related.user": [ "quaeabil" @@ -2928,9 +2928,9 @@ ], "related.ip": [ "10.8.29.219", - "10.70.7.23", + "10.40.177.138", "10.64.139.17", - "10.40.177.138" + "10.70.7.23" ], "related.user": [ "rep" @@ -2996,8 +2996,8 @@ "related.ip": [ "10.67.173.228", "10.180.62.222", - "10.67.221.220", - "10.2.189.20" + "10.2.189.20", + "10.67.221.220" ], "related.user": [ "uptasnul" @@ -3062,10 +3062,10 @@ "uian521.www.example" ], "related.ip": [ - "10.56.134.118", - "10.209.52.47", "10.196.176.243", - "10.147.127.181" + "10.147.127.181", + "10.56.134.118", + "10.209.52.47" ], "related.user": [ "tasu" @@ -3129,10 +3129,10 @@ "taliq5213.api.corp" ], "related.ip": [ - "10.226.24.84", "10.85.13.237", + "10.248.140.59", "10.231.18.90", - "10.248.140.59" + "10.226.24.84" ], "related.user": [ "Nem" @@ -3197,9 +3197,9 @@ "ntsunt4894.mail.domain" ], "related.ip": [ - "10.59.215.207", "10.207.183.204", "10.8.224.72", + "10.59.215.207", "10.203.46.215" ], "related.user": [ @@ -3265,8 +3265,8 @@ "mexer3864.api.corp" ], "related.ip": [ - "10.98.154.146", "10.73.84.95", + "10.98.154.146", "10.255.145.22", "10.230.38.148" ], @@ -3332,8 +3332,8 @@ "oluptat6960.www5.test" ], "related.ip": [ - "10.166.142.198", "10.105.120.162", + "10.166.142.198", "10.175.181.138", "10.211.29.187" ], @@ -3400,9 +3400,9 @@ "fugiatnu2498.www.localhost" ], "related.ip": [ - "10.195.139.25", - "10.220.202.102", "10.122.133.162", + "10.220.202.102", + "10.195.139.25", "10.182.213.195" ], "related.user": [ @@ -3468,9 +3468,9 @@ "ptat3230.domain" ], "related.ip": [ - "10.156.208.5", "10.53.72.161", "10.33.143.163", + "10.156.208.5", "10.247.144.9" ], "related.user": [ @@ -3536,9 +3536,9 @@ ], "related.ip": [ "10.241.143.145", - "10.113.65.192", "10.35.190.164", - "10.21.58.162" + "10.21.58.162", + "10.113.65.192" ], "related.user": [ "porin" @@ -3603,10 +3603,10 @@ "itanimi1934.home" ], "related.ip": [ - "10.19.154.103", - "10.75.113.240", "10.53.27.253", - "10.129.16.166" + "10.129.16.166", + "10.19.154.103", + "10.75.113.240" ], "related.user": [ "luptat" @@ -3671,10 +3671,10 @@ "pteurs1031.mail.corp" ], "related.ip": [ - "10.22.213.196", - "10.125.150.220", + "10.150.153.61", "10.120.50.13", - "10.150.153.61" + "10.22.213.196", + "10.125.150.220" ], "related.user": [ "inculpa" @@ -3739,10 +3739,10 @@ "edquiaco6562.api.lan" ], "related.ip": [ - "10.113.2.13", - "10.238.171.184", "10.85.52.249", - "10.229.155.171" + "10.238.171.184", + "10.229.155.171", + "10.113.2.13" ], "related.user": [ "tatiset" @@ -3807,10 +3807,10 @@ "tatis7315.mail.home" ], "related.ip": [ - "10.198.150.185", - "10.249.174.35", + "10.51.245.225", "10.220.1.249", - "10.51.245.225" + "10.249.174.35", + "10.198.150.185" ], "related.user": [ "quela" @@ -3875,10 +3875,10 @@ "eosqui3723.api.localdomain" ], "related.ip": [ - "10.38.185.31", "10.152.157.32", - "10.251.82.195", - "10.190.96.181" + "10.190.96.181", + "10.38.185.31", + "10.251.82.195" ], "related.user": [ "olorese" @@ -3942,10 +3942,10 @@ "itaedict199.mail.corp" ], "related.ip": [ + "10.190.247.194", "10.211.198.50", "10.103.102.242", - "10.230.112.179", - "10.190.247.194" + "10.230.112.179" ], "related.user": [ "tDuisaut" @@ -4009,10 +4009,10 @@ "xeaco7887.www.localdomain" ], "related.ip": [ - "10.251.101.61", - "10.101.13.122", "10.47.223.155", - "10.219.83.199" + "10.251.101.61", + "10.219.83.199", + "10.101.13.122" ], "related.user": [ "ectetur" @@ -4077,10 +4077,10 @@ "saute7421.www.invalid" ], "related.ip": [ - "10.83.136.233", - "10.31.86.83", + "10.21.30.43", "10.21.80.157", - "10.21.30.43" + "10.31.86.83", + "10.83.136.233" ], "related.user": [ "litsed" @@ -4146,8 +4146,8 @@ ], "related.ip": [ "10.45.152.205", - "10.195.90.73", "10.27.181.27", + "10.195.90.73", "10.194.197.107" ], "related.user": [ @@ -4213,10 +4213,10 @@ "ididu5505.api.localdomain" ], "related.ip": [ - "10.43.239.97", - "10.183.90.25", "10.222.2.132", - "10.129.161.18" + "10.129.161.18", + "10.183.90.25", + "10.43.239.97" ], "related.user": [ "aedicta" @@ -4280,10 +4280,10 @@ "mqui1099.api.corp" ], "related.ip": [ + "10.189.162.131", "10.67.129.100", "10.248.156.138", - "10.231.167.171", - "10.189.162.131" + "10.231.167.171" ], "related.user": [ "sedquia" @@ -4416,8 +4416,8 @@ ], "related.ip": [ "10.214.93.200", - "10.119.179.182", "10.0.202.9", + "10.119.179.182", "10.93.39.237" ], "related.user": [ @@ -4483,10 +4483,10 @@ "aute2433.mail.lan" ], "related.ip": [ - "10.30.189.166", + "10.123.154.140", "10.252.204.162", "10.28.145.163", - "10.123.154.140" + "10.30.189.166" ], "related.user": [ "imadmin" @@ -4550,10 +4550,10 @@ "idolo6535.internal.example" ], "related.ip": [ - "10.29.122.183", - "10.46.162.198", "10.145.128.250", - "10.79.49.3" + "10.29.122.183", + "10.79.49.3", + "10.46.162.198" ], "related.user": [ "eni" @@ -4618,10 +4618,10 @@ "one7728.api.localdomain" ], "related.ip": [ + "10.142.235.217", "10.166.169.167", "10.65.174.196", - "10.177.232.136", - "10.142.235.217" + "10.177.232.136" ], "related.user": [ "olors" @@ -4686,10 +4686,10 @@ "uptatem4446.internal.localhost" ], "related.ip": [ - "10.215.184.154", "10.29.217.44", - "10.53.188.140", - "10.191.78.86" + "10.215.184.154", + "10.191.78.86", + "10.53.188.140" ], "related.user": [ "iarc" @@ -4754,10 +4754,10 @@ "emq2514.api.localhost" ], "related.ip": [ - "10.135.77.156", + "10.46.222.149", "10.74.74.129", - "10.76.148.147", - "10.46.222.149" + "10.135.77.156", + "10.76.148.147" ], "related.user": [ "urve" @@ -4821,10 +4821,10 @@ "agna5654.www.corp" ], "related.ip": [ - "10.145.49.29", - "10.130.203.37", "10.96.200.223", - "10.11.146.253" + "10.11.146.253", + "10.130.203.37", + "10.145.49.29" ], "related.user": [ "mvele" @@ -4888,8 +4888,8 @@ "ipi4827.mail.lan" ], "related.ip": [ - "10.162.2.180", "10.162.78.48", + "10.162.2.180", "10.24.23.209", "10.48.75.140" ], @@ -4955,10 +4955,10 @@ "sequatD163.internal.example" ], "related.ip": [ - "10.151.206.38", + "10.97.105.115", "10.119.12.186", - "10.66.92.83", - "10.97.105.115" + "10.151.206.38", + "10.66.92.83" ], "related.user": [ "nproide" @@ -5023,9 +5023,9 @@ ], "related.ip": [ "10.201.132.114", - "10.169.139.250", "10.12.148.73", - "10.64.76.142" + "10.64.76.142", + "10.169.139.250" ], "related.user": [ "borisnis" @@ -5090,10 +5090,10 @@ "epr3512.internal.domain" ], "related.ip": [ - "10.9.236.18", "10.111.128.11", - "10.35.38.185", - "10.200.116.191" + "10.200.116.191", + "10.9.236.18", + "10.35.38.185" ], "related.user": [ "umfug" @@ -5157,10 +5157,10 @@ "uredol2174.home" ], "related.ip": [ - "10.134.238.8", - "10.191.27.182", + "10.236.67.227", "10.240.62.238", - "10.236.67.227" + "10.191.27.182", + "10.134.238.8" ], "related.user": [ "tlabo" @@ -5224,9 +5224,9 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.22.231.91", "10.65.35.64", "10.109.14.142", + "10.22.231.91", "10.165.66.92" ], "related.user": [ @@ -5291,10 +5291,10 @@ "inimav5557.www5.test" ], "related.ip": [ - "10.89.221.90", + "10.29.230.203", "10.71.112.86", - "10.64.161.215", - "10.29.230.203" + "10.89.221.90", + "10.64.161.215" ], "related.user": [ "rnatur" @@ -5358,10 +5358,10 @@ "nonn1650.www.test" ], "related.ip": [ - "10.140.118.182", - "10.88.226.76", "10.79.208.135", - "10.221.199.137" + "10.140.118.182", + "10.221.199.137", + "10.88.226.76" ], "related.user": [ "erspic" @@ -5426,10 +5426,10 @@ "acons3940.api.lan" ], "related.ip": [ - "10.126.61.230", - "10.133.48.55", "10.35.73.208", - "10.189.244.22" + "10.189.244.22", + "10.133.48.55", + "10.126.61.230" ], "related.user": [ "tia" @@ -5494,9 +5494,9 @@ ], "related.ip": [ "10.240.94.109", + "10.35.65.72", "10.239.194.105", - "10.81.154.115", - "10.35.65.72" + "10.81.154.115" ], "related.user": [ "reseo" @@ -5561,10 +5561,10 @@ "mnisiut6146.internal.local" ], "related.ip": [ + "10.52.70.192", "10.38.253.213", - "10.248.72.104", "10.150.56.227", - "10.52.70.192" + "10.248.72.104" ], "related.user": [ "ionem" @@ -5629,9 +5629,9 @@ "borios1067.www5.home" ], "related.ip": [ - "10.203.193.134", - "10.62.218.239", "10.218.15.164", + "10.62.218.239", + "10.203.193.134", "10.73.172.186" ], "related.user": [ @@ -5696,10 +5696,10 @@ "msequ323.www.example" ], "related.ip": [ - "10.10.46.43", - "10.60.20.76", "10.136.211.234", - "10.131.127.113" + "10.131.127.113", + "10.60.20.76", + "10.10.46.43" ], "related.user": [ "nev" @@ -5764,9 +5764,9 @@ "tdolorem813.internal.host" ], "related.ip": [ - "10.248.0.74", "10.233.181.250", "10.50.177.151", + "10.248.0.74", "10.187.237.220" ], "related.user": [ @@ -5832,10 +5832,10 @@ "volupt4626.internal.test" ], "related.ip": [ - "10.80.129.81", + "10.189.43.11", "10.248.248.120", - "10.96.223.46", - "10.189.43.11" + "10.80.129.81", + "10.96.223.46" ], "related.user": [ "iatn" @@ -5900,10 +5900,10 @@ "ntium5103.www5.localhost" ], "related.ip": [ + "10.91.115.139", "10.102.109.199", - "10.173.114.63", "10.66.106.186", - "10.91.115.139" + "10.173.114.63" ], "related.user": [ "tNequ" @@ -5968,9 +5968,9 @@ "orpori3334.www.local" ], "related.ip": [ + "10.221.223.127", "10.0.175.17", "10.159.155.88", - "10.221.223.127", "10.198.157.122" ], "related.user": [ @@ -6035,9 +6035,9 @@ "equu7361.www5.localdomain" ], "related.ip": [ - "10.7.212.201", "10.30.20.187", "10.189.70.237", + "10.7.212.201", "10.252.136.130" ], "related.user": [ @@ -6103,10 +6103,10 @@ "tse2979.internal.localhost" ], "related.ip": [ + "10.60.224.93", "10.83.105.69", - "10.242.121.165", "10.102.109.194", - "10.60.224.93" + "10.242.121.165" ], "related.user": [ "mni" @@ -6172,9 +6172,9 @@ ], "related.ip": [ "10.181.134.69", + "10.251.167.219", "10.219.174.45", - "10.17.20.93", - "10.251.167.219" + "10.17.20.93" ], "related.user": [ "Uteni" @@ -6239,10 +6239,10 @@ "luptate4811.mail.example" ], "related.ip": [ - "10.30.117.82", - "10.37.14.20", "10.28.233.253", - "10.223.99.90" + "10.223.99.90", + "10.30.117.82", + "10.37.14.20" ], "related.user": [ "numqua" @@ -6307,8 +6307,8 @@ "lites1614.www.corp" ], "related.ip": [ - "10.125.20.22", "10.8.32.17", + "10.125.20.22", "10.50.61.114", "10.57.85.113" ], @@ -6376,8 +6376,8 @@ ], "related.ip": [ "10.113.78.101", - "10.200.28.55", "10.215.224.27", + "10.200.28.55", "10.181.63.82" ], "related.user": [ @@ -6443,10 +6443,10 @@ "nderit6272.mail.example" ], "related.ip": [ - "10.177.14.106", - "10.243.43.168", "10.169.95.128", - "10.139.20.223" + "10.177.14.106", + "10.139.20.223", + "10.243.43.168" ], "related.user": [ "ofd" @@ -6511,9 +6511,9 @@ "ntu1279.mail.lan" ], "related.ip": [ + "10.39.100.88", "10.90.93.4", "10.92.168.198", - "10.39.100.88", "10.18.176.44" ], "related.user": [ @@ -6648,8 +6648,8 @@ "related.ip": [ "10.84.64.28", "10.209.226.7", - "10.240.47.113", - "10.31.147.51" + "10.31.147.51", + "10.240.47.113" ], "related.user": [ "ull" @@ -6714,10 +6714,10 @@ "item3647.home" ], "related.ip": [ - "10.32.20.4", - "10.225.189.229", "10.86.1.244", - "10.52.13.192" + "10.32.20.4", + "10.52.13.192", + "10.225.189.229" ], "related.user": [ "odtemp" @@ -6757,4 +6757,4 @@ ], "user.name": "odtemp" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json index a9d1a6036046..244c0a399b2d 100644 --- a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json @@ -1482,8 +1482,8 @@ "observer.vendor": "F5", "process.pid": 2943, "related.ip": [ - "10.16.181.60", - "10.142.213.80" + "10.142.213.80", + "10.16.181.60" ], "rsa.internal.messageid": "01490549", "rsa.misc.group": "tationu", @@ -1573,8 +1573,8 @@ "observer.vendor": "F5", "process.pid": 1973, "related.ip": [ - "10.47.99.72", - "10.187.64.126" + "10.187.64.126", + "10.47.99.72" ], "rsa.internal.messageid": "01490500", "rsa.misc.category": "oremipsu", diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json index 21e6b8a0246f..5880bc2c97e4 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json @@ -335,8 +335,8 @@ "enim5316.www5.local" ], "related.ip": [ - "10.202.72.124", - "10.200.188.142" + "10.200.188.142", + "10.202.72.124" ], "related.user": [ "iusmodt" @@ -521,8 +521,8 @@ "usmodte1296.www.corp" ], "related.ip": [ - "10.69.20.77", - "10.178.244.31" + "10.178.244.31", + "10.69.20.77" ], "related.user": [ "umdolor" @@ -644,8 +644,8 @@ "tatno6787.internal.localhost" ], "related.ip": [ - "10.136.252.240", - "10.65.83.160" + "10.65.83.160", + "10.136.252.240" ], "related.user": [ "ender" @@ -767,8 +767,8 @@ "ali6446.localhost" ], "related.ip": [ - "10.200.156.102", - "10.144.82.69" + "10.144.82.69", + "10.200.156.102" ], "related.user": [ "rveli" @@ -828,8 +828,8 @@ "torev7118.internal.domain" ], "related.ip": [ - "10.109.232.112", - "10.72.58.135" + "10.72.58.135", + "10.109.232.112" ], "related.user": [ "xea" @@ -1014,8 +1014,8 @@ "lumquido5839.api.corp" ], "related.ip": [ - "10.73.69.75", - "10.19.201.13" + "10.19.201.13", + "10.73.69.75" ], "related.user": [ "tat" @@ -1075,8 +1075,8 @@ "aperia4409.www5.invalid" ], "related.ip": [ - "10.84.105.75", - "10.78.151.178" + "10.78.151.178", + "10.84.105.75" ], "related.user": [ "iquaUten" @@ -1323,8 +1323,8 @@ "umexerci1284.internal.localdomain" ], "related.ip": [ - "10.141.44.153", - "10.161.57.8" + "10.161.57.8", + "10.141.44.153" ], "related.user": [ "quisnos" @@ -1446,8 +1446,8 @@ "evita5008.www.localdomain" ], "related.ip": [ - "10.134.148.219", - "10.248.204.182" + "10.248.204.182", + "10.134.148.219" ], "related.user": [ "uioffi" @@ -1507,8 +1507,8 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ - "10.178.77.231", - "10.163.5.243" + "10.163.5.243", + "10.178.77.231" ], "related.user": [ "liquide" @@ -1630,8 +1630,8 @@ "reme622.mail.example" ], "related.ip": [ - "10.241.65.49", - "10.32.239.1" + "10.32.239.1", + "10.241.65.49" ], "related.user": [ "idata" @@ -1691,8 +1691,8 @@ "non3341.mail.invalid" ], "related.ip": [ - "10.168.90.81", - "10.101.57.120" + "10.101.57.120", + "10.168.90.81" ], "related.user": [ "eporr" @@ -1938,8 +1938,8 @@ "vita2681.www5.local" ], "related.ip": [ - "10.27.14.168", - "10.66.2.232" + "10.66.2.232", + "10.27.14.168" ], "related.user": [ "uirati" @@ -2185,8 +2185,8 @@ "eumiu765.api.lan" ], "related.ip": [ - "10.4.157.1", - "10.184.18.202" + "10.184.18.202", + "10.4.157.1" ], "related.user": [ "oditem" @@ -2309,8 +2309,8 @@ "idestlab2631.www.lan" ], "related.ip": [ - "10.27.16.118", - "10.83.177.2" + "10.83.177.2", + "10.27.16.118" ], "related.user": [ "borios" @@ -2432,8 +2432,8 @@ "ccaeca7077.internal.corp" ], "related.ip": [ - "10.216.54.184", - "10.215.205.216" + "10.215.205.216", + "10.216.54.184" ], "related.user": [ "ameiusm" @@ -2494,8 +2494,8 @@ "ima2031.api.corp" ], "related.ip": [ - "10.9.18.237", - "10.9.12.248" + "10.9.12.248", + "10.9.18.237" ], "related.user": [ "uradi" @@ -2556,8 +2556,8 @@ "ian867.internal.corp" ], "related.ip": [ - "10.41.123.102", - "10.83.130.226" + "10.83.130.226", + "10.41.123.102" ], "related.user": [ "tenim" @@ -2680,8 +2680,8 @@ "gnaaliqu3935.api.test" ], "related.ip": [ - "10.142.25.100", - "10.134.18.114" + "10.134.18.114", + "10.142.25.100" ], "related.user": [ "osqui" @@ -2803,8 +2803,8 @@ "ritin2495.api.corp" ], "related.ip": [ - "10.47.28.48", - "10.110.114.175" + "10.110.114.175", + "10.47.28.48" ], "related.user": [ "plicab" @@ -2927,8 +2927,8 @@ "rem7043.localhost" ], "related.ip": [ - "10.227.173.252", - "10.65.2.106" + "10.65.2.106", + "10.227.173.252" ], "related.user": [ "itation" @@ -3050,8 +3050,8 @@ "dqu6144.api.localhost" ], "related.ip": [ - "10.150.245.88", - "10.210.89.183" + "10.210.89.183", + "10.150.245.88" ], "related.user": [ "sequa" @@ -3112,8 +3112,8 @@ "giatquov1918.internal.example" ], "related.ip": [ - "10.180.195.43", - "10.85.185.13" + "10.85.185.13", + "10.180.195.43" ], "related.user": [ "voluptas" @@ -3236,8 +3236,8 @@ "Sedut1775.www.domain" ], "related.ip": [ - "10.248.165.185", - "10.86.11.48" + "10.86.11.48", + "10.248.165.185" ], "related.user": [ "dquiac" @@ -3298,8 +3298,8 @@ "mac7484.www5.test" ], "related.ip": [ - "10.47.125.38", - "10.118.6.177" + "10.118.6.177", + "10.47.125.38" ], "related.user": [ "quunt" @@ -3422,8 +3422,8 @@ "naaliq3710.api.local" ], "related.ip": [ - "10.120.10.211", - "10.28.82.189" + "10.28.82.189", + "10.120.10.211" ], "related.user": [ "rcit" @@ -3546,8 +3546,8 @@ "onse380.internal.localdomain" ], "related.ip": [ - "10.226.5.189", - "10.125.165.144" + "10.125.165.144", + "10.226.5.189" ], "related.user": [ "mvolu" @@ -3732,8 +3732,8 @@ "minim459.mail.local" ], "related.ip": [ - "10.123.199.198", - "10.17.87.79" + "10.17.87.79", + "10.123.199.198" ], "related.user": [ "ratvolu" @@ -3794,8 +3794,8 @@ "eratv211.api.host" ], "related.ip": [ - "10.38.86.177", - "10.115.68.40" + "10.115.68.40", + "10.38.86.177" ], "related.user": [ "mpo" @@ -3856,8 +3856,8 @@ "aparia1179.www.localdomain" ], "related.ip": [ - "10.193.118.163", - "10.115.174.107" + "10.115.174.107", + "10.193.118.163" ], "related.user": [ "exeacomm" @@ -4042,8 +4042,8 @@ "mag1506.internal.domain" ], "related.ip": [ - "10.131.126.109", - "10.182.152.242" + "10.182.152.242", + "10.131.126.109" ], "related.user": [ "dolor" @@ -4104,8 +4104,8 @@ "fugits1163.host" ], "related.ip": [ - "10.77.229.168", - "10.181.247.224" + "10.181.247.224", + "10.77.229.168" ], "related.user": [ "adol" @@ -4351,8 +4351,8 @@ "abor1370.www.domain" ], "related.ip": [ - "10.77.78.180", - "10.97.236.123" + "10.97.236.123", + "10.77.78.180" ], "related.user": [ "nisi" @@ -4536,8 +4536,8 @@ "venia2079.mail.example" ], "related.ip": [ - "10.65.144.51", - "10.5.11.205" + "10.5.11.205", + "10.65.144.51" ], "related.user": [ "uptat" @@ -4660,8 +4660,8 @@ "gelitsed3249.corp" ], "related.ip": [ - "10.138.210.116", - "10.225.255.211" + "10.225.255.211", + "10.138.210.116" ], "related.user": [ "fugiatn" @@ -4722,8 +4722,8 @@ "dolor7082.internal.localhost" ], "related.ip": [ - "10.250.81.189", - "10.219.1.151" + "10.219.1.151", + "10.250.81.189" ], "related.user": [ "ori" @@ -4783,8 +4783,8 @@ "totam6886.api.localhost" ], "related.ip": [ - "10.76.125.70", - "10.54.23.133" + "10.54.23.133", + "10.76.125.70" ], "related.user": [ "oloreeu" @@ -4844,8 +4844,8 @@ "laborum5749.www.example" ], "related.ip": [ - "10.36.110.69", - "10.189.42.62" + "10.189.42.62", + "10.36.110.69" ], "related.user": [ "eque" @@ -5030,8 +5030,8 @@ "udan6536.www5.test" ], "related.ip": [ - "10.14.204.36", - "10.85.104.146" + "10.85.104.146", + "10.14.204.36" ], "related.user": [ "emp" @@ -5092,8 +5092,8 @@ "rumet6923.www5.lan" ], "related.ip": [ - "10.30.246.132", - "10.208.18.210" + "10.208.18.210", + "10.30.246.132" ], "related.user": [ "veniam" @@ -5278,8 +5278,8 @@ "enbyCi3813.api.domain" ], "related.ip": [ - "10.164.207.42", - "10.164.120.197" + "10.164.120.197", + "10.164.207.42" ], "related.user": [ "pta" @@ -5339,8 +5339,8 @@ "liquipex1155.mail.corp" ], "related.ip": [ - "10.183.189.133", - "10.154.191.225" + "10.154.191.225", + "10.183.189.133" ], "related.user": [ "ita" @@ -5648,8 +5648,8 @@ "reprehen3513.test" ], "related.ip": [ - "10.10.86.55", - "10.61.225.196" + "10.61.225.196", + "10.10.86.55" ], "related.user": [ "eniamqu" @@ -5834,8 +5834,8 @@ "rumSecti111.www5.domain" ], "related.ip": [ - "10.87.90.49", - "10.222.245.80" + "10.222.245.80", + "10.87.90.49" ], "related.user": [ "ptatemse" @@ -6020,8 +6020,8 @@ "uptasnul2751.www5.corp" ], "related.ip": [ - "10.161.64.168", - "10.194.67.223" + "10.194.67.223", + "10.161.64.168" ], "related.user": [ "tion" diff --git a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json index fa5418e95068..afb294c38149 100644 --- a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json @@ -2680,8 +2680,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "taevit4968.mail.local", - "mveni5084.internal.local" + "mveni5084.internal.local", + "taevit4968.mail.local" ], "related.ip": [ "10.62.61.1", @@ -3131,12 +3131,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "modi6930.internal.test", - "taevitae6868.www.corp" + "taevitae6868.www.corp", + "modi6930.internal.test" ], "related.ip": [ - "10.60.164.100", - "10.161.1.146" + "10.161.1.146", + "10.60.164.100" ], "rsa.email.email_dst": "nproiden", "rsa.email.email_src": "etconse", @@ -3239,8 +3239,8 @@ "uradip7802.mail.example" ], "related.ip": [ - "10.44.35.57", - "10.93.239.216" + "10.93.239.216", + "10.44.35.57" ], "rsa.email.email_dst": "ciun", "rsa.email.email_src": "vento", @@ -3845,8 +3845,8 @@ "ncu3839.www.localhost" ], "related.ip": [ - "10.201.105.58", - "10.251.183.113" + "10.251.183.113", + "10.201.105.58" ], "rsa.email.email_dst": "ionemu", "rsa.email.email_src": "ent", @@ -3889,8 +3889,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.ip": [ - "10.209.203.156", - "10.132.139.98" + "10.132.139.98", + "10.209.203.156" ], "rsa.email.email_dst": "borisnis", "rsa.email.email_src": "pariat", diff --git a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json index 8b4c2711d482..3f078976410f 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json @@ -28,8 +28,8 @@ "modtempo" ], "related.ip": [ - "10.44.173.44", "10.189.58.145", + "10.44.173.44", "10.20.234.169" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -99,8 +99,8 @@ "aer445.host" ], "related.ip": [ - "10.171.204.166", - "10.62.4.246" + "10.62.4.246", + "10.171.204.166" ], "related.user": [ "oluptas" @@ -196,9 +196,9 @@ "ccaecat" ], "related.ip": [ - "10.15.159.80", "10.94.103.117", - "10.200.188.142" + "10.200.188.142", + "10.15.159.80" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -264,8 +264,8 @@ ], "related.ip": [ "10.50.112.141", - "10.27.88.95", - "10.131.233.27" + "10.131.233.27", + "10.27.88.95" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -335,8 +335,8 @@ "olo7148.mail.home" ], "related.ip": [ - "10.87.212.179", - "10.157.213.15" + "10.157.213.15", + "10.87.212.179" ], "related.user": [ "rveli" @@ -346,8 +346,8 @@ "rsa.investigations.event_vcat": "aveniam", "rsa.misc.OS": "oll", "rsa.misc.action": [ - "ali", - "allow" + "allow", + "ali" ], "rsa.misc.category": "emeumfug", "rsa.misc.client": "caecatc", @@ -431,8 +431,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.152", "related.hosts": [ - "orain", "onse", + "orain", "agna7678.internal.host" ], "related.ip": [ @@ -447,8 +447,8 @@ "rsa.investigations.event_vcat": "mwr", "rsa.misc.OS": "imaven", "rsa.misc.action": [ - "uines", - "accept" + "accept", + "uines" ], "rsa.misc.category": "uidolo", "rsa.misc.client": "emips", @@ -532,8 +532,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4059", "related.hosts": [ - "utla", "tatn", + "utla", "equep5085.mail.domain" ], "related.ip": [ @@ -548,8 +548,8 @@ "rsa.investigations.event_vcat": "quae", "rsa.misc.OS": "qui", "rsa.misc.action": [ - "accept", - "iadese" + "iadese", + "accept" ], "rsa.misc.category": "aturve", "rsa.misc.client": "utei", @@ -634,9 +634,9 @@ "labore" ], "related.ip": [ - "10.186.85.3", + "10.114.16.155", "10.176.216.90", - "10.114.16.155" + "10.186.85.3" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -700,8 +700,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3917", "related.hosts": [ - "gnido", "sperna", + "gnido", "eturadi6608.mail.host" ], "related.ip": [ @@ -716,8 +716,8 @@ "rsa.investigations.event_vcat": "oide", "rsa.misc.OS": "gel", "rsa.misc.action": [ - "cancel", - "luptatem" + "luptatem", + "cancel" ], "rsa.misc.category": "uir", "rsa.misc.client": "ratvolu", @@ -806,8 +806,8 @@ "ipsumdol4488.api.localdomain" ], "related.ip": [ - "10.106.31.86", - "10.28.76.42" + "10.28.76.42", + "10.106.31.86" ], "related.user": [ "cons" @@ -817,8 +817,8 @@ "rsa.investigations.event_vcat": "tDuisaut", "rsa.misc.OS": "Nequepor", "rsa.misc.action": [ - "sno", - "deny" + "deny", + "sno" ], "rsa.misc.category": "idolo", "rsa.misc.client": "volu", @@ -904,8 +904,8 @@ ], "related.ip": [ "10.58.214.16", - "10.106.162.153", - "10.238.164.74" + "10.238.164.74", + "10.106.162.153" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -970,9 +970,9 @@ "tenimad" ], "related.ip": [ + "10.110.31.190", "10.225.141.20", - "10.217.150.196", - "10.110.31.190" + "10.217.150.196" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1036,13 +1036,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3319", "related.hosts": [ - "amc", "mestq", + "amc", "cusant4946.www.domain" ], "related.ip": [ - "10.137.56.173", - "10.69.103.176" + "10.69.103.176", + "10.137.56.173" ], "related.user": [ "proide" @@ -1138,9 +1138,9 @@ "onsecte" ], "related.ip": [ + "10.30.47.165", "10.5.235.217", - "10.25.212.118", - "10.30.47.165" + "10.25.212.118" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1306,9 +1306,9 @@ "xea" ], "related.ip": [ - "10.51.213.42", + "10.233.120.207", "10.98.194.212", - "10.233.120.207" + "10.51.213.42" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1374,8 +1374,8 @@ ], "related.ip": [ "10.241.132.176", - "10.67.132.242", - "10.245.187.229" + "10.245.187.229", + "10.67.132.242" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1455,8 +1455,8 @@ "rsa.investigations.event_vcat": "mes", "rsa.misc.OS": "riss", "rsa.misc.action": [ - "oin", - "allow" + "allow", + "oin" ], "rsa.misc.category": "idi", "rsa.misc.client": "modocon", @@ -1540,8 +1540,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.760", "related.hosts": [ - "dolorsit", "rcit", + "dolorsit", "mve1890.internal.home" ], "related.ip": [ @@ -1556,8 +1556,8 @@ "rsa.investigations.event_vcat": "metcons", "rsa.misc.OS": "ehende", "rsa.misc.action": [ - "umf", - "deny" + "deny", + "umf" ], "rsa.misc.category": "emUte", "rsa.misc.client": "archite", @@ -1641,8 +1641,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4450", "related.hosts": [ - "saquaea", "billoi", + "saquaea", "eturad6143.www.home" ], "related.ip": [ @@ -1657,8 +1657,8 @@ "rsa.investigations.event_vcat": "boNem", "rsa.misc.OS": "ntium", "rsa.misc.action": [ - "block", - "acommodi" + "acommodi", + "block" ], "rsa.misc.category": "inrepreh", "rsa.misc.client": "moles", @@ -1747,8 +1747,8 @@ "orinrep5386.www.corp" ], "related.ip": [ - "10.253.228.140", - "10.208.21.135" + "10.208.21.135", + "10.253.228.140" ], "related.user": [ "inculp" @@ -1758,8 +1758,8 @@ "rsa.investigations.event_vcat": "emagn", "rsa.misc.OS": "oditempo", "rsa.misc.action": [ - "ugitse", - "cancel" + "cancel", + "ugitse" ], "rsa.misc.category": "magnid", "rsa.misc.client": "sci", @@ -1843,13 +1843,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.1710", "related.hosts": [ - "Nemo", "edquia", + "Nemo", "henderi724.www5.home" ], "related.ip": [ - "10.243.226.122", - "10.3.23.172" + "10.3.23.172", + "10.243.226.122" ], "related.user": [ "olorem" @@ -1859,8 +1859,8 @@ "rsa.investigations.event_vcat": "ess", "rsa.misc.OS": "equatDu", "rsa.misc.action": [ - "emullamc", - "cancel" + "cancel", + "emullamc" ], "rsa.misc.category": "niamquis", "rsa.misc.client": "tutlabo", @@ -1949,8 +1949,8 @@ "reseosqu1629.mail.lan" ], "related.ip": [ - "10.106.85.174", - "10.94.242.80" + "10.94.242.80", + "10.106.85.174" ], "related.user": [ "lmo" @@ -2047,9 +2047,9 @@ "oluptat" ], "related.ip": [ - "10.117.63.181", "10.168.20.20", - "10.247.53.179" + "10.247.53.179", + "10.117.63.181" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2114,13 +2114,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.2208", "related.hosts": [ - "lamcola", "duntut", + "lamcola", "tasnul4179.internal.host" ], "related.ip": [ - "10.53.168.187", - "10.141.156.217" + "10.141.156.217", + "10.53.168.187" ], "related.user": [ "amqu" @@ -2130,8 +2130,8 @@ "rsa.investigations.event_vcat": "illumq", "rsa.misc.OS": "idata", "rsa.misc.action": [ - "emacc", - "block" + "block", + "emacc" ], "rsa.misc.category": "ueporro", "rsa.misc.client": "veli", @@ -2216,13 +2216,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3402", "related.hosts": [ - "tur", "imavenia", + "tur", "bore5546.www.local" ], "related.ip": [ - "10.44.198.184", - "10.189.82.19" + "10.189.82.19", + "10.44.198.184" ], "related.user": [ "iconsequ" @@ -2232,8 +2232,8 @@ "rsa.investigations.event_vcat": "eturadip", "rsa.misc.OS": "turadip", "rsa.misc.action": [ - "odoc", - "accept" + "accept", + "odoc" ], "rsa.misc.category": "volup", "rsa.misc.client": "tur", @@ -2333,8 +2333,8 @@ "rsa.investigations.event_vcat": "eturadip", "rsa.misc.OS": "onsecte", "rsa.misc.action": [ - "cancel", - "amni" + "amni", + "cancel" ], "rsa.misc.category": "umdolore", "rsa.misc.client": "modoc", @@ -2418,8 +2418,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.7278", "related.hosts": [ - "liquaUte", "ectetura", + "liquaUte", "uido2046.mail.lan" ], "related.ip": [ @@ -2434,8 +2434,8 @@ "rsa.investigations.event_vcat": "uatu", "rsa.misc.OS": "tnulapar", "rsa.misc.action": [ - "odic", - "deny" + "deny", + "odic" ], "rsa.misc.category": "deri", "rsa.misc.client": "scivelit", @@ -2520,9 +2520,9 @@ "uio" ], "related.ip": [ - "10.37.161.101", "10.17.209.252", - "10.111.182.212" + "10.111.182.212", + "10.37.161.101" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2587,9 +2587,9 @@ "itautfu" ], "related.ip": [ - "10.158.175.98", "10.170.196.181", - "10.153.166.133" + "10.153.166.133", + "10.158.175.98" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2669,8 +2669,8 @@ "rsa.investigations.event_vcat": "enimad", "rsa.misc.OS": "olor", "rsa.misc.action": [ - "accept", - "nse" + "nse", + "accept" ], "rsa.misc.category": "conseq", "rsa.misc.client": "mmo", @@ -2756,8 +2756,8 @@ ], "related.ip": [ "10.38.168.190", - "10.174.17.46", - "10.77.105.81" + "10.77.105.81", + "10.174.17.46" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2822,8 +2822,8 @@ "ons" ], "related.ip": [ - "10.166.142.198", "10.225.37.73", + "10.166.142.198", "10.36.99.207" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -3038,8 +3038,8 @@ "rsa.investigations.event_vcat": "olore", "rsa.misc.OS": "tatem", "rsa.misc.action": [ - "allow", - "itanimi" + "itanimi", + "allow" ], "rsa.misc.category": "psa", "rsa.misc.client": "ugits", @@ -3128,8 +3128,8 @@ "itaspe3216.localdomain" ], "related.ip": [ - "10.94.177.125", - "10.116.82.108" + "10.116.82.108", + "10.94.177.125" ], "related.user": [ "ecatc" @@ -3225,13 +3225,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.4442", "related.hosts": [ - "uae", "fugi", + "uae", "mea6298.api.example" ], "related.ip": [ - "10.115.121.243", - "10.113.152.241" + "10.113.152.241", + "10.115.121.243" ], "related.user": [ "norumetM" @@ -3326,8 +3326,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3804", "related.hosts": [ - "atcupi", "nder", + "atcupi", "iqu7510.internal.corp" ], "related.ip": [ @@ -3428,9 +3428,9 @@ "lors" ], "related.ip": [ - "10.205.83.138", "10.99.55.115", - "10.98.52.184" + "10.98.52.184", + "10.205.83.138" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3495,9 +3495,9 @@ "reprehe" ], "related.ip": [ + "10.197.128.162", "10.90.189.248", - "10.228.11.50", - "10.197.128.162" + "10.228.11.50" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3613,8 +3613,8 @@ "deFinibu3940.internal.lan" ], "related.ip": [ - "10.124.71.88", - "10.22.248.52" + "10.22.248.52", + "10.124.71.88" ], "related.user": [ "tcons" @@ -3624,8 +3624,8 @@ "rsa.investigations.event_vcat": "uiratio", "rsa.misc.OS": "xce", "rsa.misc.action": [ - "cons", - "cancel" + "cancel", + "cons" ], "rsa.misc.category": "ciun", "rsa.misc.client": "amquisn", @@ -3709,13 +3709,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.4895", "related.hosts": [ - "ficiade", "ipexeac", + "ficiade", "tatiset4191.localdomain" ], "related.ip": [ - "10.185.37.176", - "10.26.58.20" + "10.26.58.20", + "10.185.37.176" ], "related.user": [ "eumiure" @@ -3811,9 +3811,9 @@ "teni" ], "related.ip": [ + "10.200.12.126", "10.14.145.107", - "10.250.231.196", - "10.200.12.126" + "10.250.231.196" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3878,8 +3878,8 @@ "atuse" ], "related.ip": [ - "10.225.34.176", "10.103.36.192", + "10.225.34.176", "10.21.203.112" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -3945,9 +3945,9 @@ "usantiu" ], "related.ip": [ + "10.118.111.183", "10.140.59.161", - "10.5.67.140", - "10.118.111.183" + "10.5.67.140" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4011,8 +4011,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4493", "related.hosts": [ - "veleumiu", "labor", + "veleumiu", "nimadmi4084.api.home" ], "related.ip": [ @@ -4112,8 +4112,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.6506", "related.hosts": [ - "sedqui", "ecillum", + "sedqui", "reprehe3525.www5.example" ], "related.ip": [ @@ -4128,8 +4128,8 @@ "rsa.investigations.event_vcat": "uep", "rsa.misc.OS": "iatisund", "rsa.misc.action": [ - "nvo", - "block" + "block", + "nvo" ], "rsa.misc.category": "tenima", "rsa.misc.client": "iuntNe", @@ -4214,8 +4214,8 @@ "sitv" ], "related.ip": [ - "10.22.149.132", "10.217.145.137", + "10.22.149.132", "10.251.183.113" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4281,8 +4281,8 @@ "nisi" ], "related.ip": [ - "10.183.16.252", "10.203.66.175", + "10.183.16.252", "10.51.60.203" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4347,8 +4347,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.409", "related.hosts": [ - "magnama", "doei", + "magnama", "ursint411.www.lan" ], "related.ip": [ @@ -4363,8 +4363,8 @@ "rsa.investigations.event_vcat": "nisiuta", "rsa.misc.OS": "olu", "rsa.misc.action": [ - "tquov", - "accept" + "accept", + "tquov" ], "rsa.misc.category": "quatD", "rsa.misc.client": "acomm", @@ -4565,8 +4565,8 @@ "rsa.investigations.event_vcat": "ora", "rsa.misc.OS": "ommod", "rsa.misc.action": [ - "cancel", - "ant" + "ant", + "cancel" ], "rsa.misc.category": "rehende", "rsa.misc.client": "rehe", @@ -4666,8 +4666,8 @@ "rsa.investigations.event_vcat": "uia", "rsa.misc.OS": "mquae", "rsa.misc.action": [ - "deny", - "tenatus" + "tenatus", + "deny" ], "rsa.misc.category": "abo", "rsa.misc.client": "umtota", @@ -4753,8 +4753,8 @@ "nonnumq" ], "related.ip": [ - "10.34.41.75", "10.249.16.201", + "10.34.41.75", "10.107.168.208" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4925,8 +4925,8 @@ "ulamc767.internal.lan" ], "related.ip": [ - "10.112.155.228", - "10.47.191.95" + "10.47.191.95", + "10.112.155.228" ], "related.user": [ "aed" @@ -5022,9 +5022,9 @@ "metco" ], "related.ip": [ - "10.62.241.218", "10.103.169.94", - "10.140.137.17" + "10.140.137.17", + "10.62.241.218" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5089,8 +5089,8 @@ "litsedq" ], "related.ip": [ - "10.77.105.160", "10.251.212.166", + "10.77.105.160", "10.90.229.92" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5155,13 +5155,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.4261", "related.hosts": [ - "dutp", "ipsum", + "dutp", "spici5547.internal.test" ], "related.ip": [ - "10.216.49.112", - "10.112.242.68" + "10.112.242.68", + "10.216.49.112" ], "related.user": [ "urmag" @@ -5171,8 +5171,8 @@ "rsa.investigations.event_vcat": "animi", "rsa.misc.OS": "tisunde", "rsa.misc.action": [ - "aut", - "cancel" + "cancel", + "aut" ], "rsa.misc.category": "lamcorpo", "rsa.misc.client": "com", @@ -5272,8 +5272,8 @@ "rsa.investigations.event_vcat": "uatDuisa", "rsa.misc.OS": "citation", "rsa.misc.action": [ - "utlabore", - "accept" + "accept", + "utlabore" ], "rsa.misc.category": "reeu", "rsa.misc.client": "ntut", @@ -5358,9 +5358,9 @@ "cons" ], "related.ip": [ - "10.157.22.21", "10.228.61.5", - "10.246.41.77" + "10.246.41.77", + "10.157.22.21" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5425,9 +5425,9 @@ "llumdo" ], "related.ip": [ - "10.239.231.168", "10.188.131.18", - "10.242.119.111" + "10.242.119.111", + "10.239.231.168" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5491,13 +5491,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.979", "related.hosts": [ - "lorem", "iden", + "lorem", "tru3812.mail.lan" ], "related.ip": [ - "10.247.124.74", - "10.106.101.87" + "10.106.101.87", + "10.247.124.74" ], "related.user": [ "ainci" @@ -5639,8 +5639,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4342", "related.hosts": [ - "ten", "onsequ", + "ten", "riaturE1644.www5.example" ], "related.ip": [ @@ -5655,8 +5655,8 @@ "rsa.investigations.event_vcat": "empori", "rsa.misc.OS": "ostru", "rsa.misc.action": [ - "allow", - "quepor" + "quepor", + "allow" ], "rsa.misc.category": "cipitla", "rsa.misc.client": "exeacomm", @@ -5745,8 +5745,8 @@ "mdolo7008.api.corp" ], "related.ip": [ - "10.78.75.82", - "10.162.128.87" + "10.162.128.87", + "10.78.75.82" ], "related.user": [ "Sedutp" @@ -5842,9 +5842,9 @@ "reseosqu" ], "related.ip": [ + "10.137.36.151", "10.51.106.43", - "10.75.198.93", - "10.137.36.151" + "10.75.198.93" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5909,9 +5909,9 @@ "caecatcu" ], "related.ip": [ - "10.7.230.206", "10.249.93.150", - "10.154.151.111" + "10.154.151.111", + "10.7.230.206" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5980,8 +5980,8 @@ "itse5466.api.example" ], "related.ip": [ - "10.26.4.3", - "10.217.209.221" + "10.217.209.221", + "10.26.4.3" ], "related.user": [ "ciduntut" @@ -6177,8 +6177,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2052", "related.hosts": [ - "dat", "asp", + "dat", "amco1592.mail.host" ], "related.ip": [ @@ -6193,8 +6193,8 @@ "rsa.investigations.event_vcat": "atvolupt", "rsa.misc.OS": "riosam", "rsa.misc.action": [ - "ssitasp", - "deny" + "deny", + "ssitasp" ], "rsa.misc.category": "enimadmi", "rsa.misc.client": "uatDui", @@ -6278,13 +6278,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.2691", "related.hosts": [ - "orroqu", "ratio", + "orroqu", "dicta7226.mail.example" ], "related.ip": [ - "10.4.244.115", - "10.53.50.77" + "10.53.50.77", + "10.4.244.115" ], "related.user": [ "idolo" @@ -6294,8 +6294,8 @@ "rsa.investigations.event_vcat": "cupidata", "rsa.misc.OS": "ficiade", "rsa.misc.action": [ - "accept", - "lorem" + "lorem", + "accept" ], "rsa.misc.category": "iac", "rsa.misc.client": "tlabo", @@ -6381,8 +6381,8 @@ ], "related.ip": [ "10.221.100.157", - "10.236.211.111", - "10.120.212.78" + "10.120.212.78", + "10.236.211.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6446,8 +6446,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3052", "related.hosts": [ - "xeacom", "tenima", + "xeacom", "pidatatn2627.www.localdomain" ], "related.ip": [ @@ -6548,9 +6548,9 @@ "nimides" ], "related.ip": [ - "10.53.251.202", + "10.226.255.3", "10.123.59.69", - "10.226.255.3" + "10.53.251.202" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6615,9 +6615,9 @@ "edut" ], "related.ip": [ - "10.3.85.176", "10.212.56.26", - "10.29.141.252" + "10.29.141.252", + "10.3.85.176" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6681,8 +6681,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.95", "related.hosts": [ - "Utenimad", "inculp", + "Utenimad", "emveleu4029.api.local" ], "related.ip": [ @@ -6783,8 +6783,8 @@ "mes" ], "related.ip": [ - "10.83.98.220", "10.11.150.136", + "10.83.98.220", "10.171.60.173" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -6850,9 +6850,9 @@ "datatno" ], "related.ip": [ - "10.238.49.73", "10.74.88.209", - "10.92.3.166" + "10.92.3.166", + "10.238.49.73" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6917,9 +6917,9 @@ "ptate" ], "related.ip": [ + "10.187.107.47", "10.119.248.36", - "10.84.200.121", - "10.187.107.47" + "10.84.200.121" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7055,8 +7055,8 @@ "rspic5637.api.local" ], "related.ip": [ - "10.169.133.219", - "10.115.166.48" + "10.115.166.48", + "10.169.133.219" ], "related.user": [ "emq" @@ -7066,8 +7066,8 @@ "rsa.investigations.event_vcat": "iumdol", "rsa.misc.OS": "min", "rsa.misc.action": [ - "block", - "eleumiur" + "eleumiur", + "block" ], "rsa.misc.category": "ero", "rsa.misc.client": "gia", @@ -7320,8 +7320,8 @@ "oeius" ], "related.ip": [ - "10.186.253.240", "10.233.128.7", + "10.186.253.240", "10.66.149.234" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7454,9 +7454,9 @@ "emp" ], "related.ip": [ - "10.170.236.123", + "10.205.18.11", "10.69.130.207", - "10.205.18.11" + "10.170.236.123" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7525,8 +7525,8 @@ "velill3821.mail.invalid" ], "related.ip": [ - "10.97.254.192", - "10.124.34.251" + "10.124.34.251", + "10.97.254.192" ], "related.user": [ "epor" @@ -7536,8 +7536,8 @@ "rsa.investigations.event_vcat": "lica", "rsa.misc.OS": "taedi", "rsa.misc.action": [ - "imide", - "deny" + "deny", + "imide" ], "rsa.misc.category": "iurere", "rsa.misc.client": "ollitan", @@ -7622,8 +7622,8 @@ "ineavol" ], "related.ip": [ - "10.204.98.238", "10.9.41.221", + "10.204.98.238", "10.81.58.91" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7737,8 +7737,8 @@ ], "related.ip": [ "10.35.84.125", - "10.37.120.29", - "10.212.208.70" + "10.212.208.70", + "10.37.120.29" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7803,9 +7803,9 @@ "aturvel" ], "related.ip": [ + "10.199.201.26", "10.143.65.84", - "10.207.207.106", - "10.199.201.26" + "10.207.207.106" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7870,9 +7870,9 @@ "gni" ], "related.ip": [ - "10.204.27.48", "10.41.61.88", - "10.163.236.253" + "10.163.236.253", + "10.204.27.48" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7937,8 +7937,8 @@ "dents" ], "related.ip": [ - "10.185.44.26", "10.246.81.164", + "10.185.44.26", "10.53.110.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -8004,8 +8004,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.802", "related.hosts": [ - "lam", "proid", + "lam", "cupida6106.www5.local" ], "related.ip": [ @@ -8105,8 +8105,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2314", "related.hosts": [ - "stenat", "umtotam", + "stenat", "unt2122.internal.local" ], "related.ip": [ @@ -8121,8 +8121,8 @@ "rsa.investigations.event_vcat": "rure", "rsa.misc.OS": "iquidexe", "rsa.misc.action": [ - "allow", - "volu" + "volu", + "allow" ], "rsa.misc.category": "ium", "rsa.misc.client": "liquip", @@ -8211,8 +8211,8 @@ "luptat2613.internal.localhost" ], "related.ip": [ - "10.182.124.88", - "10.139.144.75" + "10.139.144.75", + "10.182.124.88" ], "related.user": [ "modo" @@ -8222,8 +8222,8 @@ "rsa.investigations.event_vcat": "tfug", "rsa.misc.OS": "imipsam", "rsa.misc.action": [ - "block", - "utodi" + "utodi", + "block" ], "rsa.misc.category": "cid", "rsa.misc.client": "mquaerat", @@ -8307,13 +8307,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.1386", "related.hosts": [ - "cab", "amquisn", + "cab", "neavo4796.internal.domain" ], "related.ip": [ - "10.188.124.185", - "10.35.10.19" + "10.35.10.19", + "10.188.124.185" ], "related.user": [ "dolo" @@ -8380,4 +8380,4 @@ "url.scheme": "https", "user.name": "dolo" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json index 17a2a41cc6df..271e0db04a32 100644 --- a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json +++ b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json @@ -23,12 +23,12 @@ "radipis5408.mail.local" ], "related.ip": [ - "10.70.155.35", - "10.81.122.126" + "10.81.122.126", + "10.70.155.35" ], "related.user": [ - "aqui", "tatno", + "aqui", "magn" ], "rsa.counters.dclass_c1": 5910, @@ -170,13 +170,13 @@ "elaudant5931.internal.invalid" ], "related.ip": [ - "10.232.27.250", - "10.18.124.28" + "10.18.124.28", + "10.232.27.250" ], "related.user": [ - "modocons", + "lapariat", "mquidol", - "lapariat" + "modocons" ], "rsa.counters.dclass_c1": 6564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -234,12 +234,12 @@ "amest4147.mail.host" ], "related.ip": [ - "10.197.250.10", - "10.6.137.200" + "10.6.137.200", + "10.197.250.10" ], "related.user": [ - "occae", "oluptas", + "occae", "intoc" ], "rsa.counters.event_counter": 7243, @@ -311,21 +311,21 @@ "eratv6205.internal.lan" ], "related.ip": [ - "10.179.124.125", - "10.36.194.106" + "10.36.194.106", + "10.179.124.125" ], "related.user": [ - "reme", + "ncidid", "acommod", - "ncidid" + "reme" ], "rsa.counters.event_counter": 2462, "rsa.db.database": "uaUteni", "rsa.internal.event_desc": "osqui", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "accept", - "lamcolab" + "lamcolab", + "accept" ], "rsa.misc.category": "xerc", "rsa.misc.disposition": "iutali", @@ -386,13 +386,13 @@ "didunt1355.corp" ], "related.ip": [ - "10.129.149.43", - "10.211.105.204" + "10.211.105.204", + "10.129.149.43" ], "related.user": [ + "labor", "orema", - "eveli", - "labor" + "eveli" ], "rsa.counters.dclass_c1": 6855, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -448,13 +448,13 @@ "pora6854.www5.home" ], "related.ip": [ - "10.214.191.180", - "10.112.250.193" + "10.112.250.193", + "10.214.191.180" ], "related.user": [ "ide", - "ipsumdol", - "Exc" + "Exc", + "ipsumdol" ], "rsa.counters.dclass_c1": 6852, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -509,13 +509,13 @@ "ptasn6599.www.localhost" ], "related.ip": [ - "10.251.20.13", - "10.192.34.76" + "10.192.34.76", + "10.251.20.13" ], "related.user": [ + "ovol", "tnonpro", - "iquipe", - "ovol" + "iquipe" ], "rsa.counters.dclass_c1": 3645, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -567,13 +567,13 @@ "ptasnu6684.mail.lan" ], "related.ip": [ - "10.59.138.212", - "10.74.105.218" + "10.74.105.218", + "10.59.138.212" ], "related.user": [ "archite", - "idunt", - "boree" + "boree", + "idunt" ], "rsa.counters.dclass_c1": 248, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -633,9 +633,9 @@ "10.230.173.4" ], "related.user": [ + "inci", "atemq", - "isnostr", - "inci" + "isnostr" ], "rsa.counters.dclass_c1": 6135, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -691,13 +691,13 @@ "atevelit2450.local" ], "related.ip": [ - "10.41.21.204", - "10.49.167.57" + "10.49.167.57", + "10.41.21.204" ], "related.user": [ - "ccaeca", "tali", - "sau" + "sau", + "ccaeca" ], "rsa.counters.dclass_c1": 6818, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -755,8 +755,8 @@ "itla658.api.localhost" ], "related.ip": [ - "10.62.147.186", - "10.216.125.252" + "10.216.125.252", + "10.62.147.186" ], "related.user": [ "dolore", @@ -768,8 +768,8 @@ "rsa.internal.event_desc": "aquae", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "accept", - "quasia" + "quasia", + "accept" ], "rsa.misc.category": "boreetdo", "rsa.misc.disposition": "aturve", @@ -833,13 +833,13 @@ "umdolor4389.api.home" ], "related.ip": [ - "10.204.128.215", - "10.52.125.9" + "10.52.125.9", + "10.204.128.215" ], "related.user": [ - "nci", "paquioff", - "rum" + "rum", + "nci" ], "rsa.counters.event_counter": 332, "rsa.db.database": "isau", @@ -907,8 +907,8 @@ "rationev6444.localhost" ], "related.ip": [ - "10.34.148.166", - "10.200.68.129" + "10.200.68.129", + "10.34.148.166" ], "related.user": [ "untutlab", @@ -969,9 +969,9 @@ "10.134.5.40" ], "related.user": [ - "conse", "siu", - "licabo" + "licabo", + "conse" ], "rsa.counters.dclass_c1": 6356, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1089,13 +1089,13 @@ "nimid372.api.corp" ], "related.ip": [ - "10.190.10.219", - "10.233.120.207" + "10.233.120.207", + "10.190.10.219" ], "related.user": [ "item", - "quamnih", - "accusant" + "accusant", + "quamnih" ], "rsa.counters.dclass_c1": 3278, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1179,12 +1179,12 @@ "maliquam2147.internal.home" ], "related.ip": [ - "10.100.98.56", - "10.248.184.200" + "10.248.184.200", + "10.100.98.56" ], "related.user": [ - "proident", "boru", + "ritati", "proident" ], "rsa.counters.dclass_c1": 5923, @@ -1245,8 +1245,8 @@ "10.82.28.220" ], "related.user": [ - "oluptat", "aecatcup", + "oluptat", "dtempo" ], "rsa.counters.dclass_c1": 3071, @@ -1308,8 +1308,8 @@ ], "related.user": [ "ationul", - "redol", - "asnu" + "asnu", + "redol" ], "rsa.counters.dclass_c1": 6606, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1371,17 +1371,17 @@ "10.81.184.7" ], "related.user": [ - "iameaque", + "lmole", "undeomni", - "lmole" + "iameaque" ], "rsa.counters.event_counter": 6344, "rsa.db.database": "nderi", "rsa.internal.event_desc": "iae", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "illu", - "deny" + "deny", + "illu" ], "rsa.misc.category": "quido", "rsa.misc.disposition": "emip", @@ -1443,13 +1443,13 @@ "dolorem6882.api.local" ], "related.ip": [ - "10.29.119.245", - "10.214.3.140" + "10.214.3.140", + "10.29.119.245" ], "related.user": [ - "taliqui", + "scipitl", "edolorin", - "scipitl" + "taliqui" ], "rsa.counters.dclass_c1": 5140, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1507,8 +1507,8 @@ "temaccu5302.test" ], "related.ip": [ - "10.218.123.234", - "10.110.133.7" + "10.110.133.7", + "10.218.123.234" ], "related.user": [ "caboNem", @@ -1520,8 +1520,8 @@ "rsa.internal.event_desc": "liquid", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "vitaed", - "allow" + "allow", + "vitaed" ], "rsa.misc.category": "enim", "rsa.misc.disposition": "Finibus", @@ -1587,9 +1587,9 @@ "10.182.152.242" ], "related.user": [ - "mquisn", "doeiu", - "litan" + "litan", + "mquisn" ], "rsa.counters.dclass_c1": 3474, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1726,9 +1726,9 @@ "10.72.75.207" ], "related.user": [ - "eufug", "eFini", - "urau" + "urau", + "eufug" ], "rsa.counters.dclass_c1": 3348, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1788,9 +1788,9 @@ "10.58.133.175" ], "related.user": [ - "mfu", "oco", - "nde" + "nde", + "mfu" ], "rsa.counters.dclass_c1": 3795, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1850,9 +1850,9 @@ "10.70.29.203" ], "related.user": [ - "pta", "mquisnos", - "veniamq" + "veniamq", + "pta" ], "rsa.counters.dclass_c1": 2358, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1912,9 +1912,9 @@ "10.137.85.123" ], "related.user": [ - "ames", + "Bonorum", "sis", - "Bonorum" + "ames" ], "rsa.counters.dclass_c1": 6401, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2000,12 +2000,12 @@ "upt6017.api.localdomain" ], "related.ip": [ - "10.173.178.109", - "10.64.184.196" + "10.64.184.196", + "10.173.178.109" ], "related.user": [ - "uian", "nesci", + "uian", "tam" ], "rsa.counters.event_counter": 4493, @@ -2013,8 +2013,8 @@ "rsa.internal.event_desc": "orin", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "lamco", - "block" + "block", + "lamco" ], "rsa.misc.category": "enia", "rsa.misc.disposition": "iavol", @@ -2080,8 +2080,8 @@ "10.90.50.149" ], "related.user": [ - "olu", "olupta", + "olu", "aUtenima" ], "rsa.counters.dclass_c1": 1127, @@ -2142,9 +2142,9 @@ "10.18.150.82" ], "related.user": [ - "luptat", + "mtota", "qua", - "mtota" + "luptat" ], "rsa.counters.dclass_c1": 6112, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2231,9 +2231,9 @@ "10.151.240.35" ], "related.user": [ - "ametcons", "lam", - "ama" + "ama", + "ametcons" ], "rsa.counters.dclass_c1": 4325, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2285,12 +2285,12 @@ "tium3542.internal.invalid" ], "related.ip": [ - "10.242.48.203", - "10.147.142.242" + "10.147.142.242", + "10.242.48.203" ], "related.user": [ - "quisn", "quasi", + "quisn", "ese" ], "rsa.counters.dclass_c1": 3970, @@ -2353,9 +2353,9 @@ "10.213.165.165" ], "related.user": [ - "ttenb", + "civeli", "eufugia", - "civeli" + "ttenb" ], "rsa.counters.event_counter": 7365, "rsa.db.database": "utlabore", @@ -2455,12 +2455,12 @@ "ihi7294.www5.localhost" ], "related.ip": [ - "10.169.28.157", - "10.116.1.130" + "10.116.1.130", + "10.169.28.157" ], "related.user": [ - "reseo", "eturadip", + "reseo", "amco" ], "rsa.counters.event_counter": 1295, @@ -2531,13 +2531,13 @@ "caecat4920.api.host" ], "related.ip": [ - "10.29.138.31", - "10.45.69.152" + "10.45.69.152", + "10.29.138.31" ], "related.user": [ - "volupta", "umq", - "tsunt" + "tsunt", + "volupta" ], "rsa.counters.dclass_c1": 744, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2593,13 +2593,13 @@ "setquas6188.internal.local" ], "related.ip": [ - "10.100.113.11", - "10.152.213.228" + "10.152.213.228", + "10.100.113.11" ], "related.user": [ + "itationu", "ptatev", - "velillum", - "itationu" + "velillum" ], "rsa.counters.dclass_c1": 7245, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2683,12 +2683,12 @@ "nibusBo3674.www5.localhost" ], "related.ip": [ - "10.248.102.129", - "10.208.33.55" + "10.208.33.55", + "10.248.102.129" ], "related.user": [ - "mremaper", "inimv", + "mremaper", "ulapari" ], "rsa.counters.dclass_c1": 6433, @@ -2807,13 +2807,13 @@ "idents7231.mail.home" ], "related.ip": [ - "10.117.81.75", - "10.151.203.60" + "10.151.203.60", + "10.117.81.75" ], "related.user": [ - "exeac", + "iconsequ", "dol", - "iconsequ" + "exeac" ], "rsa.counters.dclass_c1": 484, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2869,13 +2869,13 @@ "tat50.mail.host" ], "related.ip": [ - "10.45.152.205", - "10.224.217.153" + "10.224.217.153", + "10.45.152.205" ], "related.user": [ - "imav", "utlabo", - "eriti" + "eriti", + "imav" ], "rsa.counters.dclass_c1": 922, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2936,17 +2936,17 @@ "10.1.193.187" ], "related.user": [ + "hite", "adipis", - "ugi", - "hite" + "ugi" ], "rsa.counters.event_counter": 508, "rsa.db.database": "abo", "rsa.internal.event_desc": "epteurs", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "taevitae", - "allow" + "allow", + "taevitae" ], "rsa.misc.category": "itse", "rsa.misc.disposition": "rever", @@ -3012,8 +3012,8 @@ ], "related.user": [ "sum", - "mquamei", - "eiusm" + "eiusm", + "mquamei" ], "rsa.counters.dclass_c1": 3058, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3065,13 +3065,13 @@ "fde7756.mail.corp" ], "related.ip": [ - "10.86.121.152", - "10.122.127.237" + "10.122.127.237", + "10.86.121.152" ], "related.user": [ - "consecte", + "ine", "nimv", - "ine" + "consecte" ], "rsa.counters.dclass_c1": 2771, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3127,13 +3127,13 @@ "agnama5013.internal.example" ], "related.ip": [ - "10.201.223.119", - "10.204.223.184" + "10.204.223.184", + "10.201.223.119" ], "related.user": [ - "tuserror", "rcit", - "teni" + "teni", + "tuserror" ], "rsa.counters.dclass_c1": 4113, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3189,13 +3189,13 @@ "edictas4693.home" ], "related.ip": [ - "10.200.12.126", - "10.223.56.33" + "10.223.56.33", + "10.200.12.126" ], "related.user": [ "Nequepo", - "magnido", - "elitsedd" + "elitsedd", + "magnido" ], "rsa.counters.dclass_c1": 3243, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3257,9 +3257,9 @@ "10.94.89.177" ], "related.user": [ - "emquel", + "tuserror", "citation", - "tuserror" + "emquel" ], "rsa.counters.event_counter": 2513, "rsa.db.database": "rspiciat", @@ -3328,8 +3328,8 @@ "tsun7120.home" ], "related.ip": [ - "10.191.184.105", - "10.65.174.196" + "10.65.174.196", + "10.191.184.105" ], "related.user": [ "iin", @@ -3388,21 +3388,21 @@ "lumquid6940.mail.localdomain" ], "related.ip": [ - "10.41.181.179", - "10.224.148.48" + "10.224.148.48", + "10.41.181.179" ], "related.user": [ + "iosamn", "niam", - "equepor", - "iosamn" + "equepor" ], "rsa.counters.event_counter": 7468, "rsa.db.database": "erspicia", "rsa.internal.event_desc": "ibusB", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "deny", - "rumwr" + "rumwr", + "deny" ], "rsa.misc.category": "rporis", "rsa.misc.disposition": "etco", @@ -3464,13 +3464,13 @@ "amcorp7299.api.example" ], "related.ip": [ - "10.21.208.103", - "10.21.61.134" + "10.21.61.134", + "10.21.208.103" ], "related.user": [ - "imidest", + "ostr", "mipsa", - "ostr" + "imidest" ], "rsa.counters.dclass_c1": 7766, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3530,8 +3530,8 @@ "10.23.6.216" ], "related.user": [ - "iarchit", "tevelite", + "iarchit", "iamquisn" ], "rsa.counters.dclass_c1": 639, @@ -3590,21 +3590,21 @@ "tionevol3157.mail.invalid" ], "related.ip": [ - "10.240.62.238", - "10.191.142.143" + "10.191.142.143", + "10.240.62.238" ], "related.user": [ - "animide", "nofde", - "modtempo" + "modtempo", + "animide" ], "rsa.counters.event_counter": 7580, "rsa.db.database": "Lore", "rsa.internal.event_desc": "nto", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "cancel", - "ali" + "ali", + "cancel" ], "rsa.misc.category": "sciv", "rsa.misc.disposition": "tlabo", @@ -3667,21 +3667,21 @@ "mquis319.api.local" ], "related.ip": [ - "10.178.79.217", - "10.111.22.134" + "10.111.22.134", + "10.178.79.217" ], "related.user": [ "inibusBo", - "ccusan", - "tqui" + "tqui", + "ccusan" ], "rsa.counters.event_counter": 3538, "rsa.db.database": "sequun", "rsa.internal.event_desc": "adeseru", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "deny", - "orisnis" + "orisnis", + "deny" ], "rsa.misc.category": "sitas", "rsa.misc.disposition": "eni", @@ -3746,9 +3746,9 @@ "10.161.225.172" ], "related.user": [ - "xerc", "rcit", - "meaqu" + "meaqu", + "xerc" ], "rsa.counters.dclass_c1": 7286, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3803,13 +3803,13 @@ "enbyCic4659.www5.example" ], "related.ip": [ - "10.186.133.184", - "10.211.161.187" + "10.211.161.187", + "10.186.133.184" ], "related.user": [ "sci", - "boriosa", - "acons" + "acons", + "boriosa" ], "rsa.counters.dclass_c1": 1578, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3864,9 +3864,9 @@ "10.160.147.230" ], "related.user": [ + "nimvenia", "ndeomnis", - "illoin", - "nimvenia" + "illoin" ], "rsa.counters.dclass_c1": 5988, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3918,8 +3918,8 @@ "reseo2067.api.localdomain" ], "related.ip": [ - "10.182.197.243", - "10.40.24.93" + "10.40.24.93", + "10.182.197.243" ], "related.user": [ "orisnis", @@ -3985,8 +3985,8 @@ ], "related.user": [ "exeacomm", - "uisautei", - "colab" + "colab", + "uisautei" ], "rsa.counters.dclass_c1": 1044, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4044,21 +4044,21 @@ "caboNemo274.www.host" ], "related.ip": [ - "10.64.94.174", - "10.39.244.49" + "10.39.244.49", + "10.64.94.174" ], "related.user": [ - "estiae", + "iunt", "Sedut", - "iunt" + "estiae" ], "rsa.counters.event_counter": 7128, "rsa.db.database": "eFinibu", "rsa.internal.event_desc": "enimips", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "gna", - "cancel" + "cancel", + "gna" ], "rsa.misc.category": "Nequepor", "rsa.misc.disposition": "nisiu", @@ -4179,8 +4179,8 @@ "10.115.203.143" ], "related.user": [ - "utoditau", "involu", + "utoditau", "orpori" ], "rsa.counters.dclass_c1": 7868, @@ -4237,13 +4237,13 @@ "iamq2577.internal.corp" ], "related.ip": [ - "10.251.212.166", - "10.43.244.252" + "10.43.244.252", + "10.251.212.166" ], "related.user": [ + "gnido", "inculp", - "uptat", - "gnido" + "uptat" ], "rsa.counters.dclass_c1": 6947, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4331,9 +4331,9 @@ "10.20.231.188" ], "related.user": [ - "mqu", + "uatDuisa", "tesseq", - "uatDuisa" + "mqu" ], "rsa.counters.dclass_c1": 1623, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4417,8 +4417,8 @@ "abor3266.mail.home" ], "related.ip": [ - "10.225.11.197", - "10.231.77.26" + "10.231.77.26", + "10.225.11.197" ], "related.user": [ "rehe", @@ -4481,8 +4481,8 @@ "10.106.166.105" ], "related.user": [ - "avolup", "olupt", + "avolup", "usa" ], "rsa.counters.dclass_c1": 2658, @@ -4539,9 +4539,9 @@ "10.57.169.205" ], "related.user": [ - "ipsu", "ctas", - "iuta" + "iuta", + "ipsu" ], "rsa.counters.dclass_c1": 392, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4602,8 +4602,8 @@ ], "related.user": [ "tisundeo", - "tevelit", - "dquia" + "dquia", + "tevelit" ], "rsa.counters.dclass_c1": 6709, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4659,13 +4659,13 @@ "ididu5928.www5.local" ], "related.ip": [ - "10.111.132.221", - "10.76.121.224" + "10.76.121.224", + "10.111.132.221" ], "related.user": [ - "scive", + "ali", "oloremi", - "ali" + "scive" ], "rsa.counters.dclass_c1": 6155, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4783,12 +4783,12 @@ "ssusc1892.internal.host" ], "related.ip": [ - "10.179.60.167", - "10.173.13.179" + "10.173.13.179", + "10.179.60.167" ], "related.user": [ - "apar", "isn", + "apar", "ptasn" ], "rsa.counters.dclass_c1": 758, @@ -4845,13 +4845,13 @@ "iatisund424.mail.localdomain" ], "related.ip": [ - "10.42.135.34", - "10.178.190.123" + "10.178.190.123", + "10.42.135.34" ], "related.user": [ - "orsi", + "ore", "tiset", - "ore" + "orsi" ], "rsa.counters.dclass_c1": 2290, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4939,9 +4939,9 @@ "10.207.198.239" ], "related.user": [ - "incididu", "aUteni", - "Loremips" + "Loremips", + "incididu" ], "rsa.counters.dclass_c1": 3043, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4996,13 +4996,13 @@ "dmini3435.internal.domain" ], "related.ip": [ - "10.116.26.185", - "10.206.221.180" + "10.206.221.180", + "10.116.26.185" ], "related.user": [ + "nseq", "oNe", - "litesseq", - "nseq" + "litesseq" ], "rsa.counters.dclass_c1": 3218, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5054,13 +5054,13 @@ "nibusBo1864.domain" ], "related.ip": [ - "10.253.127.130", - "10.86.180.150" + "10.86.180.150", + "10.253.127.130" ], "related.user": [ - "itasper", "etconsec", - "mnisis" + "mnisis", + "itasper" ], "rsa.counters.dclass_c1": 4564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5224,9 +5224,9 @@ "10.248.16.82" ], "related.user": [ - "ditautf", + "tuserror", "res", - "tuserror" + "ditautf" ], "rsa.counters.dclass_c1": 4367, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5286,9 +5286,9 @@ "10.173.19.140" ], "related.user": [ - "orsi", + "Except", "olo", - "Except" + "orsi" ], "rsa.counters.dclass_c1": 5844, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5401,13 +5401,13 @@ "isetqu2843.www.invalid" ], "related.ip": [ - "10.253.175.129", - "10.213.214.118" + "10.213.214.118", + "10.253.175.129" ], "related.user": [ "epteurs", - "ate", - "nrep" + "nrep", + "ate" ], "rsa.counters.dclass_c1": 6260, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5470,8 +5470,8 @@ ], "related.user": [ "orumetMa", - "aboris", - "atus" + "atus", + "aboris" ], "rsa.counters.event_counter": 5863, "rsa.db.database": "inventor", @@ -5625,8 +5625,8 @@ ], "related.user": [ "erit", - "usmodte", - "untex" + "untex", + "usmodte" ], "rsa.counters.event_counter": 4029, "rsa.db.database": "ommodi", @@ -5696,8 +5696,8 @@ "isau4356.www.home" ], "related.ip": [ - "10.161.212.150", - "10.115.42.231" + "10.115.42.231", + "10.161.212.150" ], "related.user": [ "tasnul", @@ -5760,13 +5760,13 @@ "labo3477.www5.domain" ], "related.ip": [ - "10.247.108.144", - "10.226.75.20" + "10.226.75.20", + "10.247.108.144" ], "related.user": [ "maccusan", - "fugia", - "tema" + "tema", + "fugia" ], "rsa.counters.event_counter": 3711, "rsa.db.database": "psa", @@ -5839,9 +5839,9 @@ "10.192.15.65" ], "related.user": [ - "rExcep", + "illumd", "nimides", - "illumd" + "rExcep" ], "rsa.counters.dclass_c1": 4173, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5895,13 +5895,13 @@ "duntutl3396.api.host" ], "related.ip": [ - "10.116.76.161", - "10.197.254.133" + "10.197.254.133", + "10.116.76.161" ], "related.user": [ - "trudex", "ide", - "idu" + "idu", + "trudex" ], "rsa.counters.event_counter": 2608, "rsa.db.database": "ncul", @@ -6031,13 +6031,13 @@ "tsunti1164.www.example" ], "related.ip": [ - "10.18.15.43", - "10.248.177.182" + "10.248.177.182", + "10.18.15.43" ], "related.user": [ - "quaturve", + "quei", "caecat", - "quei" + "quaturve" ], "rsa.counters.dclass_c1": 983, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -6065,4 +6065,4 @@ ], "user.name": "quei" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json index ddd233180e09..66df7548ac2b 100644 --- a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json +++ b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json @@ -371,8 +371,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.155.162.162", - "10.66.171.247" + "10.66.171.247", + "10.155.162.162" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -456,8 +456,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.179.26.34", - "10.38.77.13" + "10.38.77.13", + "10.179.26.34" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1800,8 +1800,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.28.226.128", - "10.122.76.148" + "10.122.76.148", + "10.28.226.128" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1928,8 +1928,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.31.177.226", - "10.98.209.10" + "10.98.209.10", + "10.31.177.226" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1966,8 +1966,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.44.47.27", - "10.179.210.218" + "10.179.210.218", + "10.44.47.27" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -2253,8 +2253,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.151.129.181", - "10.55.156.64" + "10.55.156.64", + "10.151.129.181" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", diff --git a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json index b2abd9c2604b..bf35e8659161 100644 --- a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json @@ -59,9 +59,9 @@ "10.57.170.140" ], "related.user": [ - "dexeac", "icistatuscode=giatquov", - "sunt" + "sunt", + "dexeac" ], "rsa.db.index": "run", "rsa.identity.logon_type": "nofdeF", @@ -70,8 +70,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "block", - "ugiatnu" + "ugiatnu", + "block" ], "rsa.misc.comments": "colabo", "rsa.misc.content_type": "sedd", @@ -169,8 +169,8 @@ "10.106.239.55" ], "related.user": [ - "eaq", - "itquiin" + "itquiin", + "eaq" ], "rsa.identity.logon_type": "stquidol", "rsa.internal.event_desc": "bor", @@ -893,8 +893,8 @@ "elites4713.www.localhost" ], "related.ip": [ - "10.52.190.18", - "10.161.51.135" + "10.161.51.135", + "10.52.190.18" ], "rsa.internal.event_desc": "portscan", "rsa.internal.messageid": "ulogd", @@ -980,8 +980,8 @@ "10.232.108.32" ], "related.user": [ - "rsp", - "llum" + "llum", + "rsp" ], "rsa.identity.logon_type": "ntut", "rsa.internal.event_desc": "ittenb", @@ -1054,8 +1054,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "iuntN", - "deny" + "deny", + "iuntN" ], "rsa.misc.comments": "onorume", "rsa.misc.content_type": "lapa", @@ -1869,8 +1869,8 @@ "tenbyCi4371.www5.localdomain" ], "related.ip": [ - "10.214.167.164", - "10.98.126.206" + "10.98.126.206", + "10.214.167.164" ], "related.user": [ "amremapstatuscode=dolorsit", @@ -2053,8 +2053,8 @@ "iusmo901.www.home" ], "related.ip": [ - "10.2.24.156", - "10.92.93.236" + "10.92.93.236", + "10.2.24.156" ], "related.user": [ "ulpaq", @@ -2152,13 +2152,13 @@ "mni4032.lan" ], "related.ip": [ - "10.180.169.49", - "10.202.65.2" + "10.202.65.2", + "10.180.169.49" ], "related.user": [ + "tasu", "atatno", - "iscivelistatuscode=urve", - "tasu" + "iscivelistatuscode=urve" ], "rsa.db.index": "amrem", "rsa.identity.logon_type": "nulamcol", @@ -2357,8 +2357,8 @@ ], "related.user": [ "umqustatuscode=ntexpli", - "eturadip", - "porincid" + "porincid", + "eturadip" ], "rsa.db.index": "dolor", "rsa.identity.logon_type": "eturadi", @@ -2367,8 +2367,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "doc", - "cancel" + "cancel", + "doc" ], "rsa.misc.comments": "riosam", "rsa.misc.content_type": "iusmo", @@ -2525,8 +2525,8 @@ "10.32.85.21" ], "related.user": [ - "antium", - "etconsec" + "etconsec", + "antium" ], "rsa.identity.logon_type": "umiurere", "rsa.internal.event_desc": "serro", @@ -2676,10 +2676,10 @@ "10.210.175.52" ], "related.user": [ - "reetd", - "rExce", "Loremi", - "inimastatuscode=emipsum" + "inimastatuscode=emipsum", + "rExce", + "reetd" ], "rsa.db.index": "apa", "rsa.identity.logon_type": "sedquia", @@ -3515,8 +3515,8 @@ "imv1805.api.host" ], "related.ip": [ - "10.248.62.55", - "10.96.243.231" + "10.96.243.231", + "10.248.62.55" ], "rsa.internal.event_desc": "ICMP", "rsa.internal.messageid": "ulogd", @@ -3776,4 +3776,4 @@ "forwarded" ] } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index a0bf899ae2ca..f70c9ad8ae49 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -26,8 +26,8 @@ "login.yahoo.com" ], "related.ip": [ - "209.73.177.115", - "10.105.21.199" + "10.105.21.199", + "209.73.177.115" ], "related.user": [ "badeyek" @@ -37,8 +37,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "CONNECT" + "CONNECT", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -97,8 +97,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -109,8 +109,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -172,8 +172,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -184,8 +184,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -246,8 +246,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -371,8 +371,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -443,8 +443,8 @@ "www.google-analytics.com" ], "related.ip": [ - "66.102.9.147", - "10.105.21.199" + "10.105.21.199", + "66.102.9.147" ], "related.user": [ "badeyek" @@ -684,8 +684,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -746,8 +746,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -810,8 +810,8 @@ "as.casalemedia.com" ], "related.ip": [ - "10.105.21.199", - "209.85.16.38" + "209.85.16.38", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -880,8 +880,8 @@ "us.bc.yahoo.com" ], "related.ip": [ - "68.142.213.132", - "10.105.21.199" + "10.105.21.199", + "68.142.213.132" ], "related.user": [ "badeyek" @@ -949,8 +949,8 @@ "impgb.tradedoubler.com" ], "related.ip": [ - "10.105.21.199", - "217.212.240.172" + "217.212.240.172", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -961,8 +961,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "302", @@ -1024,8 +1024,8 @@ "4.adbrite.com" ], "related.ip": [ - "10.105.21.199", - "206.169.136.22" + "206.169.136.22", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -1036,8 +1036,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -1099,8 +1099,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -1239,8 +1239,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -1251,8 +1251,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -1310,8 +1310,8 @@ "4.adbrite.com" ], "related.ip": [ - "64.127.126.178", - "10.105.21.199" + "10.105.21.199", + "64.127.126.178" ], "related.user": [ "badeyek" @@ -1387,8 +1387,8 @@ "ff.connextra.com" ], "related.ip": [ - "10.105.21.199", - "213.160.98.161" + "213.160.98.161", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -1399,8 +1399,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "302", @@ -1593,8 +1593,8 @@ "login.yahoo.com" ], "related.ip": [ - "209.73.177.115", - "10.105.21.199" + "10.105.21.199", + "209.73.177.115" ], "related.user": [ "badeyek" @@ -1663,8 +1663,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -1723,8 +1723,8 @@ "shttp.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.155.194.239" + "216.155.194.239", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -1869,8 +1869,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -1939,8 +1939,8 @@ "shttp.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.155.194.239" + "216.155.194.239", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2135,8 +2135,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "304", @@ -2198,8 +2198,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -2210,8 +2210,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -2283,8 +2283,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -2421,8 +2421,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -2482,8 +2482,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2565,8 +2565,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "POST" + "POST", + "TCP_MISS" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -2623,8 +2623,8 @@ "address.yahoo.com" ], "related.ip": [ - "209.191.93.51", - "10.105.33.214" + "10.105.33.214", + "209.191.93.51" ], "related.user": [ "adeolaegbedokun" @@ -2783,8 +2783,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2913,8 +2913,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "CONNECT" + "CONNECT", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -2971,8 +2971,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_DENIED" + "TCP_DENIED", + "POST" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3032,8 +3032,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "POST" + "POST", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3092,8 +3092,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_DENIED" + "TCP_DENIED", + "CONNECT" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3162,8 +3162,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3222,8 +3222,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "CONNECT" + "CONNECT", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3280,8 +3280,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -3292,8 +3292,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3363,8 +3363,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -3435,8 +3435,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3496,8 +3496,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -3508,8 +3508,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3569,8 +3569,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "POST" + "POST", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3922,8 +3922,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3984,8 +3984,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_IMS_HIT" + "TCP_IMS_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -4108,8 +4108,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -4169,8 +4169,8 @@ "newsrss.bbc.co.uk" ], "related.ip": [ - "212.58.226.33", - "10.105.21.199" + "10.105.21.199", + "212.58.226.33" ], "related.user": [ "badeyek" @@ -4254,8 +4254,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -4316,8 +4316,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_IMS_HIT" + "TCP_IMS_HIT", + "GET" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "304", @@ -4378,8 +4378,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "200", @@ -4439,8 +4439,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4514,8 +4514,8 @@ "us.news1.yimg.com" ], "related.ip": [ - "213.160.98.159", - "10.105.33.214" + "10.105.33.214", + "213.160.98.159" ], "related.user": [ "adeolaegbedokun" @@ -4526,8 +4526,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "200", @@ -4587,8 +4587,8 @@ "radio.music.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4747,8 +4747,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -4810,8 +4810,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -4873,8 +4873,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -4938,8 +4938,8 @@ "us.news1.yimg.com" ], "related.ip": [ - "213.160.98.159", - "10.105.33.214" + "10.105.33.214", + "213.160.98.159" ], "related.user": [ "adeolaegbedokun" @@ -5011,8 +5011,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5023,8 +5023,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "302", @@ -5085,8 +5085,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -5162,8 +5162,8 @@ "us.a2.yimg.com" ], "related.ip": [ - "213.160.98.152", - "10.105.33.214" + "10.105.33.214", + "213.160.98.152" ], "related.user": [ "adeolaegbedokun" @@ -5174,8 +5174,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "application/x-shockwave-flash", "rsa.misc.result_code": "200", @@ -5236,8 +5236,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -5248,8 +5248,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -5320,8 +5320,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -5461,8 +5461,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "CONNECT" + "CONNECT", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -5520,8 +5520,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "304", @@ -5584,8 +5584,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5660,8 +5660,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "213.160.98.159", - "10.105.33.214" + "10.105.33.214", + "213.160.98.159" ], "related.user": [ "adeolaegbedokun" @@ -5672,8 +5672,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5732,8 +5732,8 @@ "login.yahoo.com" ], "related.ip": [ - "10.105.21.199", - "209.73.177.115" + "209.73.177.115", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -5743,8 +5743,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_MISS" + "TCP_MISS", + "CONNECT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -5804,8 +5804,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "213.160.98.167", - "10.105.33.214" + "10.105.33.214", + "213.160.98.167" ], "related.user": [ "adeolaegbedokun" @@ -5816,8 +5816,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -5880,8 +5880,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -6018,8 +6018,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.167" + "213.160.98.167", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -6170,8 +6170,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.167" + "213.160.98.167", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -6244,8 +6244,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "GET" + "GET", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -6377,8 +6377,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -6510,8 +6510,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "CONNECT" + "CONNECT", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -6571,8 +6571,8 @@ "us.js2.yimg.com" ], "related.ip": [ - "213.160.98.169", - "10.105.21.199" + "10.105.21.199", + "213.160.98.169" ], "related.user": [ "badeyek" @@ -6709,8 +6709,8 @@ "us.js2.yimg.com" ], "related.ip": [ - "213.160.98.169", - "10.105.21.199" + "10.105.21.199", + "213.160.98.169" ], "related.user": [ "badeyek" @@ -6845,8 +6845,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "200", diff --git a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json index 77f61afdee9b..070d7b72259a 100644 --- a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json @@ -18,20 +18,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "example.net" + "example.net", + "example.org" ], "related.ip": [ - "10.251.224.219", - "10.234.224.44" + "10.234.224.44", + "10.251.224.219" ], "related.user": [ "tation" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "deny" + "deny", + "PROPFIND" ], "rsa.misc.content_type": "ciade", "rsa.misc.result_code": "liqua", @@ -172,12 +172,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "example.net" + "example.net", + "internal.example.com" ], "related.ip": [ - "10.15.135.248", - "10.142.172.64" + "10.142.172.64", + "10.15.135.248" ], "related.user": [ "tia" @@ -253,16 +253,16 @@ "mail.example.com" ], "related.ip": [ - "10.81.122.126", - "10.44.134.153" + "10.44.134.153", + "10.81.122.126" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "GET" + "GET", + "allow" ], "rsa.misc.content_type": "taev", "rsa.misc.result_code": "quiavo", @@ -327,8 +327,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "api.example.org" + "api.example.org", + "api.example.net" ], "related.ip": [ "10.160.95.56", @@ -405,12 +405,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www.example.org" + "www.example.org", + "api.example.com" ], "related.ip": [ - "10.175.107.139", - "10.12.195.60" + "10.12.195.60", + "10.175.107.139" ], "related.user": [ "mrema" @@ -420,8 +420,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "HEAD", - "accept" + "accept", + "HEAD" ], "rsa.misc.content_type": "aboris", "rsa.misc.result_code": "natura", @@ -564,8 +564,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "internal.example.net" + "internal.example.net", + "example.org" ], "related.ip": [ "10.116.120.216", @@ -576,8 +576,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "accept" + "accept", + "PROPFIND" ], "rsa.misc.content_type": "mol", "rsa.misc.result_code": "apariat", @@ -640,12 +640,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "mail.example.net" + "mail.example.net", + "example.org" ], "related.ip": [ - "10.90.131.186", - "10.30.216.41" + "10.30.216.41", + "10.90.131.186" ], "related.user": [ "saute" @@ -810,8 +810,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "allow", - "UNLOCK" + "UNLOCK", + "allow" ], "rsa.misc.content_type": "adm", "rsa.misc.result_code": "quam", @@ -876,20 +876,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "internal.example.net" + "internal.example.net", + "example.net" ], "related.ip": [ - "10.201.76.240", - "10.210.74.24" + "10.210.74.24", + "10.201.76.240" ], "related.user": [ "uines" ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "OPTIONS", - "accept" + "accept", + "OPTIONS" ], "rsa.misc.content_type": "emips", "rsa.misc.result_code": "onse", @@ -950,12 +950,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "www.example.net" + "www.example.net", + "api.example.org" ], "related.ip": [ - "10.114.138.121", - "10.206.136.206" + "10.206.136.206", + "10.114.138.121" ], "related.user": [ "xeac" @@ -1028,8 +1028,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.org", - "www.example.com" + "www.example.com", + "mail.example.org" ], "related.ip": [ "10.200.199.166", @@ -1040,8 +1040,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "block", - "COPY" + "COPY", + "block" ], "rsa.misc.content_type": "uptat", "rsa.misc.result_code": "maccusa", @@ -1106,8 +1106,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "www5.example.com" + "www5.example.com", + "www.example.org" ], "related.ip": [ "10.122.46.71", @@ -1196,8 +1196,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "accept" + "accept", + "PROPFIND" ], "rsa.misc.content_type": "asun", "rsa.misc.result_code": "lit", @@ -1262,8 +1262,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.net" + "api.example.net", + "www5.example.net" ], "related.ip": [ "10.61.242.75", @@ -1343,20 +1343,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www5.example.com" + "www5.example.com", + "internal.example.net" ], "related.ip": [ - "10.13.59.31", - "10.214.7.83" + "10.214.7.83", + "10.13.59.31" ], "related.user": [ "etdol" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "block", - "PUT" + "PUT", + "block" ], "rsa.misc.content_type": "eprehend", "rsa.misc.result_code": "boN", @@ -1421,8 +1421,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "internal.example.net" + "internal.example.net", + "api.example.org" ], "related.ip": [ "10.89.201.140", @@ -1436,8 +1436,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "GET" + "GET", + "accept" ], "rsa.misc.content_type": "tcons", "rsa.misc.result_code": "tsu", @@ -1502,8 +1502,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "mail.example.net" + "mail.example.net", + "api.example.org" ], "related.ip": [ "10.90.86.89", @@ -1580,20 +1580,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.org" + "api.example.org", + "www5.example.net" ], "related.ip": [ - "10.14.211.43", - "10.14.48.16" + "10.14.48.16", + "10.14.211.43" ], "related.user": [ "volupt" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "cancel" + "cancel", + "PROPFIND" ], "rsa.misc.content_type": "Utenima", "rsa.misc.result_code": "uiinea", @@ -1658,12 +1658,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "internal.example.net" + "internal.example.net", + "example.com" ], "related.ip": [ - "10.47.25.230", - "10.93.123.174" + "10.93.123.174", + "10.47.25.230" ], "related.user": [ "reetdolo" @@ -1672,8 +1672,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "iusmodi", "rsa.misc.result_code": "etcons", @@ -1738,8 +1738,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "www5.example.com" + "www5.example.com", + "www5.example.net" ], "related.ip": [ "10.233.48.103", @@ -1820,8 +1820,8 @@ "mail.example.com" ], "related.ip": [ - "10.93.220.10", - "10.27.58.92" + "10.27.58.92", + "10.93.220.10" ], "related.user": [ "qui" @@ -1894,12 +1894,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "www.example.net" + "www.example.net", + "example.net" ], "related.ip": [ - "10.213.144.249", - "10.135.217.12" + "10.135.217.12", + "10.213.144.249" ], "related.user": [ "ntexplic" @@ -1976,8 +1976,8 @@ "internal.example.com" ], "related.ip": [ - "10.13.226.57", - "10.233.239.112" + "10.233.239.112", + "10.13.226.57" ], "related.user": [ "mquelau" @@ -2050,12 +2050,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "internal.example.org" + "internal.example.org", + "example.com" ], "related.ip": [ - "10.161.203.252", - "10.21.169.127" + "10.21.169.127", + "10.161.203.252" ], "related.user": [ "ice" @@ -2130,12 +2130,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.net", - "api.example.com" + "api.example.com", + "www.example.net" ], "related.ip": [ - "10.17.215.111", - "10.69.139.26" + "10.69.139.26", + "10.17.215.111" ], "related.user": [ "edqui" @@ -2208,8 +2208,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.org", - "www5.example.org" + "www5.example.org", + "mail.example.org" ], "related.ip": [ "10.104.80.189", @@ -2220,8 +2220,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "accept", - "COPY" + "COPY", + "accept" ], "rsa.misc.content_type": "onulam", "rsa.misc.result_code": "ugiat", @@ -2298,8 +2298,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "block", - "UNLOCK" + "UNLOCK", + "block" ], "rsa.misc.content_type": "emUteni", "rsa.misc.result_code": "utlab", @@ -2360,20 +2360,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.com" + "api.example.com", + "mail.example.com" ], "related.ip": [ - "10.26.96.202", - "10.119.90.128" + "10.119.90.128", + "10.26.96.202" ], "related.user": [ "oraincid" ], "rsa.internal.messageid": "ICP_QUERY", "rsa.misc.action": [ - "ICP_QUERY", - "cancel" + "cancel", + "ICP_QUERY" ], "rsa.misc.content_type": "ntsunt", "rsa.misc.result_code": "amcorp", @@ -2438,12 +2438,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.net" + "api.example.net", + "mail.example.com" ], "related.ip": [ - "10.0.98.205", - "10.76.110.144" + "10.76.110.144", + "10.0.98.205" ], "related.user": [ "upt" @@ -2519,12 +2519,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "internal.example.org" + "internal.example.org", + "www.example.org" ], "related.ip": [ - "10.224.11.165", - "10.135.46.242" + "10.135.46.242", + "10.224.11.165" ], "related.user": [ "xeaco" @@ -2608,8 +2608,8 @@ ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "TRACE", - "accept" + "accept", + "TRACE" ], "rsa.misc.content_type": "plica", "rsa.misc.result_code": "cidunt", @@ -2678,16 +2678,16 @@ "example.org" ], "related.ip": [ - "10.93.39.140", - "10.150.245.88" + "10.150.245.88", + "10.93.39.140" ], "related.user": [ "reetd" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "cancel", - "COPY" + "COPY", + "cancel" ], "rsa.misc.content_type": "iusmodte", "rsa.misc.result_code": "ntutlabo", @@ -2752,20 +2752,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "www5.example.com" + "www5.example.com", + "mail.example.net" ], "related.ip": [ - "10.61.92.2", - "10.73.207.70" + "10.73.207.70", + "10.61.92.2" ], "related.user": [ "atu" ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "UNLOCK", - "block" + "block", + "UNLOCK" ], "rsa.misc.content_type": "commodi", "rsa.misc.result_code": "ssecil", @@ -2829,20 +2829,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "internal.example.com" + "internal.example.com", + "example.net" ], "related.ip": [ - "10.84.32.178", - "10.50.124.116" + "10.50.124.116", + "10.84.32.178" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "allow" + "allow", + "GET" ], "rsa.misc.content_type": "numquam", "rsa.misc.result_code": "temUt", @@ -2909,8 +2909,8 @@ "api.example.org" ], "related.ip": [ - "10.211.234.224", - "10.173.222.131" + "10.173.222.131", + "10.211.234.224" ], "related.user": [ "Duisa" @@ -2987,8 +2987,8 @@ "www5.example.net" ], "related.ip": [ - "10.0.157.225", - "10.11.83.126" + "10.11.83.126", + "10.0.157.225" ], "related.user": [ "atu" @@ -3061,20 +3061,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.com", - "api.example.com" + "api.example.com", + "www5.example.com" ], "related.ip": [ - "10.228.77.21", - "10.92.237.93" + "10.92.237.93", + "10.228.77.21" ], "related.user": [ "onse" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "cancel" + "cancel", + "PUT" ], "rsa.misc.content_type": "mod", "rsa.misc.result_code": "gnaa", @@ -3139,12 +3139,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www5.example.net" + "www5.example.net", + "api.example.com" ], "related.ip": [ - "10.102.215.23", - "10.20.28.92" + "10.20.28.92", + "10.102.215.23" ], "related.user": [ "ntexpl" @@ -3153,8 +3153,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "POST" + "POST", + "allow" ], "rsa.misc.content_type": "nderiti", "rsa.misc.result_code": "litsedq", @@ -3221,16 +3221,16 @@ "api.example.net" ], "related.ip": [ - "10.45.28.159", - "10.17.87.79" + "10.17.87.79", + "10.45.28.159" ], "related.user": [ "tionula" ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "NONE", - "block" + "block", + "NONE" ], "rsa.misc.content_type": "uamei", "rsa.misc.result_code": "ecatcupi", @@ -3295,8 +3295,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "api.example.com" + "api.example.com", + "internal.example.com" ], "related.ip": [ "10.177.238.45", @@ -3307,8 +3307,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "allow", - "DELETE" + "DELETE", + "allow" ], "rsa.misc.content_type": "acom", "rsa.misc.result_code": "metco", @@ -3373,8 +3373,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "example.com" + "example.com", + "www.example.org" ], "related.ip": [ "10.101.85.169", @@ -3388,8 +3388,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "GET" + "GET", + "accept" ], "rsa.misc.content_type": "radi", "rsa.misc.result_code": "Finibus", @@ -3454,12 +3454,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "www5.example.org" + "www5.example.org", + "example.net" ], "related.ip": [ - "10.24.54.129", - "10.231.7.209" + "10.231.7.209", + "10.24.54.129" ], "related.user": [ "eavol" @@ -3537,8 +3537,8 @@ "www.example.com" ], "related.ip": [ - "10.121.163.5", - "10.77.129.175" + "10.77.129.175", + "10.121.163.5" ], "related.user": [ "BCS" @@ -3625,8 +3625,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "deny", - "PROPFIND" + "PROPFIND", + "deny" ], "rsa.misc.content_type": "est", "rsa.misc.result_code": "agnaaliq", @@ -3703,8 +3703,8 @@ ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "block", - "NONE" + "NONE", + "block" ], "rsa.misc.content_type": "tobeatae", "rsa.misc.result_code": "tion", @@ -3773,16 +3773,16 @@ "api.example.org" ], "related.ip": [ - "10.4.69.152", - "10.150.198.112" + "10.150.198.112", + "10.4.69.152" ], "related.user": [ "mexer" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "block", - "PUT" + "PUT", + "block" ], "rsa.misc.content_type": "abor", "rsa.misc.result_code": "uipexe", @@ -3847,8 +3847,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "www5.example.org" + "www5.example.org", + "example.org" ], "related.ip": [ "10.45.114.111", @@ -3861,8 +3861,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "accept" + "accept", + "POST" ], "rsa.misc.content_type": "mdolors", "rsa.misc.result_code": "edictasu", @@ -3924,21 +3924,21 @@ "observer.product": "Proxy", "observer.type": "Proxies", "observer.vendor": "Squid", - "related.hosts": [ - "www5.example.net", - "mail.example.net" + "related.hosts": [ + "mail.example.net", + "www5.example.net" ], "related.ip": [ - "10.49.242.174", - "10.205.28.24" + "10.205.28.24", + "10.49.242.174" ], "related.user": [ "untutlab" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "allow", - "TRACE" + "TRACE", + "allow" ], "rsa.misc.content_type": "pteurs", "rsa.misc.result_code": "consecte", @@ -4018,8 +4018,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "HEAD" + "HEAD", + "deny" ], "rsa.misc.content_type": "elites", "rsa.misc.result_code": "itseddoe", @@ -4080,12 +4080,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "internal.example.net" + "internal.example.net", + "internal.example.org" ], "related.ip": [ - "10.81.140.173", - "10.88.172.222" + "10.88.172.222", + "10.81.140.173" ], "related.user": [ "etdol" @@ -4158,16 +4158,16 @@ "example.com" ], "related.ip": [ - "10.247.53.179", - "10.162.129.196" + "10.162.129.196", + "10.247.53.179" ], "related.user": [ "identsu" ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "OPTIONS", - "accept" + "accept", + "OPTIONS" ], "rsa.misc.content_type": "eacommo", "rsa.misc.result_code": "hend", @@ -4232,16 +4232,16 @@ "www5.example.net" ], "related.ip": [ - "10.110.86.230", - "10.172.148.223" + "10.172.148.223", + "10.110.86.230" ], "related.user": [ "enimadm" ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "accept", - "UNLOCK" + "UNLOCK", + "accept" ], "rsa.misc.content_type": "atc", "rsa.misc.result_code": "upta", @@ -4306,8 +4306,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www5.example.com" + "www5.example.com", + "internal.example.net" ], "related.ip": [ "10.93.159.170", @@ -4321,8 +4321,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "deny" + "deny", + "GET" ], "rsa.misc.content_type": "eriam", "rsa.misc.result_code": "sseq", @@ -4465,8 +4465,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "internal.example.net" + "internal.example.net", + "www5.example.net" ], "related.ip": [ "10.41.156.88", @@ -4543,12 +4543,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www.example.org" + "www.example.org", + "internal.example.net" ], "related.ip": [ - "10.101.183.86", - "10.54.44.231" + "10.54.44.231", + "10.101.183.86" ], "related.user": [ "mcorpo" @@ -4557,8 +4557,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "allow" + "allow", + "CONNECT" ], "rsa.misc.content_type": "oreverit", "rsa.misc.result_code": "abor", @@ -4619,8 +4619,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "www.example.net" + "www.example.net", + "internal.example.org" ], "related.ip": [ "10.181.177.74", @@ -4697,8 +4697,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "api.example.org" + "api.example.org", + "api.example.net" ], "related.ip": [ "10.76.220.3", @@ -4712,8 +4712,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "deny" + "deny", + "GET" ], "rsa.misc.content_type": "liqua", "rsa.misc.result_code": "nderi", @@ -4778,12 +4778,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www.example.com" + "www.example.com", + "api.example.com" ], "related.ip": [ - "10.166.160.217", - "10.219.245.58" + "10.219.245.58", + "10.166.160.217" ], "related.user": [ "radip" @@ -4932,8 +4932,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.com", - "internal.example.net" + "internal.example.net", + "www.example.com" ], "related.ip": [ "10.202.224.209", @@ -4944,8 +4944,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "OPTIONS", - "cancel" + "cancel", + "OPTIONS" ], "rsa.misc.content_type": "umf", "rsa.misc.result_code": "obeataev", @@ -5010,12 +5010,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "mail.example.com" + "mail.example.com", + "mail.example.net" ], "related.ip": [ - "10.170.234.233", - "10.72.99.69" + "10.72.99.69", + "10.170.234.233" ], "related.user": [ "uatu" @@ -5088,20 +5088,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "internal.example.net" + "internal.example.net", + "api.example.org" ], "related.ip": [ - "10.245.240.47", - "10.142.130.227" + "10.142.130.227", + "10.245.240.47" ], "related.user": [ "odic" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "allow", - "DELETE" + "DELETE", + "allow" ], "rsa.misc.content_type": "scivelit", "rsa.misc.result_code": "liquaUte", @@ -5170,16 +5170,16 @@ "api.example.net" ], "related.ip": [ - "10.62.188.193", - "10.61.110.7" + "10.61.110.7", + "10.62.188.193" ], "related.user": [ "quaU" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "deny" + "deny", + "DELETE" ], "rsa.misc.content_type": "tmo", "rsa.misc.result_code": "cin", @@ -5240,20 +5240,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "www.example.net" + "www.example.net", + "mail.example.net" ], "related.ip": [ - "10.172.139.78", - "10.68.198.188" + "10.68.198.188", + "10.172.139.78" ], "related.user": [ "onsectet" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "COPY", - "block" + "block", + "COPY" ], "rsa.misc.content_type": "animid", "rsa.misc.result_code": "inea", @@ -5332,8 +5332,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "teturad", "rsa.misc.result_code": "avolu", @@ -5408,8 +5408,8 @@ ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "cancel" + "cancel", + "PURGE" ], "rsa.misc.content_type": "laboree", "rsa.misc.result_code": "oll", @@ -5476,8 +5476,8 @@ "www5.example.com" ], "related.ip": [ - "10.176.62.146", - "10.255.40.12" + "10.255.40.12", + "10.176.62.146" ], "related.user": [ "oeiusmo" @@ -5553,8 +5553,8 @@ "internal.example.com" ], "related.ip": [ - "10.194.198.46", - "10.88.98.31" + "10.88.98.31", + "10.194.198.46" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", @@ -5627,8 +5627,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.com" + "www5.example.com", + "example.com" ], "related.ip": [ "10.1.27.133", @@ -5639,8 +5639,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "block", - "OPTIONS" + "OPTIONS", + "block" ], "rsa.misc.content_type": "ntutlab", "rsa.misc.result_code": "olore", @@ -5707,8 +5707,8 @@ "internal.example.com" ], "related.ip": [ - "10.11.73.145", - "10.70.244.155" + "10.70.244.155", + "10.11.73.145" ], "related.user": [ "caboNemo" @@ -5717,8 +5717,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "POST" + "POST", + "accept" ], "rsa.misc.content_type": "ons", "rsa.misc.result_code": "temaccus", @@ -5795,8 +5795,8 @@ ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "accept", - "PURGE" + "PURGE", + "accept" ], "rsa.misc.content_type": "uames", "rsa.misc.result_code": "moenimip", @@ -5873,8 +5873,8 @@ ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "block" + "block", + "PURGE" ], "rsa.misc.content_type": "uis", "rsa.misc.result_code": "maveniam", @@ -5939,8 +5939,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "mail.example.com" + "mail.example.com", + "api.example.net" ], "related.ip": [ "10.191.220.1", @@ -5951,8 +5951,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "deny" + "deny", + "DELETE" ], "rsa.misc.content_type": "tdolo", "rsa.misc.result_code": "ntu", @@ -6021,8 +6021,8 @@ "www5.example.org" ], "related.ip": [ - "10.254.10.98", - "10.109.88.27" + "10.109.88.27", + "10.254.10.98" ], "related.user": [ "aparia" @@ -6173,8 +6173,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "internal.example.org" + "internal.example.org", + "example.com" ], "related.ip": [ "10.18.199.203", @@ -6333,16 +6333,16 @@ "api.example.org" ], "related.ip": [ - "10.153.109.61", - "10.22.34.206" + "10.22.34.206", + "10.153.109.61" ], "related.user": [ "mve" ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "block" + "block", + "PURGE" ], "rsa.misc.content_type": "velites", "rsa.misc.result_code": "uasiarch", @@ -6407,8 +6407,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "www.example.net" + "www.example.net", + "mail.example.com" ], "related.ip": [ "10.62.168.226", @@ -6421,8 +6421,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "CONNECT" + "CONNECT", + "allow" ], "rsa.misc.content_type": "sequamn", "rsa.misc.result_code": "eirured", @@ -6491,16 +6491,16 @@ "www5.example.com" ], "related.ip": [ - "10.128.84.27", - "10.97.33.56" + "10.97.33.56", + "10.128.84.27" ], "related.user": [ "ptate" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "block", - "COPY" + "COPY", + "block" ], "rsa.misc.content_type": "utla", "rsa.misc.result_code": "umfu", @@ -6563,8 +6563,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.com" + "www5.example.com", + "example.com" ], "related.ip": [ "10.49.169.175", @@ -6646,8 +6646,8 @@ "www.example.org" ], "related.ip": [ - "10.213.100.153", - "10.33.112.100" + "10.33.112.100", + "10.213.100.153" ], "related.user": [ "enimad" @@ -6724,8 +6724,8 @@ "example.net" ], "related.ip": [ - "10.25.53.93", - "10.216.143.226" + "10.216.143.226", + "10.25.53.93" ], "related.user": [ "oremeu" @@ -6734,8 +6734,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "cancel", - "CONNECT" + "CONNECT", + "cancel" ], "rsa.misc.content_type": "urau", "rsa.misc.result_code": "lla", @@ -6815,8 +6815,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "HEAD" + "HEAD", + "allow" ], "rsa.misc.content_type": "tevel", "rsa.misc.result_code": "taevitae", @@ -6881,20 +6881,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "www5.example.com" + "www5.example.com", + "api.example.net" ], "related.ip": [ - "10.82.148.126", - "10.60.56.205" + "10.60.56.205", + "10.82.148.126" ], "related.user": [ "ita" ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "block", - "NONE" + "NONE", + "block" ], "rsa.misc.content_type": "nder", "rsa.misc.result_code": "nihilmol", @@ -6963,16 +6963,16 @@ "www5.example.net" ], "related.ip": [ - "10.245.251.98", - "10.6.11.124" + "10.6.11.124", + "10.245.251.98" ], "related.user": [ "tvolu" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "accept" + "accept", + "DELETE" ], "rsa.misc.content_type": "onsequ", "rsa.misc.result_code": "strud", @@ -7041,8 +7041,8 @@ "www5.example.org" ], "related.ip": [ - "10.99.55.115", - "10.145.25.55" + "10.145.25.55", + "10.99.55.115" ], "related.user": [ "lumd" @@ -7051,8 +7051,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "iamquisn", "rsa.misc.result_code": "lorem", @@ -7195,12 +7195,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "mail.example.org" + "mail.example.org", + "example.com" ], "related.ip": [ - "10.252.146.132", - "10.163.9.35" + "10.163.9.35", + "10.252.146.132" ], "related.user": [ "umq" @@ -7275,20 +7275,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.org" + "api.example.org", + "mail.example.com" ], "related.ip": [ - "10.235.160.245", - "10.249.101.177" + "10.249.101.177", + "10.235.160.245" ], "related.user": [ "upta" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "deny", - "DELETE" + "DELETE", + "deny" ], "rsa.misc.content_type": "uameiu", "rsa.misc.result_code": "porinc", @@ -7357,16 +7357,16 @@ "mail.example.org" ], "related.ip": [ - "10.73.218.58", - "10.140.170.171" + "10.140.170.171", + "10.73.218.58" ], "related.user": [ "tinv" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "block", - "TRACE" + "TRACE", + "block" ], "rsa.misc.content_type": "umq", "rsa.misc.result_code": "nse", @@ -7443,8 +7443,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "deny", - "OPTIONS" + "OPTIONS", + "deny" ], "rsa.misc.content_type": "siuta", "rsa.misc.result_code": "emveleum", @@ -7505,8 +7505,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "api.example.com" + "api.example.com", + "www.example.org" ], "related.ip": [ "10.37.33.179", @@ -7595,8 +7595,8 @@ ], "rsa.internal.messageid": "MKOL", "rsa.misc.action": [ - "deny", - "MKOL" + "MKOL", + "deny" ], "rsa.misc.content_type": "itse", "rsa.misc.result_code": "qui", @@ -7675,8 +7675,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "POST" + "POST", + "deny" ], "rsa.misc.content_type": "rerepr", "rsa.misc.result_code": "mcorpor", @@ -7745,8 +7745,8 @@ "api.example.com" ], "related.ip": [ - "10.195.4.70", - "10.229.39.190" + "10.229.39.190", + "10.195.4.70" ], "related.user": [ "edictas" diff --git a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json index dbc2c9245df3..d9de4e04efd5 100644 --- a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json @@ -3,7 +3,7 @@ "@timestamp": "2019-11-06T09:03:00.989Z", "destination.address": "198.41.0.4", "destination.as.number": 20172, - "destination.as.organization.name": "VGRS-AC27", + "destination.as.organization.name": "VeriSign Global Registry Services", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.country_name": "United States", @@ -44,4 +44,4 @@ ], "zeek.session_id": "Ctefoj1tgOPt4D0EK2" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json index cd89fe2d77a8..5c4bd9fc36cb 100644 --- a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json +++ b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json @@ -26,8 +26,8 @@ "rci737.www5.example" ], "related.ip": [ - "10.206.191.17", - "10.176.10.114" + "10.176.10.114", + "10.206.191.17" ], "related.user": [ "sumdo" @@ -270,8 +270,8 @@ "tempor4496.www.localdomain" ], "related.ip": [ - "10.252.125.53", - "10.103.246.190" + "10.103.246.190", + "10.252.125.53" ], "related.user": [ "equun" @@ -434,8 +434,8 @@ "ollit4105.mail.localdomain" ], "related.ip": [ - "10.183.16.166", - "10.66.250.92" + "10.66.250.92", + "10.183.16.166" ], "related.user": [ "tessec" @@ -449,8 +449,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "avol", "rsa.misc.action": [ - "Allowed", - "ist" + "ist", + "Allowed" ], "rsa.misc.category": "lorema", "rsa.misc.filter": "sun", @@ -762,8 +762,8 @@ "sitvolup368.internal.host" ], "related.ip": [ - "10.71.170.37", - "10.135.225.244" + "10.135.225.244", + "10.71.170.37" ], "related.user": [ "atu" @@ -844,8 +844,8 @@ "ite2026.www.invalid" ], "related.ip": [ - "10.223.247.86", - "10.19.145.131" + "10.19.145.131", + "10.223.247.86" ], "related.user": [ "tNequepo" @@ -859,8 +859,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "sci", "rsa.misc.action": [ - "emseq", - "Allowed" + "Allowed", + "emseq" ], "rsa.misc.category": "exercit", "rsa.misc.filter": "taevit", @@ -941,8 +941,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "dolorem", "rsa.misc.action": [ - "Allowed", - "lorsitam" + "lorsitam", + "Allowed" ], "rsa.misc.category": "proide", "rsa.misc.filter": "pariatu", @@ -1023,8 +1023,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "catc", "rsa.misc.action": [ - "Allowed", - "veni" + "veni", + "Allowed" ], "rsa.misc.category": "sBono", "rsa.misc.filter": "isnisiu", @@ -1090,8 +1090,8 @@ "piscin6866.internal.host" ], "related.ip": [ - "10.135.160.125", - "10.0.55.9" + "10.0.55.9", + "10.135.160.125" ], "related.user": [ "volupta" @@ -1105,8 +1105,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iurer", "rsa.misc.action": [ - "ionevo", - "Allowed" + "Allowed", + "ionevo" ], "rsa.misc.category": "tinvolu", "rsa.misc.filter": "idex", @@ -1187,8 +1187,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "nnum", "rsa.misc.action": [ - "Allowed", - "ntoccae" + "ntoccae", + "Allowed" ], "rsa.misc.category": "tium", "rsa.misc.filter": "uteirure", @@ -1515,8 +1515,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "oNemoeni", "rsa.misc.action": [ - "Blocked", - "nre" + "nre", + "Blocked" ], "rsa.misc.category": "labo", "rsa.misc.filter": "tutlab", @@ -1761,8 +1761,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tdol", "rsa.misc.action": [ - "Allowed", - "nte" + "nte", + "Allowed" ], "rsa.misc.category": "adeseru", "rsa.misc.filter": "mac", @@ -2007,8 +2007,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "riss", "rsa.misc.action": [ - "Blocked", - "risnis" + "risnis", + "Blocked" ], "rsa.misc.category": "emqu", "rsa.misc.filter": "oluptas", @@ -2074,8 +2074,8 @@ "dquia107.www.test" ], "related.ip": [ - "10.128.173.19", - "10.88.172.34" + "10.88.172.34", + "10.128.173.19" ], "related.user": [ "agnaaliq" @@ -2156,8 +2156,8 @@ "lloin4019.www.localhost" ], "related.ip": [ - "10.130.241.232", - "10.238.224.49" + "10.238.224.49", + "10.130.241.232" ], "related.user": [ "onse" @@ -2253,8 +2253,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "quatD", "rsa.misc.action": [ - "Allowed", - "tatem" + "tatem", + "Allowed" ], "rsa.misc.category": "aincidun", "rsa.misc.filter": "uela", @@ -2402,8 +2402,8 @@ "utaliqu4248.www.localhost" ], "related.ip": [ - "10.18.226.72", - "10.101.85.169" + "10.101.85.169", + "10.18.226.72" ], "related.user": [ "rroqu" @@ -2648,8 +2648,8 @@ "lapar1599.www.lan" ], "related.ip": [ - "10.106.77.138", - "10.193.66.155" + "10.193.66.155", + "10.106.77.138" ], "related.user": [ "iusmodt" @@ -2663,8 +2663,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uteir", "rsa.misc.action": [ - "Allowed", - "Section" + "Section", + "Allowed" ], "rsa.misc.category": "cididu", "rsa.misc.filter": "Utenima", @@ -2730,8 +2730,8 @@ "aquioff3853.www.localdomain" ], "related.ip": [ - "10.236.230.136", - "10.54.159.1" + "10.54.159.1", + "10.236.230.136" ], "related.user": [ "mUteni" @@ -2812,8 +2812,8 @@ "ura675.mail.localdomain" ], "related.ip": [ - "10.49.242.174", - "10.131.246.134" + "10.131.246.134", + "10.49.242.174" ], "related.user": [ "umdolo" @@ -2827,8 +2827,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tvolup", "rsa.misc.action": [ - "utemvel", - "Allowed" + "Allowed", + "utemvel" ], "rsa.misc.category": "untutlab", "rsa.misc.filter": "dol", @@ -2976,8 +2976,8 @@ "eaque6543.api.domain" ], "related.ip": [ - "10.138.188.201", - "10.128.184.241" + "10.128.184.241", + "10.138.188.201" ], "related.user": [ "etur" @@ -3058,8 +3058,8 @@ "eufug1756.mail.corp" ], "related.ip": [ - "10.213.57.165", - "10.53.101.131" + "10.53.101.131", + "10.213.57.165" ], "related.user": [ "isau" @@ -3073,8 +3073,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ese", "rsa.misc.action": [ - "litanim", - "Allowed" + "Allowed", + "litanim" ], "rsa.misc.category": "idata", "rsa.misc.filter": "urerepre", @@ -3155,8 +3155,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "idolores", "rsa.misc.action": [ - "lestia", - "Blocked" + "Blocked", + "lestia" ], "rsa.misc.category": "risni", "rsa.misc.filter": "emacc", @@ -3222,8 +3222,8 @@ "pariatur7238.www5.invalid" ], "related.ip": [ - "10.202.224.79", - "10.33.144.10" + "10.33.144.10", + "10.202.224.79" ], "related.user": [ "rios" @@ -3386,8 +3386,8 @@ "mquisnos7453.home" ], "related.ip": [ - "10.134.128.27", - "10.118.177.136" + "10.118.177.136", + "10.134.128.27" ], "related.user": [ "Utenima" @@ -3483,8 +3483,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "amni", "rsa.misc.action": [ - "Allowed", - "edutp" + "edutp", + "Allowed" ], "rsa.misc.category": "ames", "rsa.misc.filter": "dmi", @@ -3550,8 +3550,8 @@ "remagnam796.mail.corp" ], "related.ip": [ - "10.137.164.122", - "10.143.0.78" + "10.143.0.78", + "10.137.164.122" ], "related.user": [ "orissus" @@ -3565,8 +3565,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "etdol", "rsa.misc.action": [ - "mwrit", - "Blocked" + "Blocked", + "mwrit" ], "rsa.misc.category": "inim", "rsa.misc.filter": "aturQu", @@ -3632,8 +3632,8 @@ "etdolore4227.internal.corp" ], "related.ip": [ - "10.30.87.51", - "10.156.177.53" + "10.156.177.53", + "10.30.87.51" ], "related.user": [ "psaquaea" @@ -3729,8 +3729,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tatemse", "rsa.misc.action": [ - "Blocked", - "upta" + "upta", + "Blocked" ], "rsa.misc.category": "tlabo", "rsa.misc.filter": "aliqui", @@ -3893,8 +3893,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mipsumq", "rsa.misc.action": [ - "citation", - "Allowed" + "Allowed", + "citation" ], "rsa.misc.category": "usant", "rsa.misc.filter": "Nem", @@ -3973,8 +3973,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ento", "rsa.misc.action": [ - "Bonoru", - "Blocked" + "Blocked", + "Bonoru" ], "rsa.misc.category": "luptasnu", "rsa.misc.filter": "quamni", @@ -4040,8 +4040,8 @@ "ine3181.www.invalid" ], "related.ip": [ - "10.119.53.68", - "10.121.9.5" + "10.121.9.5", + "10.119.53.68" ], "related.user": [ "ssec" @@ -4217,8 +4217,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "epor", "rsa.misc.action": [ - "etquasia", - "Allowed" + "Allowed", + "etquasia" ], "rsa.misc.category": "iaturE", "rsa.misc.filter": "rep", @@ -4377,8 +4377,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ecillum", "rsa.misc.action": [ - "emp", - "Blocked" + "Blocked", + "emp" ], "rsa.misc.category": "ciati", "rsa.misc.filter": "elit", @@ -4459,8 +4459,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iuntN", "rsa.misc.action": [ - "nim", - "Allowed" + "Allowed", + "nim" ], "rsa.misc.category": "etco", "rsa.misc.filter": "autodita", @@ -4608,8 +4608,8 @@ "pici1525.www5.corp" ], "related.ip": [ - "10.155.252.123", - "10.178.148.188" + "10.178.148.188", + "10.155.252.123" ], "related.user": [ "inrepreh" @@ -4623,8 +4623,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "inimve", "rsa.misc.action": [ - "Allowed", - "niam" + "niam", + "Allowed" ], "rsa.misc.category": "perspici", "rsa.misc.filter": "uipe", @@ -4705,8 +4705,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uamquaer", "rsa.misc.action": [ - "Blocked", - "aerat" + "aerat", + "Blocked" ], "rsa.misc.category": "quela", "rsa.misc.filter": "qui", @@ -4785,8 +4785,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tin", "rsa.misc.action": [ - "urau", - "Allowed" + "Allowed", + "urau" ], "rsa.misc.category": "isiut", "rsa.misc.filter": "cons", @@ -4852,8 +4852,8 @@ "ionu3320.api.localhost" ], "related.ip": [ - "10.250.48.82", - "10.195.153.42" + "10.195.153.42", + "10.250.48.82" ], "related.user": [ "tsedquia" @@ -4867,8 +4867,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tDuisaut", "rsa.misc.action": [ - "upidatat", - "Allowed" + "Allowed", + "upidatat" ], "rsa.misc.category": "aliquide", "rsa.misc.filter": "deriti", @@ -4934,8 +4934,8 @@ "remips1499.www.local" ], "related.ip": [ - "10.60.52.219", - "10.252.164.230" + "10.252.164.230", + "10.60.52.219" ], "related.user": [ "gnamali" @@ -5094,8 +5094,8 @@ "iatnulap7662.internal.local" ], "related.ip": [ - "10.120.215.174", - "10.248.108.55" + "10.248.108.55", + "10.120.215.174" ], "related.user": [ "prehend" @@ -5271,8 +5271,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "vento", "rsa.misc.action": [ - "reh", - "Blocked" + "Blocked", + "reh" ], "rsa.misc.category": "atev", "rsa.misc.filter": "umq", @@ -5338,8 +5338,8 @@ "onproi4354.www5.invalid" ], "related.ip": [ - "10.29.162.157", - "10.185.107.27" + "10.185.107.27", + "10.29.162.157" ], "related.user": [ "evelite" @@ -5420,8 +5420,8 @@ "beataevi7552.api.test" ], "related.ip": [ - "10.215.63.248", - "10.138.0.214" + "10.138.0.214", + "10.215.63.248" ], "related.user": [ "eavolupt" @@ -5435,8 +5435,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "odita", "rsa.misc.action": [ - "Blocked", - "dqu" + "dqu", + "Blocked" ], "rsa.misc.category": "ipex", "rsa.misc.filter": "ine", @@ -5517,8 +5517,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tNequepo", "rsa.misc.action": [ - "rmagnido", - "Allowed" + "Allowed", + "rmagnido" ], "rsa.misc.category": "luptatem", "rsa.misc.filter": "deritq", @@ -5584,8 +5584,8 @@ "quia7214.example" ], "related.ip": [ - "10.193.152.42", - "10.91.20.27" + "10.91.20.27", + "10.193.152.42" ], "related.user": [ "edict" @@ -5599,8 +5599,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "plicab", "rsa.misc.action": [ - "umq", - "Blocked" + "Blocked", + "umq" ], "rsa.misc.category": "eruntmol", "rsa.misc.filter": "labore", @@ -5681,8 +5681,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "omnisi", "rsa.misc.action": [ - "Allowed", - "userro" + "userro", + "Allowed" ], "rsa.misc.category": "etd", "rsa.misc.filter": "loremeum", @@ -5763,8 +5763,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Utenim", "rsa.misc.action": [ - "onevo", - "Allowed" + "Allowed", + "onevo" ], "rsa.misc.category": "tdolore", "rsa.misc.filter": "ptasn", @@ -5912,8 +5912,8 @@ "agna5654.www.corp" ], "related.ip": [ - "10.200.74.101", - "10.203.47.23" + "10.203.47.23", + "10.200.74.101" ], "related.user": [ "litesse" @@ -5927,8 +5927,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "nde", "rsa.misc.action": [ - "Allowed", - "iqu" + "iqu", + "Allowed" ], "rsa.misc.category": "ametco", "rsa.misc.filter": "ntincul", @@ -5994,8 +5994,8 @@ "ites5711.internal.host" ], "related.ip": [ - "10.162.78.48", - "10.24.23.209" + "10.24.23.209", + "10.162.78.48" ], "related.user": [ "ntore" @@ -6009,8 +6009,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ereprehe", "rsa.misc.action": [ - "Blocked", - "tutl" + "tutl", + "Blocked" ], "rsa.misc.category": "mip", "rsa.misc.filter": "umSecti", @@ -6091,8 +6091,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "diconseq", "rsa.misc.action": [ - "umet", - "Allowed" + "Allowed", + "umet" ], "rsa.misc.category": "ciad", "rsa.misc.filter": "oeiusmod", @@ -6158,8 +6158,8 @@ "ngelitse7535.internal.lan" ], "related.ip": [ - "10.209.203.156", - "10.110.16.169" + "10.110.16.169", + "10.209.203.156" ], "related.user": [ "mes" @@ -6173,8 +6173,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iamquisn", "rsa.misc.action": [ - "lupta", - "Blocked" + "Blocked", + "lupta" ], "rsa.misc.category": "uasiarch", "rsa.misc.filter": "usBonor", @@ -6240,8 +6240,8 @@ "tiumtot3611.internal.localdomain" ], "related.ip": [ - "10.107.68.114", - "10.84.9.150" + "10.84.9.150", + "10.107.68.114" ], "related.user": [ "sequatDu" @@ -6255,8 +6255,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "omnis", "rsa.misc.action": [ - "uianonnu", - "Allowed" + "Allowed", + "uianonnu" ], "rsa.misc.category": "Excepteu", "rsa.misc.filter": "enimadmi", @@ -6486,8 +6486,8 @@ "iavol5202.api.example" ], "related.ip": [ - "10.14.37.8", - "10.121.181.243" + "10.121.181.243", + "10.14.37.8" ], "related.user": [ "umwr" @@ -6501,8 +6501,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "vitaedic", "rsa.misc.action": [ - "Blocked", - "rinc" + "rinc", + "Blocked" ], "rsa.misc.category": "prehende", "rsa.misc.filter": "rume", @@ -6732,8 +6732,8 @@ "elit912.www5.test" ], "related.ip": [ - "10.176.233.249", - "10.75.144.118" + "10.75.144.118", + "10.176.233.249" ], "related.user": [ "isnos" @@ -6747,8 +6747,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "essequa", "rsa.misc.action": [ - "odic", - "Blocked" + "Blocked", + "odic" ], "rsa.misc.category": "cto", "rsa.misc.filter": "odite", @@ -6896,8 +6896,8 @@ "uis5050.www.local" ], "related.ip": [ - "10.97.202.149", - "10.13.125.101" + "10.13.125.101", + "10.97.202.149" ], "related.user": [ "colab" @@ -6911,8 +6911,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "atcupi", "rsa.misc.action": [ - "Blocked", - "uaUten" + "uaUten", + "Blocked" ], "rsa.misc.category": "modt", "rsa.misc.filter": "magnidol", @@ -6978,8 +6978,8 @@ "ficiad1312.api.host" ], "related.ip": [ - "10.141.66.163", - "10.230.61.102" + "10.230.61.102", + "10.141.66.163" ], "related.user": [ "umdolo" @@ -7157,8 +7157,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "neavolu", "rsa.misc.action": [ - "Blocked", - "nofdeF" + "nofdeF", + "Blocked" ], "rsa.misc.category": "remagnam", "rsa.misc.filter": "maveniam", @@ -7239,8 +7239,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ilmoles", "rsa.misc.action": [ - "tatisetq", - "Blocked" + "Blocked", + "tatisetq" ], "rsa.misc.category": "ametco", "rsa.misc.filter": "liquide", @@ -7306,8 +7306,8 @@ "Bonoru7444.www5.example" ], "related.ip": [ - "10.166.205.159", - "10.154.188.132" + "10.154.188.132", + "10.166.205.159" ], "related.user": [ "uptat" @@ -7321,8 +7321,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "proid", "rsa.misc.action": [ - "onevolu", - "Allowed" + "Allowed", + "onevolu" ], "rsa.misc.category": "iratio", "rsa.misc.filter": "odita", @@ -7399,8 +7399,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "upta", "rsa.misc.action": [ - "uovolup", - "Allowed" + "Allowed", + "uovolup" ], "rsa.misc.category": "todit", "rsa.misc.filter": "atisetq", @@ -7462,8 +7462,8 @@ "oloremeu5047.www5.invalid" ], "related.ip": [ - "10.254.119.31", - "10.172.159.251" + "10.172.159.251", + "10.254.119.31" ], "related.user": [ "usm" @@ -7544,8 +7544,8 @@ "edutpe1255.internal.lan" ], "related.ip": [ - "10.195.62.230", - "10.98.126.206" + "10.98.126.206", + "10.195.62.230" ], "related.user": [ "ptassit" @@ -7641,8 +7641,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntut", "rsa.misc.action": [ - "Blocked", - "nima" + "nima", + "Blocked" ], "rsa.misc.category": "boru", "rsa.misc.filter": "umquia", @@ -7723,8 +7723,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tquovo", "rsa.misc.action": [ - "qua", - "Allowed" + "Allowed", + "qua" ], "rsa.misc.category": "ectet", "rsa.misc.filter": "lites", @@ -7790,8 +7790,8 @@ "oremeum4231.internal.host" ], "related.ip": [ - "10.131.81.172", - "10.139.90.218" + "10.139.90.218", + "10.131.81.172" ], "related.user": [ "hende" @@ -7805,8 +7805,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "rrorsi", "rsa.misc.action": [ - "exe", - "Allowed" + "Allowed", + "exe" ], "rsa.misc.category": "mnihi", "rsa.misc.filter": "consequa", @@ -7872,8 +7872,8 @@ "ueip6097.api.host" ], "related.ip": [ - "10.152.217.174", - "10.128.43.71" + "10.128.43.71", + "10.152.217.174" ], "related.user": [ "mquiado" @@ -7887,8 +7887,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "olupt", "rsa.misc.action": [ - "Blocked", - "temvele" + "temvele", + "Blocked" ], "rsa.misc.category": "natuser", "rsa.misc.filter": "amnihil", @@ -8133,8 +8133,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "exeacomm", "rsa.misc.action": [ - "Blocked", - "volup" + "volup", + "Blocked" ], "rsa.misc.category": "ten", "rsa.misc.filter": "ssecil",