diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9d2388169a91..5c3b795b3c50 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] +- All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699] *Heartbeat* @@ -404,6 +405,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928] - Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038] - Change `checkpoint.source_object` from Long to Keyword. {issue}25124[25124] {pull}25145[25145] +- Fix Nginx module pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* @@ -847,6 +849,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] +- Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index a9f23eb4a103..37543d119e33 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -4,25 +4,37 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "-" %{NUMBER:http.response.status_code:long} -' - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} - %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: message + field: + - _tmp.url_orig + ignore_missing: true - set: field: event.kind value: event @@ -97,6 +109,31 @@ processors: } ctx.tls.version_protocol = parts[0]; +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + on_failure: - set: field: error.message diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json index 1f3600f2e09c..468bfb9fdc9e 100644 --- a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -17,6 +18,7 @@ "source.address": "::1", "source.ip": "::1", "url.original": "/", + "url.path": "/", "user.name": "-" }, { @@ -25,6 +27,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -36,7 +39,9 @@ "service.type": "apache", "source.address": "::1", "source.ip": "::1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-" }, { @@ -45,6 +50,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, @@ -61,6 +67,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -83,6 +90,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user.name": "-" }, { @@ -91,6 +99,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -113,6 +122,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/notfound", + "url.path": "/notfound", "user.name": "-" }, { @@ -121,6 +131,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -143,6 +154,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/hmm", + "url.path": "/hmm", "user.name": "-" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index 3eb3e283b198..6b4288724b18 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", "fileset.name": "access", "http.request.method": "GET", "http.response.body.bytes": 1375, @@ -19,7 +20,10 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", - "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" + "url.extension": "php", + "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "url.path": "/nagiosxi/ajaxhelper.php", + "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" }, { "@timestamp": "2019-10-16T09:53:47.000Z", @@ -29,6 +33,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "[16/Oct/2019:11:53:47 +0200] 11.19.0.217 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", "fileset.name": "access", "http.request.method": "GET", "http.version": "1.1", @@ -45,6 +50,9 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", - "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" + "url.extension": "php", + "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "url.path": "/appl/ajaxhelper.php", + "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json index b332788ad2b0..64f4926d5865 100644 --- a/filebeat/module/apache/access/test/test-vhost.log-expected.json +++ b/filebeat/module/apache/access/test/test-vhost.log-expected.json @@ -6,6 +6,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -17,7 +18,9 @@ "log.offset": 0, "service.type": "apache", "source.ip": "192.168.33.2", + "url.domain": "vhost1.domaine.fr", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/apache/access/test/test.log b/filebeat/module/apache/access/test/test.log index 7422d4415bd9..362543c91a8f 100644 --- a/filebeat/module/apache/access/test/test.log +++ b/filebeat/module/apache/access/test/test.log @@ -4,3 +4,4 @@ 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index ebe888475861..84bf4ea81a13 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -16,7 +17,9 @@ "service.type": "apache", "source.address": "::1", "source.ip": "::1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-" }, { @@ -25,6 +28,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -38,6 +42,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -53,6 +58,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, @@ -69,6 +75,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -82,6 +89,7 @@ "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", @@ -97,6 +105,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -110,6 +119,7 @@ "source.address": "monitoring-server", "source.domain": "monitoring-server", "url.original": "/status", + "url.path": "/status", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", @@ -125,6 +135,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "event.outcome": "failure", "fileset.name": "access", "http.request.referrer": "-", @@ -139,5 +150,36 @@ "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "-" + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": "web", + "event.dataset": "apache.access", + "event.kind": "event", + "event.module": "apache", + "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "event.outcome": "success", + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 612, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 666, + "service.type": "apache", + "source.address": "monitoring-server", + "source.domain": "monitoring-server", + "url.extension": "mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox Alpha", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "user_agent.os.full": "Windows 7", + "user_agent.os.name": "Windows", + "user_agent.os.version": "7", + "user_agent.version": "15.0.a2" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json index e9680e5b7fbc..bcff56976927 100644 --- a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -18,6 +19,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Wget", @@ -31,6 +33,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -44,6 +47,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -59,6 +63,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -71,7 +76,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -87,6 +94,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -100,6 +108,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -115,6 +124,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -127,7 +137,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -143,6 +155,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -155,7 +168,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -171,6 +186,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -184,6 +200,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/test", + "url.path": "/test", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -199,6 +216,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -212,6 +230,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -227,6 +246,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -240,6 +260,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/crap", + "url.path": "/crap", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index aad4c3f4a5f5..4b8495dd9c89 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -3,8 +3,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} @@ -14,6 +17,13 @@ processors: pattern_definitions: APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' ignore_missing: true +- grok: + field: message + patterns: + - "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}" + - "File does not exist: %{URIPATH:file.path}" + ignore_missing: true + ignore_failure: true - date: if: ctx.event.timezone == null field: apache.error.timestamp @@ -38,7 +48,9 @@ processors: field: error.message value: '{{ _ingest.on_failure_message }}' - remove: - field: apache.error.timestamp + field: + - apache.error.timestamp + - _tmp.url_orig ignore_failure: true - set: field: event.kind @@ -82,6 +94,30 @@ processors: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json index 4453f2bebec4..90990fc95a47 100644 --- a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json @@ -6,6 +6,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -23,6 +24,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json index d9f470db46a4..6bbb6067469b 100644 --- a/filebeat/module/apache/error/test/test.log-expected.json +++ b/filebeat/module/apache/error/test/test.log-expected.json @@ -5,8 +5,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -23,6 +25,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -40,8 +43,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/usr/local/apache2/htdocs/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -70,6 +75,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", diff --git a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json index 41adf5d7ae58..c946f0c739ad 100644 --- a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -20,9 +21,12 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", + "http.request.referrer": "http://192.168.33.72/", "input.type": "log", "log.level": "error", "log.offset": 100, @@ -37,8 +41,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -54,8 +60,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -71,8 +79,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/test", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -88,8 +98,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/hello", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -105,8 +117,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/crap", "fileset.name": "error", "input.type": "log", "log.level": "error", diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 2e92efd48d56..afbd20c78baf 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -4,42 +4,65 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{NOTSPACE:iis.access.site_name}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:iis.access.cookie}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NOTSPACE:destination.domain}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:http.response.body.bytes:long}) (?:-|%{NUMBER:http.request.body.bytes:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{NOTSPACE:iis.access.site_name}) (?:-|%{NOTSPACE:iis.access.server_name}) - (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:url.path}) - (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.url_path}) + (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|HTTP/%{NUMBER:http.version}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:iis.access.cookie}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NOTSPACE:destination.domain}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:http.response.body.bytes:long}) (?:-|%{NUMBER:http.request.body.bytes:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} \[%{IPORHOST:destination.address}\]\(http://%{IPORHOST:destination.address}\) - (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) + (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) \[%{IPORHOST:source.address}\]\(http://%{IPORHOST:source.address}\) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' ignore_missing: true +- uri_parts: + field: _tmp.url_path + ignore_failure: true +- set: + field: url.original + value: "{{_tmp.url_path}}" + ignore_failure: true + if: ctx?._tmp?.url_path != null && ctx?.url?.original == null +- urldecode: + field: _tmp.url_query + target_field: url.query + ignore_missing: true + ignore_failure: true + if: ctx.url?.query == null +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: message + field: + - _tmp + ignore_missing: true - rename: field: '@timestamp' target_field: event.created @@ -133,6 +156,30 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json index 64ad587bb8bc..5f37c7d4e43b 100644 --- a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 0", "event.outcome": "failure", "event.type": [ "connection" @@ -30,8 +31,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -53,6 +56,7 @@ "event.duration": 46000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 46", "event.outcome": "failure", "event.type": [ "connection" @@ -71,8 +75,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -94,6 +100,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /Director - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -112,6 +119,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/Director", "url.path": "/Director", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -134,6 +142,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET / - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -152,6 +161,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -174,6 +184,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 15", "event.outcome": "failure", "event.type": [ "connection" @@ -192,8 +203,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json index 952105369253..df3c9df6d04d 100644 --- a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 792000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-08-28 18:24:25 [10.100.220.70](http://10.100.220.70) GET / - 80 - [10.100.118.31](http://10.100.118.31) Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729) 404 4 2 792", "event.outcome": "failure", "event.type": [ "connection" @@ -30,6 +31,7 @@ "service.type": "iis", "source.address": "10.100.118.31", "source.ip": "10.100.118.31", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -52,6 +54,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -70,6 +73,7 @@ "service.type": "iis", "source.address": "10.0.140.2", "source.ip": "10.0.140.2", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" }, { @@ -85,6 +89,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -103,6 +108,7 @@ "service.type": "iis", "source.address": "10.0.140.2", "source.ip": "10.0.140.2", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" }, { @@ -118,6 +124,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 2001:cdba:0000:0000:0000:0000:3257:9652 GET /health-monitoring - 80 - 2001:cdba:0000:0000:0000:0000:3257:9652 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -136,6 +143,7 @@ "service.type": "iis", "source.address": "2001:cdba:0000:0000:0000:0000:3257:9652", "source.ip": "2001:cdba:0000:0000:0000:0000:3257:9652", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" } ] \ No newline at end of file diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json index c3f4a4932dac..35ce856496dc 100644 --- a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json +++ b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json @@ -13,6 +13,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME ::1%0 GET / - 80 - ::1%0 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "event.type": [ "connection" @@ -36,6 +37,8 @@ "service.type": "iis", "source.address": "::1%0", "source.ip": "::1", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", diff --git a/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json b/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json index 565bdfca17d6..edcefdf8e65c 100644 --- a/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json +++ b/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json @@ -13,6 +13,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-04 22:00:34 W3SVC2 freca1 10.24.129.162 GET /favicon.ico - 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64;+rv:81.0)+Gecko/20100101+Firefox/81.0 - https://images.hogeschoolrotterdam.nl/Blob/adeec119008c48758c1a6be53aeeb2ac/34ff475072d54117bcb46ea7f023bd87.jpg?width=1200&height=630&mode=crop images.hogeschoolrotterdam.nl 404 0 2 1437 534 0 116.189.86.89", "event.outcome": "failure", "event.type": [ "connection" @@ -38,6 +39,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "ico", + "url.original": "/favicon.ico", "url.path": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -61,6 +65,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:40:30 W3SVC2 freca1 10.24.129.162 GET /robots.txt - 80 - 10.24.136.240 HTTP/1.1 Twitterbot/1.0 - - images.hogeschoolrotterdam.nl 200 0 0 346 306 0 119.16.157.180", "event.outcome": "success", "event.type": [ "connection" @@ -85,6 +90,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "txt", + "url.original": "/robots.txt", "url.path": "/robots.txt", "user_agent.device.name": "Spider", "user_agent.name": "Twitterbot", @@ -105,6 +113,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:48:33 W3SVC2 freca1 10.24.129.162 GET /app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg width=35&height=38&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36+Edge/18.18362 BIGipServerYAkgvoMsadfHHYQAGkWEWnyAqAads=27246369425.20480.0000 https://www.rotterdamuas.com/ images.hogeschoolrotterdam.nl 200 0 0 2007 581 15 119.160.162.213", "event.outcome": "success", "event.type": [ "connection" @@ -131,6 +140,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg", "url.path": "/app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg", "url.query": "width=35&height=38&mode=crop", "user_agent.device.name": "Other", @@ -155,6 +167,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:48:33 W3SVC2 freca1 10.24.129.162 GET /app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg width=75&height=40&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36+Edge/18.18362 BIGipServerYAkgvoMHHYQAGkWEWadfsadfnyAqAere=27246369425.20480.0000 https://www.rotterdamuas.com/ images.hogeschoolrotterdam.nl 200 0 0 2926 581 0 119.160.162.213", "event.outcome": "success", "event.type": [ "connection" @@ -181,6 +194,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg", "url.path": "/app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg", "url.query": "width=75&height=40&mode=crop", "user_agent.device.name": "Other", @@ -205,6 +221,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-08 22:00:22 W3SVC2 freca1 10.24.129.162 GET /Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png width=60&height=20&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(iPhone;+CPU+iPhone+OS+13_7+like+Mac+OS+X)+AppleWebKit/605.1.15+(KHTML,+like+Gecko)+Version/13.1.2+Mobile/15E148+Safari/604.1 imagedpi=1;+imagequality=30;+_ga=GA1.2.14440223724.16021494379;+_gid=GA1.2.12504478161.16021944379;+_gat_UA-155746052-5=1;+BIGipServerYAkgvoMHHYQAGkWsadfsdfEWnyAqA=!vbipVwbt0UWJ4QSsPvjjVxViTXMqdXWYaferu1dRyQoUTVVBGLZPB39PoNX5Tw66+GT0ChJCfnp/xfhM8lI=;+_gcl_au=1.1.1297303538.1602194379 https://www.hogeschoolrotterdam.nl/opleidingen/bachelor/elektrotechniek/voltijd/ images.hogeschoolrotterdam.nl 304 0 0 388 979 15 156.189.143.218", "event.outcome": "success", "event.type": [ "connection" @@ -231,6 +248,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "png", + "url.original": "/Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png", "url.path": "/Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png", "url.query": "width=60&height=20&mode=crop", "user_agent.device.name": "iPhone", @@ -255,6 +275,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-08 22:00:22 W3SVC2 freca1 10.24.129.162 GET /Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg width=60&height=20&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(iPhone;+CPU+iPhone+OS+13_7+like+Mac+OS+X)+AppleWebKit/605.1.15+(KHTML,+like+Gecko)+Version/13.1.2+Mobile/15E148+Safari/604.1 imagedpi=1;+imagequality=30;+_ga=GA1.2.14440223724.16021494379;+_gid=GA1.2.12504748161.16021944379;+_gat_UA-155764052-5=1;+BIGipServerYAkgvoMHHYQAGkWEsadfsdfsWnyAqA=!vbipVwbt0UWJ4QSsPvjjVxViTXMqdXWYu1dRyQoUTVerwerVBGLZPB39PoNX5Tw66+GT0ChJCfnp/xfhM8lI=;+_gcl_au=1.1.1297303538.1602194379 https://www.hogeschoolrotterdam.nl/opleidingen/bachelor/elektrotechniek/voltijd/ images.hogeschoolrotterdam.nl 304 0 0 388 979 0 156.189.143.218", "event.outcome": "success", "event.type": [ "connection" @@ -281,6 +302,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg", "url.path": "/Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg", "url.query": "width=60&height=20&mode=crop", "user_agent.device.name": "iPhone", diff --git a/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json b/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json index 4d8ace5a7fb5..8977e920292e 100644 --- a/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json +++ b/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 26000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 26 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -32,6 +33,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -54,6 +56,7 @@ "event.duration": 32000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 32 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -74,6 +77,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -96,6 +100,7 @@ "event.duration": 46000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 46 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -116,6 +121,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -138,6 +144,7 @@ "event.duration": 32000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 GET /Production-UI/data/finance/legacy/GLAPAprvMaster $filter=BatchId%20eq%20%27FY21HSNG0820%27&$orderby=Subsys,Ref&$skip=0&$top=20 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 32 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -158,8 +165,9 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/data/finance/legacy/GLAPAprvMaster", "url.path": "/Production-UI/data/finance/legacy/GLAPAprvMaster", - "url.query": "$filter=BatchId%20eq%20%27FY21HSNG0820%27&$orderby=Subsys,Ref&$skip=0&$top=20", + "url.query": "$filter=BatchId eq 'FY21HSNG0820'&$orderby=Subsys,Ref&$skip=0&$top=20", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", @@ -181,6 +189,7 @@ "event.duration": 166000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 GET /Production-UI/data/finance/legacy/GLATrnsDetail $filter=Subsys%20eq%20%27JE%27%20and%20Ref%20eq%20%27HSNG08-MR%27%20and%20BatchId%20eq%20%27FY21HSNG0820%27&$orderby=RecNo&$skip=0&$top=20 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 166 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -201,8 +210,9 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/data/finance/legacy/GLATrnsDetail", "url.path": "/Production-UI/data/finance/legacy/GLATrnsDetail", - "url.query": "$filter=Subsys%20eq%20%27JE%27%20and%20Ref%20eq%20%27HSNG08-MR%27%20and%20BatchId%20eq%20%27FY21HSNG0820%27&$orderby=RecNo&$skip=0&$top=20", + "url.query": "$filter=Subsys eq 'JE' and Ref eq 'HSNG08-MR' and BatchId eq 'FY21HSNG0820'&$orderby=RecNo&$skip=0&$top=20", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", @@ -224,6 +234,7 @@ "event.duration": 60000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 GET /Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 60 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -244,6 +255,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB", "url.path": "/Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -266,6 +278,7 @@ "event.duration": 72000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/ - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 72 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -286,6 +299,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/", "url.path": "/Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -308,6 +322,7 @@ "event.duration": 88000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/ - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 88 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -328,6 +343,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/", "url.path": "/Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -350,6 +366,7 @@ "event.duration": 286000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:07:02 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/attachDoc - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 286 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -370,6 +387,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/attachDoc", "url.path": "/Production-UI/api/finance/legacy/documents/attachDoc", "user_agent.device.name": "Other", "user_agent.name": "Chrome", diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 786333c1379d..cd809c61bc7e 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 123000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 08:09:10 127.0.0.1 GET / q=100 80 - 85.181.35.98 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - 200 0 0 123", "event.outcome": "success", "event.type": [ "connection" @@ -40,6 +41,7 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.original": "/", "url.path": "/", "url.query": "q=100", "user_agent.device.name": "Other", @@ -61,6 +63,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 09:10:11 W3SVC1 GET / - 80 - 127.0.0.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "fileset.name": "access", "http.request.body.bytes": 456, @@ -78,6 +81,8 @@ "service.type": "iis", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -101,6 +106,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "event.type": [ "connection" @@ -134,6 +140,8 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -156,6 +164,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:52:33 10.44.0.136 GET / redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()} 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -174,8 +183,9 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/", "url.path": "/", - "url.query": "redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()}", + "url.query": "redirect:${#req=#context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),#webroot=#req.getSession().getServletContext().getRealPath('/'),#resp.println(#webroot),#resp.flush(),#resp.close()}", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -197,6 +207,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:52:33 10.44.0.136 GET /${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 0", "event.outcome": "failure", "event.type": [ "connection" @@ -215,7 +226,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", - "url.path": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", + "url.original": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index a16fde841daf..6a09d937fed0 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -3,21 +3,34 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) - (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:url.original}) (?:%{NUMBER}|-) + (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:_tmp.url_orig}) (?:%{NUMBER}|-) (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:-|%{NOTSPACE:iis.error.reason_phrase}) (?:-|%{NOTSPACE:iis.error.queue_name})' - '%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) - (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:url.original}) (?:%{NUMBER:http.response.status_code:long}|-) + (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:_tmp.url_orig}) (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:-|%{NOTSPACE:iis.error.reason_phrase}) (?:-|%{NOTSPACE:iis.error.queue_name})' ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- set: + field: url.original + value: "{{{_tmp.url_orig}}}" + ignore_failure: true + if: ctx?._tmp?.url_orig != null && ctx?.url?.original == null - remove: - field: message + field: + - _tmp + ignore_missing: true - rename: field: '@timestamp' target_field: event.created @@ -90,6 +103,30 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/iis/error/test/iis_error_url.log-expected.json b/filebeat/module/iis/error/test/iis_error_url.log-expected.json index 8adb2e6d1d06..dcfa5353878e 100644 --- a/filebeat/module/iis/error/test/iis_error_url.log-expected.json +++ b/filebeat/module/iis/error/test/iis_error_url.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/0.9 t3 12.2.1 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -35,7 +36,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "12.2.1" + "url.extension": "1", + "url.original": "12.2.1", + "url.path": "12.2.1" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -49,6 +52,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET ./././././../../../../../../../../ 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -73,7 +77,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "./././././../../../../../../../../" + "url.extension": "/", + "url.original": "./././././../../../../../../../../", + "url.path": "./././././../../../../../../../../" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -87,6 +93,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /..\\pixfir~1\\how_to_login.html 403 - Forbidden -", "event.outcome": "failure", "event.type": [ "connection" @@ -125,6 +132,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET ..\\..\\..\\..\\..\\..\\winnt\\win.ini 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -163,6 +171,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini 404 - NotFound -", "event.outcome": "failure", "event.type": [ "connection" @@ -187,7 +196,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini" + "url.extension": "ini", + "url.original": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini", + "url.path": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -201,6 +212,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /nessus\\..\\..\\..\\..\\..\\..\\winnt\\win.ini 403 - Forbidden -", "event.outcome": "failure", "event.type": [ "connection" @@ -239,6 +251,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 OPTIONS * 404 - NotFound -", "event.outcome": "failure", "event.type": [ "connection" @@ -263,7 +276,8 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "*" + "url.original": "*", + "url.path": "*" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -277,6 +291,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /fee&fie=foe 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -301,6 +316,7 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/fee&fie=foe" + "url.original": "/fee&fie=foe", + "url.path": "/fee&fie=foe" } ] \ No newline at end of file diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json index ebcb2f36b842..e77f92bcc56a 100644 --- a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-30 14:22:07 ::1%0 49958 ::1%0 80 - - - - - - Timer_ConnectionIdle -", "event.type": [ "connection" ], diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json index 506ee6ba2edd..fdf8fa4bdf16 100644 --- a/filebeat/module/iis/error/test/test.log-expected.json +++ b/filebeat/module/iis/error/test/test.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 08:09:10 172.31.77.6 2094 172.31.77.6 80 HTTP/1.1 GET /qos/1kbfile.txt 503 - ConnLimit -", "event.outcome": "failure", "event.type": [ "connection" @@ -30,7 +31,9 @@ "source.address": "172.31.77.6", "source.ip": "172.31.77.6", "source.port": 2094, - "url.original": "/qos/1kbfile.txt" + "url.extension": "txt", + "url.original": "/qos/1kbfile.txt", + "url.path": "/qos/1kbfile.txt" }, { "@timestamp": "2018-01-01T09:10:11.000Z", @@ -44,6 +47,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 09:10:11 85.181.35.98 2780 127.0.0.1 80 HTTP/1.1 GET /ThisIsMyUrl.htm 400 - Hostname -", "event.outcome": "failure", "event.type": [ "connection" @@ -73,7 +77,9 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "source.port": 2780, - "url.original": "/ThisIsMyUrl.htm" + "url.extension": "htm", + "url.original": "/ThisIsMyUrl.htm", + "url.path": "/ThisIsMyUrl.htm" }, { "@timestamp": "2018-01-01T10:11:12.000Z", @@ -87,6 +93,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 85.181.35.98 2894 127.0.0.1 80 HTTP/2.0 GET / 505 - Version_N/S -", "event.outcome": "failure", "event.type": [ "connection" @@ -116,7 +123,8 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "source.port": 2894, - "url.original": "/" + "url.original": "/", + "url.path": "/" }, { "@timestamp": "2018-01-01T11:12:13.000Z", @@ -130,6 +138,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 11:12:13 85.181.35.98 64388 127.0.0.1 80 - - - - - Timer_MinBytesPerSecond -", "event.type": [ "connection" ], diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index 57fe9031b557..4a597e30d391 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -4,8 +4,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" @@ -19,11 +22,21 @@ processors: - grok: field: nginx.access.info patterns: - - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' + - '%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}' - "" ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: nginx.access.info + field: + - nginx.access.info + - _tmp.url_orig + ignore_missing: true - split: field: nginx.access.remote_ip_list separator: '"?,?\s+' @@ -93,8 +106,6 @@ processors: patterns: - ^%{IP:source.ip}$ ignore_failure: true -- remove: - field: message - rename: field: '@timestamp' target_field: event.created @@ -161,6 +172,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/nginx/access/test/access.log b/filebeat/module/nginx/access/test/access.log index 58991d26b3b9..7acb1428af89 100644 --- a/filebeat/module/nginx/access/test/access.log +++ b/filebeat/module/nginx/access/test/access.log @@ -10,3 +10,5 @@ 127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] "GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index 7981a316c952..6a0af6f499c7 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -39,6 +40,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -55,6 +57,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -87,7 +90,9 @@ "source.geo.region_iso_code": "DE-RP", "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -104,6 +109,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -136,6 +142,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/adsasd", + "url.path": "/adsasd", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -152,6 +159,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -184,6 +192,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -200,6 +209,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -232,7 +242,9 @@ "source.geo.region_iso_code": "DE-RP", "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -249,6 +261,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -281,6 +294,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", + "url.path": "/test", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -297,6 +311,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -329,6 +344,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", + "url.path": "/test", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -345,6 +361,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -377,6 +394,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test1", + "url.path": "/test1", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -393,6 +411,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -415,6 +434,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/test1", + "url.path": "/test1", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -431,6 +451,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -453,6 +474,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -469,6 +491,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -491,6 +514,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -507,6 +531,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -529,6 +554,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/taga", + "url.path": "/taga", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -536,5 +562,92 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.12", "user_agent.version": "49.0." + }, + { + "@timestamp": "2020-06-09T19:10:39.000Z", + "destination.domain": "lessons.example.com", + "event.category": [ + "web" + ], + "event.dataset": "nginx.access", + "event.kind": "event", + "event.module": "nginx", + "event.original": "lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "http.response.body.bytes": 7648063, + "http.response.status_code": 206, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2346, + "nginx.access.remote_ip_list": [ + "192.168.0.1" + ], + "related.ip": [ + "192.168.0.1" + ], + "service.type": "nginx", + "source.address": "192.168.0.1", + "source.ip": "192.168.0.1", + "url.domain": "lessons.example.com", + "url.extension": "mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user_agent.device.name": "Kindle", + "user_agent.name": "Amazon Silk", + "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", + "user_agent.os.full": "Android 5.1.1", + "user_agent.os.name": "Android", + "user_agent.os.version": "5.1.1", + "user_agent.version": "81.2.16" + }, + { + "@timestamp": "2020-06-09T19:15:39.000Z", + "destination.domain": "lessons.example.com", + "event.category": [ + "web" + ], + "event.dataset": "nginx.access", + "event.kind": "event", + "event.module": "nginx", + "event.original": "lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "http.response.body.bytes": 7648063, + "http.response.status_code": 206, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2727, + "nginx.access.remote_ip_list": [ + "192.168.0.1" + ], + "related.ip": [ + "192.168.0.1" + ], + "service.type": "nginx", + "source.address": "192.168.0.1", + "source.ip": "192.168.0.1", + "url.domain": "lessons.example.com", + "url.original": "/%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/", + "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", + "user_agent.device.name": "Kindle", + "user_agent.name": "Amazon Silk", + "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", + "user_agent.os.full": "Android 5.1.1", + "user_agent.os.name": "Android", + "user_agent.os.version": "5.1.1", + "user_agent.version": "81.2.16" } ] \ No newline at end of file diff --git a/filebeat/module/nginx/access/test/test-with-host.log-expected.json b/filebeat/module/nginx/access/test/test-with-host.log-expected.json index e07836ce5208..85ba8c494f37 100644 --- a/filebeat/module/nginx/access/test/test-with-host.log-expected.json +++ b/filebeat/module/nginx/access/test/test-with-host.log-expected.json @@ -8,6 +8,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -31,7 +32,9 @@ "service.type": "nginx", "source.address": "10.0.0.2", "source.ip": "10.0.0.2", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -49,6 +52,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -70,7 +74,9 @@ "service.type": "nginx", "source.address": "172.17.0.1", "source.ip": "172.17.0.1", + "url.domain": "example.com", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -88,6 +94,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -121,7 +128,9 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -140,6 +149,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -171,7 +181,9 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -190,6 +202,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -224,7 +237,10 @@ "source.geo.region_iso_code": "US-IL", "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", + "url.domain": "example.com", "url.original": "/assets/xxxx?q=100", + "url.path": "/assets/xxxx", + "url.query": "q=100", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -238,6 +254,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -267,7 +284,9 @@ "source.geo.location.lat": 39.5, "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.extension": "html", "url.original": "/test.html", + "url.path": "/test.html", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -283,6 +302,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -314,6 +334,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -325,7 +346,8 @@ "input.type": "log", "log.offset": 1269, "service.type": "nginx", - "source.address": "unix:" + "source.address": "unix:", + "url.domain": "example.com" }, { "@timestamp": "2017-05-29T19:02:48.000Z", @@ -336,6 +358,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -357,6 +380,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -374,6 +398,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -392,7 +417,9 @@ ], "service.type": "nginx", "source.address": "localhost", + "url.domain": "example.com", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index b27c9ccf19bc..60731f0cc107 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -31,6 +32,7 @@ "source.address": "10.0.0.2", "source.ip": "10.0.0.2", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -47,6 +49,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -69,6 +72,7 @@ "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -85,6 +89,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -119,6 +124,7 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -135,6 +141,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -167,6 +174,7 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -183,6 +191,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -218,6 +227,8 @@ "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", "url.original": "/assets/xxxx?q=100", + "url.path": "/assets/xxxx", + "url.query": "q=100", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -230,6 +241,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -258,7 +270,9 @@ "source.geo.location.lat": 39.5, "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.extension": "html", "url.original": "/test.html", + "url.path": "/test.html", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -272,6 +286,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -300,6 +315,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -321,6 +337,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -339,6 +356,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -355,6 +373,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -374,6 +393,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 05691eeb7375..51170f6cbbbf 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -3,8 +3,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}' diff --git a/filebeat/module/nginx/error/test/error.log-expected.json b/filebeat/module/nginx/error/test/error.log-expected.json index 8896a4907059..ac5e5f83caf7 100644 --- a/filebeat/module/nginx/error/test/error.log-expected.json +++ b/filebeat/module/nginx/error/test/error.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2016/10/25 14:49:34 [error] 54053#0: *1 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "event.timezone": "-02:00", "event.type": [ "error" @@ -29,6 +30,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2016/10/25 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "event.timezone": "-02:00", "event.type": [ "error" @@ -51,6 +53,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "event.timezone": "-02:00", "event.type": [ "error" @@ -76,6 +79,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2019/11/05 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "event.timezone": "-02:00", "event.type": [ "error" diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 64fd7567ba1e..14a5da9cae96 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -5,8 +5,11 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - grok: + - rename: field: message + target_field: event.original + - grok: + field: event.original patterns: - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" @@ -31,6 +34,13 @@ processors: - "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}" - "" ignore_missing: true + - uri_parts: + field: url.original + ignore_failure: true + - set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: field: nginx.ingress_controller.info - split: @@ -198,8 +208,6 @@ processors: patterns: - ^%{IP:source.ip}$ ignore_failure: true - - remove: - field: message - rename: field: "@timestamp" target_field: event.created @@ -266,6 +274,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index c8ba580f64d0..ef502ebe86be 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -21,3 +21,4 @@ 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f 192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 +192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index c9cbfb36b7fd..7a9e13bf58ea 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:48:51 +0000] \"POST /products HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 89 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 529a007902362a5f51385a5fa7049884", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -52,6 +53,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products", + "url.path": "/products", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -65,6 +67,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:15 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 91 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 68fa971ce4dfce685fdc01c877bfa645", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -110,6 +113,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -123,6 +127,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:30 +0000] \"DELETE /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 94 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 0be411044cb1cb67580e115413b2da60", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -168,6 +173,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -181,6 +187,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:43 +0000] \"PATCH /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 93 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 f479ab1d9cc8afcbac9e9f958ff8babc", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -226,6 +233,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -239,6 +247,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:50 +0000] \"PATCHp /products/42 HTTP/1.1\" 400 163 \"-\" \"-\" 0 0.000 [] [] - - - - 4c7d2079340e68353c7d0dfff00b904b", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -266,7 +275,8 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42" + "url.original": "/products/42", + "url.path": "/products/42" }, { "@timestamp": "2020-02-07T11:50:09.000Z", @@ -276,6 +286,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:50:09 +0000] \"geti /products/42 HTTP/1.1\" 400 163 \"-\" \"-\" 0 0.000 [] [] - - - - efb0c5aa8be6cdeb4a7e7bd090e3d893", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -303,7 +314,8 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42" + "url.original": "/products/42", + "url.path": "/products/42" }, { "@timestamp": "2020-02-07T11:55:05.000Z", @@ -313,6 +325,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:05 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Wget/1.20.3 (darwin18.6.0)\" 157 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 457b71c3e1ee1887bb809effd301a0ec", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -358,6 +371,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Wget", "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", @@ -371,6 +385,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 450 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 da29abf31e4d6324cebe5e7bca370709", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -416,6 +431,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -432,6 +448,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/products/42\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 381 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 e983c8cf3d713548baa50c9e2fffeb34", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -477,7 +494,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -494,6 +513,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 441 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 3d7ff18ff4181a7db5013a76f975d900", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -539,6 +559,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -555,6 +576,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/v2\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 372 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 d131fe4bcd359cf947f75efca4bfa553", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -600,7 +622,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -617,6 +641,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 ef6629fcaaa1ea0d1a843cf2bf40571d", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -662,6 +687,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -678,6 +704,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/products/42\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 325 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 2593a1126588922449c183c8d9ddbbeb", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -723,7 +750,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -740,6 +769,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.002 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 0f76ea730f282d5759018eb756b23b14", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -785,6 +815,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -801,6 +832,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET / HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 358 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 21efd18e3a7952fc78c0f2dcc1f05e69", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -846,6 +878,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -862,6 +895,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 314 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 e096809c2cb46f004c3b538b23916e5b", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -907,7 +941,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -924,6 +960,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 360 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.002 200 0a2a92d080e664dd4e95c85d097c9d3d", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -969,6 +1006,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -985,6 +1023,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/v2\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 316 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 4c024ebfc20acfb2d59e542e3ed60789", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1030,7 +1069,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -1047,6 +1088,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:00:28 +0000] \"GET /products/42?address=delhi+technological+university HTTP/1.1\" 200 59 \"-\" \"python-requests/2.22.0\" 197 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 9a7babf34ca4ee59d90ac48d452a9214", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1092,6 +1134,8 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42?address=delhi+technological+university", + "url.path": "/products/42", + "url.query": "address=delhi+technological+university", "user_agent.device.name": "Other", "user_agent.name": "Python Requests", "user_agent.original": "python-requests/2.22.0", @@ -1105,6 +1149,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1150,6 +1195,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1166,6 +1212,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1210,7 +1257,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1227,6 +1276,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] \"GET /v2/some HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1272,6 +1322,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2/some", + "url.path": "/v2/some", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1288,6 +1339,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /v2/some HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1337,6 +1389,75 @@ "source.address": "192.168.64.14", "source.ip": "192.168.64.14", "url.original": "/v2/some", + "url.path": "/v2/some", + "user_agent.device.name": "Mac", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + }, + { + "@timestamp": "2020-02-07T12:02:42.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "http.request.method": "GET", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 6026, + "nginx.ingress_controller.http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "nginx.ingress_controller.http.request.length": 348, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.14" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.7", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, + "nginx.ingress_controller.upstream.response.length": 100, + "nginx.ingress_controller.upstream.response.length_list": [ + "61", + "100" + ], + "nginx.ingress_controller.upstream.response.status_code": 203, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200", + "203" + ], + "nginx.ingress_controller.upstream.response.time": 0.104, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.100", + "0.004" + ], + "nginx.ingress_controller.upstream_address_list": [ + "172.17.0.6:8080", + "172.17.0.7:8080" + ], + "related.ip": [ + "192.168.64.14" + ], + "service.type": "nginx", + "source.address": "192.168.64.14", + "source.ip": "192.168.64.14", + "url.extension": "mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index dd5de1b0b0b6..3b63a05fe8d1 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -4,10 +4,13 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- dissect: +- rename: field: message + target_field: event.original +- dissect: + field: event.original pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] - "%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code} + "%{http.request.method} %{temp.url_orig} HTTP/%{http.version}" %{http.response.status_code} %{traefik.access.message}' - grok: field: traefik.access.message @@ -17,12 +20,9 @@ processors: (?:"%{DATA:traefik.access.frontend_name}"|-)?)?( "%{DATA:traefik.access.backend_url}")?( %{NUMBER:temp.duration:long}ms)?)? ignore_missing: true -- remove: - field: message - ignore_missing: true -- remove: - field: traefik.access.message - ignore_missing: true +- uri_parts: + field: temp.url_orig + ignore_failure: true - rename: field: '@timestamp' target_field: event.created @@ -31,8 +31,6 @@ processors: target_field: '@timestamp' formats: - dd/MMM/yyyy:H:m:s Z -- remove: - field: traefik.access.time - convert: field: http.response.status_code type: long @@ -47,7 +45,10 @@ processors: scale: 1000000 if: ctx.temp?.duration != null - remove: - field: temp.duration + field: + - temp + - traefik.access.message + - traefik.access.time ignore_missing: true - user_agent: field: user_agent.original @@ -103,6 +104,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null && ctx.user.name != '-'" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json index 631673fe3515..ce695210bef1 100644 --- a/filebeat/module/traefik/access/test/test.log-expected.json +++ b/filebeat/module/traefik/access/test/test.log-expected.json @@ -8,6 +8,7 @@ "event.duration": 2000000, "event.kind": "event", "event.module": "traefik", + "event.original": "192.168.33.1 - - [02/Oct/2017:20:22:07 +0000] \"GET /ui/favicons/favicon-16x16.png HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 262 \"Host-host-1\" \"http://172.19.0.3:5601\" 2ms", "event.outcome": "success", "event.type": [ "access" @@ -30,7 +31,9 @@ "traefik.access.frontend_name": "Host-host-1", "traefik.access.request_count": 262, "traefik.access.user_identifier": "-", + "url.extension": "png", "url.original": "/ui/favicons/favicon-16x16.png", + "url.path": "/ui/favicons/favicon-16x16.png", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -47,6 +50,7 @@ "event.duration": 3000000, "event.kind": "event", "event.module": "traefik", + "event.original": "85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] \"GET /ui/favicons/favicon.ico HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 271 \"Host-host1\" \"http://172.19.0.3:5601\" 3ms", "event.outcome": "success", "event.type": [ "access" @@ -79,7 +83,9 @@ "traefik.access.frontend_name": "Host-host1", "traefik.access.request_count": 271, "traefik.access.user_identifier": "-", + "url.extension": "ico", "url.original": "/ui/favicons/favicon.ico", + "url.path": "/ui/favicons/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -96,6 +102,7 @@ "event.duration": 247000000, "event.kind": "event", "event.module": "traefik", + "event.original": "70.29.80.15 - - [28/Feb/2018:17:30:33 +0000] \"GET /en/ HTTP/2.0\" 200 2814 - \"Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1\" 13 \"Host-host1-com-0\" \"http://172.19.0.6:14008\" 247ms", "event.outcome": "success", "event.type": [ "access" @@ -128,6 +135,7 @@ "traefik.access.request_count": 13, "traefik.access.user_identifier": "-", "url.original": "/en/", + "url.path": "/en/", "user.name": "-", "user_agent.device.name": "iPhone", "user_agent.name": "Mobile Safari", @@ -146,6 +154,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "traefik", + "event.original": "::1 - - [29/Nov/2018:15:03:51 +0000] \"GET / HTTP/1.1\" 404 19 \"-\" \"curl/7.62.0\" 10 \"backend not found\" \"/\" 0ms", "event.outcome": "failure", "event.type": [ "access" @@ -169,6 +178,7 @@ "traefik.access.request_count": 10, "traefik.access.user_identifier": "-", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -184,6 +194,7 @@ "event.duration": 13000000, "event.kind": "event", "event.module": "traefik", + "event.original": "94.254.131.115 - - [19/Jan/2018:10:01:02 +0000] \"GET /assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo HTTP/1.1\" 200 85 - \"Android\" 623112 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.9:4140\" 13ms", "event.outcome": "success", "event.type": [ "access" @@ -216,6 +227,8 @@ "traefik.access.request_count": 623112, "traefik.access.user_identifier": "-", "url.original": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", + "url.path": "/assets/52f8f2e711d235d76044799e/owners", + "url.query": "oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", "user.name": "-", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Other", @@ -231,6 +244,7 @@ "event.duration": 8000000, "event.kind": "event", "event.module": "traefik", + "event.original": "89.64.35.193 - - [19/Jan/2018:10:01:02 +0000] \"GET /marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM HTTP/1.1\" 200 150 - \"Android\" 623114 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.6:4140\" 8ms", "event.outcome": "success", "event.type": [ "access" @@ -263,6 +277,8 @@ "traefik.access.request_count": 623114, "traefik.access.user_identifier": "-", "url.original": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", + "url.path": "/marketplace/tax", + "url.query": "oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", "user.name": "-", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Other", @@ -277,6 +293,7 @@ "event.dataset": "traefik.access", "event.kind": "event", "event.module": "traefik", + "event.original": "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326", "event.outcome": "success", "event.type": [ "access" @@ -298,7 +315,9 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "traefik.access.user_identifier": "-", + "url.extension": "gif", "url.original": "/apache_pb.gif", + "url.path": "/apache_pb.gif", "user.name": "frank" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index 4dea7d027c6f..8cfaa7109c01 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -34,18 +34,19 @@ processors: field: aws.s3access.request_uri ignore_failure: true patterns: - - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' + - '%{NOTSPACE:http.request.method} %{NOTSPACE:_temp_.uri_orig} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' - # - # Best-effort parse of url.original in the form /path?query" - # - grok: - field: url.original + field: aws.s3access.host_header ignore_failure: true patterns: - - '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$' - pattern_definitions: - ABS_PATH: '/[^?]*' + - '(s3[a-z-]*).%{NOTSPACE:cloud.region}.%{WORD}.%{WORD}' + if: ctx.aws?.s3access?.host_header != null + + - uri_parts: + field: _temp_.uri_orig + ignore_failure: true + if: ctx?._temp_?.uri_orig != null - append: if: "ctx?.aws?.s3access?.bucket_owner != null" field: related.user @@ -188,6 +189,31 @@ processors: field: _temp_ ignore_missing: true + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + on_failure: - set: field: "error.message" diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log index f96091a76798..72b45f88ef24 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log @@ -3,4 +3,4 @@ 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json index aa9d1bf6938a..37345ff30fc9 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json @@ -22,6 +22,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -96,6 +97,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -171,6 +173,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.BUCKET", "event.category": "web", "event.dataset": "aws.s3access", @@ -245,6 +248,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -317,6 +321,7 @@ "client.ip": "77.227.156.41", "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", "cloud.provider": "aws", + "cloud.region": "eu-central-1", "event.action": "BATCH.DELETE.OBJECT", "event.category": "web", "event.dataset": "aws.s3access", @@ -360,7 +365,7 @@ "aws.s3access.bucket": "test-s3-ks", "aws.s3access.bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", "aws.s3access.cipher_suite": "ECDHE-RSA-AES128-SHA", - "aws.s3access.host_header": "s3-ap-southeast-1.amazonaws.com", + "aws.s3access.host_header": "s3.ap-southeast-1.amazonaws.com", "aws.s3access.host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=", "aws.s3access.http_status": 204, "aws.s3access.key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png", @@ -375,13 +380,14 @@ "client.ip": "174.29.206.152", "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "BATCH.DELETE.OBJECT", "event.category": "web", "event.dataset": "aws.s3access", "event.id": "6CE38F1312D32BDD", "event.kind": "event", "event.module": "aws", - "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2", + "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "event.outcome": "success", "event.type": [ "access" diff --git a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json index f6ca4d4edf36..94866cfe505c 100644 --- a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json @@ -305,6 +305,7 @@ "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.1", "tls.version_protocol": "tls", + "url.extension": "pdf", "url.original": "/awsexamplebucket/s3-dg.pdf", "url.path": "/awsexamplebucket/s3-dg.pdf", "user_agent.device.name": "Other", @@ -368,6 +369,7 @@ "tls.version": "1.1", "tls.version_protocol": "tls", "url.original": "*", + "url.path": "*", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "S3Console/0.4" diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 35c5882513c8..7f641d9becdb 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -2499,7 +2499,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/", + "url.path": "/", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2543,7 +2546,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/IOFUHSIU98[0]" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/IOFUHSIU98[0]", + "url.path": "/IOFUHSIU98[0]", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2587,7 +2593,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23", + "url.path": "/some/longer/url-asd-er9789870[0]_=23", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2631,7 +2640,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/", + "url.path": "/", + "url.scheme": "http" }, { "cisco.asa.connection_id": "2751765169", diff --git a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json index 6a04d9e08e43..7a8c5d428483 100644 --- a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json @@ -3516,7 +3516,8 @@ "cisco-asa", "forwarded" ], - "url.original": "/app" + "url.original": "/app", + "url.path": "/app" }, { "@timestamp": "2009-11-16T14:12:36.000-02:00", @@ -3558,7 +3559,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://example.com" + "url.domain": "example.com", + "url.original": "http://example.com", + "url.path": "", + "url.scheme": "http" }, { "@timestamp": "2009-11-16T14:12:37.000-02:00", @@ -3602,7 +3606,11 @@ "cisco-asa", "forwarded" ], - "url.original": "http://www.example.net/images/favicon.ico" + "url.domain": "www.example.net", + "url.extension": "ico", + "url.original": "http://www.example.net/images/favicon.ico", + "url.path": "/images/favicon.ico", + "url.scheme": "http" }, { "@timestamp": "2021-01-13T19:12:37.000-02:00", diff --git a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json index 865c5a2764e5..a287b33f2525 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json @@ -3502,7 +3502,8 @@ "cisco-ftd", "forwarded" ], - "url.original": "/app" + "url.original": "/app", + "url.path": "/app" }, { "@timestamp": "2009-11-16T14:12:36.000-02:00", @@ -3543,7 +3544,10 @@ "cisco-ftd", "forwarded" ], - "url.original": "http://example.com" + "url.domain": "example.com", + "url.original": "http://example.com", + "url.path": "", + "url.scheme": "http" }, { "@timestamp": "2009-11-16T14:12:37.000-02:00", @@ -3586,6 +3590,10 @@ "cisco-ftd", "forwarded" ], - "url.original": "http://www.example.net/images/favicon.ico" + "url.domain": "www.example.net", + "url.extension": "ico", + "url.original": "http://www.example.net/images/favicon.ico", + "url.path": "/images/favicon.ico", + "url.scheme": "http" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json index be1d11ad0afc..f65ccf20d71b 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json @@ -609,7 +609,10 @@ "forwarded" ], "url.domain": "eu-central-1.ec2.archive.ubuntu.com", + "url.extension": "deb", "url.original": "http://eu-central-1.ec2.archive.ubuntu.com/ubuntu/pool/main/m/manpages/manpages-dev_4.15-1_all.deb", + "url.path": "/ubuntu/pool/main/m/manpages/manpages-dev_4.15-1_all.deb", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "Debian APT-HTTP/1.3 (1.6.11)" @@ -822,7 +825,10 @@ "forwarded" ], "url.domain": "www.eicar.org", + "url.extension": "zip", "url.original": "http://www.eicar.org/download/eicar_com.zip", + "url.path": "/download/eicar_com.zip", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "curl/7.58.0" @@ -1014,8 +1020,15 @@ "cisco-ftd", "forwarded" ], - "url.domain": "10.0.100.30:8000", + "url.domain": [ + "10.0.100.30", + "10.0.100.30:8000" + ], + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "curl/7.58.0" diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json index 135a29792105..367c559c2e67 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json @@ -69,7 +69,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "exe", "url.original": "http://10.0.100.30:8000/exploit.exe", + "url.path": "/exploit.exe", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -143,7 +148,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "exe", "url.original": "http://10.0.100.30:8000/exploit.exe", + "url.path": "/exploit.exe", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -217,7 +227,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "com", "url.original": "http://10.0.100.30:8000/eicar.com", + "url.path": "/eicar.com", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -291,7 +306,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "txt", "url.original": "http://10.0.100.30:8000/eicar.com.txt", + "url.path": "/eicar.com.txt", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -374,7 +394,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -457,7 +482,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -544,7 +574,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -640,7 +675,11 @@ "cisco-ftd", "forwarded" ], + "url.domain": "www.eicar.org", + "url.extension": "zip", "url.original": "http://www.eicar.org/download/eicar_com.zip", + "url.path": "/download/eicar_com.zip", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -726,7 +765,10 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", "url.original": "http://10.0.100.30/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.path": "/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -823,7 +865,10 @@ "cisco-ftd", "forwarded" ], + "url.domain": "18.197.225.123", "url.original": "http://18.197.225.123/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.path": "/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" } diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json index b23b07b6ac23..e98e7fc90cdf 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json @@ -117,8 +117,14 @@ "cisco-ftd", "forwarded" ], - "url.domain": "eyedropper-color-pick.info", + "url.domain": [ + "bad-malwaresite-grr.info", + "eyedropper-color-pick.info" + ], + "url.extension": "ico", "url.original": "http://bad-malwaresite-grr.info/favicon.ico", + "url.path": "/favicon.ico", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36" diff --git a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml index a18507659aa5..b80d31afad41 100644 --- a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml @@ -10,6 +10,25 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json index 32182b7f2576..826a4f921947 100644 --- a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json @@ -17,8 +17,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.15.44.253", - "10.193.124.51" + "10.193.124.51", + "10.15.44.253" ], "rsa.internal.event_desc": "olaborissecurity_event tur", "rsa.internal.messageid": "security_event", @@ -40,7 +40,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.org/odoco/ria.jpg?ritin=uredolor#tatemac" + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "tatemac", + "url.original": "https://example.org/odoco/ria.jpg?ritin=uredolor#tatemac", + "url.path": "/odoco/ria.jpg", + "url.query": "ritin=uredolor", + "url.scheme": "https" }, { "@timestamp": "2016-02-12T13:12:33.000Z", @@ -60,8 +66,8 @@ "appliance" ], "related.ip": [ - "10.15.16.212", - "10.102.218.31" + "10.102.218.31", + "10.15.16.212" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -214,7 +220,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://api.example.org/liqu/lorem.gif?ueipsaqu=uidolore#niamqu" + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "niamqu", + "url.original": "https://api.example.org/liqu/lorem.gif?ueipsaqu=uidolore#niamqu", + "url.path": "/liqu/lorem.gif", + "url.query": "ueipsaqu=uidolore", + "url.scheme": "https" }, { "@timestamp": "2016-04-24T00:25:25.000Z", @@ -377,7 +389,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.net/sun/essecill.html?saute=vel#quu" + "url.domain": "www5.example.net", + "url.extension": "html", + "url.fragment": "quu", + "url.original": "https://www5.example.net/sun/essecill.html?saute=vel#quu", + "url.path": "/sun/essecill.html", + "url.query": "saute=vel", + "url.scheme": "https" }, { "@timestamp": "2016-07-04T11:38:16.000Z", @@ -569,8 +587,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.187.77.245", - "10.88.231.224" + "10.88.231.224", + "10.187.77.245" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -618,7 +636,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.com/quae/maccusa.htm?rQuisau=idex#xerci" + "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "xerci", + "url.original": "https://internal.example.com/quae/maccusa.htm?rQuisau=idex#xerci", + "url.path": "/quae/maccusa.htm", + "url.query": "rQuisau=idex", + "url.scheme": "https" }, { "@timestamp": "2016-10-12T12:56:16.000Z", @@ -710,8 +734,8 @@ "appliance" ], "related.ip": [ - "10.182.178.217", - "10.63.194.87" + "10.63.194.87", + "10.182.178.217" ], "rsa.counters.dclass_r1": "fdeFi", "rsa.internal.messageid": "events", @@ -730,7 +754,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.net/ris/uamqu.txt?liqui=quioffi#uptate" + "url.domain": "internal.example.net", + "url.extension": "txt", + "url.fragment": "uptate", + "url.original": "https://internal.example.net/ris/uamqu.txt?liqui=quioffi#uptate", + "url.path": "/ris/uamqu.txt", + "url.query": "liqui=quioffi", + "url.scheme": "https" }, { "@timestamp": "2016-11-24T10:03:59.000Z", @@ -773,7 +803,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.com/icabo/itatio.jpg?eleum=sintoc#volupt" + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "volupt", + "url.original": "https://mail.example.com/icabo/itatio.jpg?eleum=sintoc#volupt", + "url.path": "/icabo/itatio.jpg", + "url.query": "eleum=sintoc", + "url.scheme": "https" }, { "@timestamp": "2016-12-08T17:06:33.000Z", @@ -926,8 +962,8 @@ "appliance" ], "related.ip": [ - "10.93.68.231", - "10.135.217.12" + "10.135.217.12", + "10.93.68.231" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -993,8 +1029,8 @@ "appliance" ], "related.ip": [ - "10.247.30.212", - "10.66.89.5" + "10.66.89.5", + "10.247.30.212" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1058,8 +1094,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.221.102.245", - "10.173.136.186" + "10.173.136.186", + "10.221.102.245" ], "rsa.internal.event_desc": "idestlab", "rsa.internal.messageid": "security_event", @@ -1097,8 +1133,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.54.37.86", - "10.58.64.108" + "10.58.64.108", + "10.54.37.86" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1216,8 +1252,8 @@ "appliance" ], "related.ip": [ - "10.28.144.180", - "10.148.124.84" + "10.148.124.84", + "10.28.144.180" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -1277,7 +1313,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.net/ites/isetq.gif?nisiut=tur#avolupt" + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "avolupt", + "url.original": "https://example.net/ites/isetq.gif?nisiut=tur#avolupt", + "url.path": "/ites/isetq.gif", + "url.query": "nisiut=tur", + "url.scheme": "https" }, { "@timestamp": "2017-06-26T19:42:33.000Z", @@ -1309,7 +1351,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.org/ueipsa/tae.html?eriti=atcupi#corpori" + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "corpori", + "url.original": "https://www.example.org/ueipsa/tae.html?eriti=atcupi#corpori", + "url.path": "/ueipsa/tae.html", + "url.query": "eriti=atcupi", + "url.scheme": "https" }, { "@timestamp": "2017-07-11T02:45:07.000Z", @@ -1354,8 +1402,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.150.245.88", - "10.242.77.170" + "10.242.77.170", + "10.150.245.88" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1391,8 +1439,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.180.195.43", - "10.247.139.239" + "10.247.139.239", + "10.180.195.43" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1453,8 +1501,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.94.6.140", - "10.147.15.213" + "10.147.15.213", + "10.94.6.140" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1554,7 +1602,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/irured/illumqui.txt?tionula=ritqu#ecatcupi" + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "ecatcupi", + "url.original": "https://www.example.net/irured/illumqui.txt?tionula=ritqu#ecatcupi", + "url.path": "/irured/illumqui.txt", + "url.query": "tionula=ritqu", + "url.scheme": "https" }, { "@timestamp": "2017-10-19T04:03:07.000Z", @@ -1735,7 +1789,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.net/uat/lupta.html?uptassit=ncidi#tlabori" + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "tlabori", + "url.original": "https://mail.example.net/uat/lupta.html?uptassit=ncidi#tlabori", + "url.path": "/uat/lupta.html", + "url.query": "uptassit=ncidi", + "url.scheme": "https" }, { "@timestamp": "2018-01-12T22:18:32.000Z", @@ -1919,8 +1979,8 @@ "appliance" ], "related.ip": [ - "10.39.172.93", - "10.83.131.245" + "10.83.131.245", + "10.39.172.93" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1958,8 +2018,8 @@ "appliance" ], "related.ip": [ - "10.201.168.116", - "10.86.188.179" + "10.86.188.179", + "10.201.168.116" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -1996,8 +2056,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.122.204.151", - "10.148.211.222" + "10.148.211.222", + "10.122.204.151" ], "rsa.internal.event_desc": "umexercisecurity_event duntut", "rsa.internal.messageid": "security_event", @@ -2019,7 +2079,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.com/prehend/eufug.htm?eufug=est#civelits" + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "civelits", + "url.original": "https://mail.example.com/prehend/eufug.htm?eufug=est#civelits", + "url.path": "/prehend/eufug.htm", + "url.query": "eufug=est", + "url.scheme": "https" }, { "@timestamp": "2018-05-07T06:39:06.000Z", @@ -2069,8 +2135,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.120.4.9", - "10.97.46.16" + "10.97.46.16", + "10.120.4.9" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2105,8 +2171,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.165.173.162", - "10.171.206.139" + "10.171.206.139", + "10.165.173.162" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2143,8 +2209,8 @@ "uames4985.mail.localdomain" ], "related.ip": [ - "10.144.57.239", - "10.150.163.151" + "10.150.163.151", + "10.144.57.239" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2317,8 +2383,8 @@ "appliance" ], "related.ip": [ - "10.103.49.129", - "10.2.110.73" + "10.2.110.73", + "10.103.49.129" ], "rsa.counters.dclass_r1": "orumS", "rsa.internal.messageid": "events", @@ -2337,7 +2403,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/doconse/eni.html?mSec=smoditem#tatisetq" + "url.domain": "www.example.net", + "url.extension": "html", + "url.fragment": "tatisetq", + "url.original": "https://www.example.net/doconse/eni.html?mSec=smoditem#tatisetq", + "url.path": "/doconse/eni.html", + "url.query": "mSec=smoditem", + "url.scheme": "https" }, { "@timestamp": "2018-09-12T22:02:15.000Z", @@ -2380,7 +2452,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.com/seq/moll.htm?sunt=dquianon#urExc" + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "urExc", + "url.original": "https://www.example.com/seq/moll.htm?sunt=dquianon#urExc", + "url.path": "/seq/moll.htm", + "url.query": "sunt=dquianon", + "url.scheme": "https" }, { "@timestamp": "2018-09-27T05:04:49.000Z", @@ -2402,8 +2480,8 @@ "lors2232.api.example" ], "related.ip": [ - "10.105.136.146", - "10.46.217.155" + "10.46.217.155", + "10.105.136.146" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2535,7 +2613,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/sectetur/edquian.html?turQuis=taevi#uames" + "url.domain": "www.example.net", + "url.extension": "html", + "url.fragment": "uames", + "url.original": "https://www.example.net/sectetur/edquian.html?turQuis=taevi#uames", + "url.path": "/sectetur/edquian.html", + "url.query": "turQuis=taevi", + "url.scheme": "https" }, { "@timestamp": "2018-11-23T09:15:06.000Z", @@ -2558,8 +2642,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.34.62.190", - "10.246.152.72" + "10.246.152.72", + "10.34.62.190" ], "rsa.internal.event_desc": "Nem", "rsa.internal.messageid": "security_event", @@ -2719,7 +2803,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.com/totamrem/aliqu.htm?sBonorum=moenimi#lor" + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "lor", + "url.original": "https://www.example.com/totamrem/aliqu.htm?sBonorum=moenimi#lor", + "url.path": "/totamrem/aliqu.htm", + "url.query": "sBonorum=moenimi", + "url.scheme": "https" }, { "@timestamp": "2019-02-17T03:30:32.000Z", @@ -3117,8 +3207,8 @@ "appliance" ], "related.ip": [ - "10.247.118.132", - "10.254.96.130" + "10.254.96.130", + "10.247.118.132" ], "rsa.counters.dclass_r1": "ectet", "rsa.internal.messageid": "events", @@ -3137,7 +3227,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.org/eturadi/umS.txt?mSecti=henderi#taevitae" + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "taevitae", + "url.original": "https://www5.example.org/eturadi/umS.txt?mSecti=henderi#taevitae", + "url.path": "/eturadi/umS.txt", + "url.query": "mSecti=henderi", + "url.scheme": "https" }, { "@timestamp": "2019-08-07T16:01:23.000Z", @@ -3160,8 +3256,8 @@ "appliance" ], "related.ip": [ - "10.101.13.122", - "10.200.98.243" + "10.200.98.243", + "10.101.13.122" ], "rsa.counters.dclass_r1": "uteirur", "rsa.internal.messageid": "events", @@ -3180,7 +3276,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.org/fdeFi/ratv.htm?sequatu=tiumtot#tate" + "url.domain": "internal.example.org", + "url.extension": "htm", + "url.fragment": "tate", + "url.original": "https://internal.example.org/fdeFi/ratv.htm?sequatu=tiumtot#tate", + "url.path": "/fdeFi/ratv.htm", + "url.query": "sequatu=tiumtot", + "url.scheme": "https" }, { "@timestamp": "2019-08-21T23:03:57.000Z", @@ -3213,7 +3315,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.net/uov/pariat.htm?litsed=lumd#tiaec" + "url.domain": "www5.example.net", + "url.extension": "htm", + "url.fragment": "tiaec", + "url.original": "https://www5.example.net/uov/pariat.htm?litsed=lumd#tiaec", + "url.path": "/uov/pariat.htm", + "url.query": "litsed=lumd", + "url.scheme": "https" }, { "@timestamp": "2019-09-05T06:06:31.000Z", @@ -3299,8 +3407,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.137.166.97", - "10.162.202.14" + "10.162.202.14", + "10.137.166.97" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -3347,7 +3455,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.com/illoinve/etcon.htm?nevolup=erspici#itinvolu" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "itinvolu", + "url.original": "https://www5.example.com/illoinve/etcon.htm?nevolup=erspici#itinvolu", + "url.path": "/illoinve/etcon.htm", + "url.query": "nevolup=erspici", + "url.scheme": "https" }, { "@timestamp": "2019-11-01T10:16:48.000Z", @@ -3446,7 +3560,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.com/qui/itse.gif?orsitame=tasn#exeaco" + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "exeaco", + "url.original": "https://example.com/qui/itse.gif?orsitame=tasn#exeaco", + "url.path": "/qui/itse.gif", + "url.query": "orsitame=tasn", + "url.scheme": "https" }, { "@timestamp": "2019-12-14T07:24:31.000Z", diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index b016a5c3fd06..19775fe0be0b 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1646,6 +1646,23 @@ processors: ) ) + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + # # Populate ECS event.code # @@ -1882,6 +1899,30 @@ processors: value: "{{source.domain}}" if: ctx.source?.domain != null && ctx.source?.domain != '' allow_duplicates: false + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: # Copy any fields under _temp_.cisco to its final destination. Those can help # with diagnosing the failure. diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml index 2a602ff23317..b659b2fd5edd 100644 --- a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -120,6 +120,11 @@ processors: value: firewall if: ctx?.log?.file?.path.contains('cloudfirewalllogs') +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null + # Identifies is a field that includes any sort of username, device or other asset that is included in the request. # Converting this to an array to make it easier to use in searches and visualizations - split: @@ -198,6 +203,8 @@ processors: - "(?:%{IP:destination.ip}|%{GREEDYDATA:destination.domain})" ignore_failure: true +- community_id: + ignore_missing: true ###################### # Related ECS Fields # ###################### diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json index 65aabab5a88d..6cedb19cced9 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json @@ -16,6 +16,7 @@ "input.type": "log", "log.offset": 0, "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "network.community_id": "1:9s/mgn3WChavy6kpCk25Ed4VCcU=", "network.direction": "outbound", "network.transport": "1", "observer.product": "Umbrella", @@ -52,6 +53,7 @@ "input.type": "log", "log.offset": 128, "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "network.community_id": "1:9s/mgn3WChavy6kpCk25Ed4VCcU=", "network.direction": "inbound", "network.transport": "1", "observer.product": "Umbrella", diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json index fd474d2d0293..4f0b9552eb48 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json @@ -34,7 +34,10 @@ "source.address": "192.168.1.1", "source.ip": "192.168.1.1", "source.nat.ip": "1.1.1.1", - "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "url.domain": "elastic.co", + "url.original": "https://elastic.co/blog/ext_id=Anyclip", + "url.path": "/blog/ext_id=Anyclip", + "url.scheme": "https", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" }, { @@ -72,7 +75,10 @@ "source.address": "192.168.1.1", "source.ip": "192.168.1.1", "source.nat.ip": "1.1.1.1", - "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "url.domain": "elastic.co", + "url.original": "https://elastic.co/blog/ext_id=Anyclip", + "url.path": "/blog/ext_id=Anyclip", + "url.scheme": "https", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" }, { @@ -109,7 +115,11 @@ "source.address": "ActiveDirectoryUserName,ADSite,Network", "source.domain": "ActiveDirectoryUserName,ADSite,Network", "source.nat.ip": "192.192.192.135", - "url.full": "http://google.com/the.js", + "url.domain": "google.com", + "url.extension": "js", + "url.original": "http://google.com/the.js", + "url.path": "/the.js", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml index 1c939d65a22f..ae772c9b6a33 100644 --- a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json index 0a259d4811c6..f2227c07cb11 100644 --- a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json @@ -24,10 +24,10 @@ "tatemac3541.api.corp" ], "related.ip": [ + "10.165.201.71", "10.208.121.85", - "10.11.196.142", "10.228.193.207", - "10.165.201.71" + "10.11.196.142" ], "related.user": [ "billoi" @@ -92,9 +92,9 @@ "enatus2114.mail.home" ], "related.ip": [ + "10.92.202.200", "10.51.132.10", "10.162.9.235", - "10.92.202.200", "10.94.67.230" ], "related.user": [ @@ -161,9 +161,9 @@ ], "related.ip": [ "10.82.56.117", + "10.122.116.161", "10.209.155.149", - "10.191.68.244", - "10.122.116.161" + "10.191.68.244" ], "related.user": [ "seq" @@ -228,8 +228,8 @@ ], "related.ip": [ "10.50.112.141", - "10.131.233.27", "10.12.44.169", + "10.131.233.27", "10.202.66.28" ], "related.user": [ @@ -295,10 +295,10 @@ "emquiavo452.internal.localhost" ], "related.ip": [ - "10.159.182.171", "10.96.35.212", - "10.206.197.113", - "10.151.111.38" + "10.151.111.38", + "10.159.182.171", + "10.206.197.113" ], "related.user": [ "mol" @@ -363,10 +363,10 @@ "sun1403.www.invalid" ], "related.ip": [ - "10.213.113.28", - "10.169.144.147", "10.126.177.162", - "10.89.163.114" + "10.213.113.28", + "10.89.163.114", + "10.169.144.147" ], "related.user": [ "ist" @@ -430,9 +430,9 @@ "ittenbyC7838.api.localdomain" ], "related.ip": [ - "10.18.124.28", "10.101.223.43", "10.103.107.47", + "10.18.124.28", "10.146.88.52" ], "related.user": [ @@ -498,10 +498,10 @@ "ume465.corp" ], "related.ip": [ - "10.150.220.75", - "10.110.99.17", "10.189.109.245", - "10.69.57.206" + "10.110.99.17", + "10.69.57.206", + "10.150.220.75" ], "related.user": [ "onse" @@ -565,10 +565,10 @@ "iciatisu1463.www5.localdomain" ], "related.ip": [ + "10.121.219.204", "10.153.136.222", - "10.19.194.101", "10.199.34.241", - "10.121.219.204" + "10.19.194.101" ], "related.user": [ "temveleu" @@ -632,10 +632,10 @@ "aliqu6801.api.localdomain" ], "related.ip": [ + "10.46.27.57", "10.57.103.192", "10.64.141.105", - "10.182.199.231", - "10.46.27.57" + "10.182.199.231" ], "related.user": [ "ice" @@ -699,10 +699,10 @@ "itame189.domain" ], "related.ip": [ - "10.160.210.31", - "10.164.6.207", "10.3.134.237", - "10.32.67.231" + "10.32.67.231", + "10.164.6.207", + "10.160.210.31" ], "related.user": [ "pic" @@ -768,9 +768,9 @@ ], "related.ip": [ "10.235.101.253", - "10.201.6.10", "10.42.138.192", - "10.182.178.217" + "10.182.178.217", + "10.201.6.10" ], "related.user": [ "giatnu" @@ -836,9 +836,9 @@ ], "related.ip": [ "10.151.161.70", + "10.22.102.198", "10.194.247.171", - "10.86.101.235", - "10.22.102.198" + "10.86.101.235" ], "related.user": [ "nse" @@ -970,9 +970,9 @@ "smo7167.www.test" ], "related.ip": [ - "10.182.191.174", "10.214.249.164", "10.99.249.210", + "10.182.191.174", "10.81.26.208" ], "related.user": [ @@ -1037,10 +1037,10 @@ "sauteiru4554.api.domain" ], "related.ip": [ - "10.101.226.128", "10.220.5.143", - "10.88.101.53", - "10.201.238.90" + "10.201.238.90", + "10.101.226.128", + "10.88.101.53" ], "related.user": [ "porro" @@ -1104,9 +1104,9 @@ "untut4046.internal.domain" ], "related.ip": [ + "10.217.150.196", "10.30.133.66", "10.157.18.252", - "10.217.150.196", "10.243.218.215" ], "related.user": [ @@ -1171,10 +1171,10 @@ "quid3147.mail.home" ], "related.ip": [ - "10.181.133.187", - "10.148.161.250", "10.66.181.6", - "10.167.227.44" + "10.181.133.187", + "10.167.227.44", + "10.148.161.250" ], "related.user": [ "adipisc" @@ -1239,9 +1239,9 @@ "umdolo1029.mail.localhost" ], "related.ip": [ - "10.74.11.43", "10.84.163.178", "10.107.9.163", + "10.74.11.43", "10.54.17.32" ], "related.user": [ @@ -1306,10 +1306,10 @@ "lorsita2019.internal.home" ], "related.ip": [ + "10.184.73.211", "10.192.229.221", "10.112.32.213", - "10.230.129.252", - "10.184.73.211" + "10.230.129.252" ], "related.user": [ "odi" @@ -1375,9 +1375,9 @@ ], "related.ip": [ "10.198.213.189", - "10.161.148.64", + "10.7.200.140", "10.199.216.143", - "10.7.200.140" + "10.161.148.64" ], "related.user": [ "ccaeca" @@ -1441,10 +1441,10 @@ "mex2054.mail.corp" ], "related.ip": [ + "10.65.232.27", "10.206.96.56", "10.128.157.27", - "10.22.187.69", - "10.65.232.27" + "10.22.187.69" ], "related.user": [ "uaeab" @@ -1508,10 +1508,10 @@ "avolupt7576.api.corp" ], "related.ip": [ - "10.71.114.14", - "10.194.210.62", "10.68.253.120", - "10.183.130.225" + "10.71.114.14", + "10.183.130.225", + "10.194.210.62" ], "related.user": [ "admin" @@ -1577,8 +1577,8 @@ ], "related.ip": [ "10.107.45.175", - "10.31.177.226", "10.47.255.237", + "10.31.177.226", "10.45.253.103" ], "related.user": [ @@ -1644,10 +1644,10 @@ "nsequat1971.internal.invalid" ], "related.ip": [ - "10.55.105.113", + "10.213.94.135", "10.225.212.189", - "10.44.58.106", - "10.213.94.135" + "10.55.105.113", + "10.44.58.106" ], "related.user": [ "dquia" @@ -1711,10 +1711,10 @@ "ectiono2241.lan" ], "related.ip": [ - "10.163.209.70", "10.255.74.136", - "10.69.161.78", - "10.2.114.9" + "10.2.114.9", + "10.163.209.70", + "10.69.161.78" ], "related.user": [ "olabor" @@ -1778,10 +1778,10 @@ "umetMal1664.mail.lan" ], "related.ip": [ - "10.184.59.148", + "10.252.102.110", "10.12.129.137", "10.46.115.216", - "10.252.102.110" + "10.184.59.148" ], "related.user": [ "perspici" @@ -1846,10 +1846,10 @@ "derit5270.mail.local" ], "related.ip": [ + "10.199.194.79", "10.81.184.7", "10.155.204.243", - "10.105.52.140", - "10.199.194.79" + "10.105.52.140" ], "related.user": [ "eetd" @@ -1914,10 +1914,10 @@ "orisni5238.mail.lan" ], "related.ip": [ - "10.18.226.72", + "10.251.231.142", "10.177.238.45", "10.110.2.166", - "10.251.231.142" + "10.18.226.72" ], "related.user": [ "taliqui" @@ -1982,9 +1982,9 @@ "iutali7297.www.domain" ], "related.ip": [ - "10.190.122.27", - "10.100.199.226", "10.99.202.229", + "10.100.199.226", + "10.190.122.27", "10.192.98.247" ], "related.user": [ @@ -2050,10 +2050,10 @@ "orumw5960.www5.home" ], "related.ip": [ - "10.172.154.97", "10.248.111.207", - "10.37.193.70", - "10.162.97.197" + "10.172.154.97", + "10.162.97.197", + "10.37.193.70" ], "related.user": [ "culpaq" @@ -2117,10 +2117,10 @@ "oinv5493.internal.domain" ], "related.ip": [ - "10.171.221.230", - "10.45.35.180", "10.36.63.31", - "10.222.165.250" + "10.171.221.230", + "10.222.165.250", + "10.45.35.180" ], "related.user": [ "otamr" @@ -2184,9 +2184,9 @@ "tnonproi195.api.home" ], "related.ip": [ - "10.199.127.211", - "10.83.238.145", "10.238.4.219", + "10.83.238.145", + "10.199.127.211", "10.1.171.61" ], "related.user": [ @@ -2251,9 +2251,9 @@ "edictasu5362.internal.localhost" ], "related.ip": [ - "10.44.226.104", "10.170.252.219", "10.65.141.244", + "10.44.226.104", "10.74.213.42" ], "related.user": [ @@ -2318,9 +2318,9 @@ "uido492.www5.home" ], "related.ip": [ - "10.225.255.211", "10.225.141.172", "10.183.223.149", + "10.225.255.211", "10.180.48.221" ], "related.user": [ @@ -2385,9 +2385,9 @@ "redo6311.api.invalid" ], "related.ip": [ + "10.169.123.103", "10.176.64.28", "10.97.138.181", - "10.169.123.103", "10.205.174.181" ], "related.user": [ @@ -2453,10 +2453,10 @@ "dolorem1698.www.domain" ], "related.ip": [ - "10.53.101.131", - "10.204.4.40", "10.75.120.11", - "10.169.101.161" + "10.204.4.40", + "10.169.101.161", + "10.53.101.131" ], "related.user": [ "tquo" @@ -2523,8 +2523,8 @@ "related.ip": [ "10.28.51.219", "10.6.222.112", - "10.87.120.87", - "10.156.117.169" + "10.156.117.169", + "10.87.120.87" ], "related.user": [ "onsequu" @@ -2588,10 +2588,10 @@ "arc2412.mail.lan" ], "related.ip": [ - "10.57.89.155", - "10.4.126.103", + "10.247.44.59", "10.253.167.17", - "10.247.44.59" + "10.4.126.103", + "10.57.89.155" ], "related.user": [ "ntorever" @@ -2655,10 +2655,10 @@ "olorsi2746.internal.localhost" ], "related.ip": [ - "10.143.183.208", - "10.36.69.125", "10.15.240.220", - "10.248.206.210" + "10.248.206.210", + "10.36.69.125", + "10.143.183.208" ], "related.user": [ "met" @@ -2723,10 +2723,10 @@ "edqu2208.www.localhost" ], "related.ip": [ + "10.34.133.2", "10.69.170.107", - "10.6.32.7", "10.142.186.43", - "10.34.133.2" + "10.6.32.7" ], "related.user": [ "ipitlabo" @@ -2792,9 +2792,9 @@ ], "related.ip": [ "10.142.22.24", - "10.59.103.10", "10.170.165.164", - "10.121.153.197" + "10.121.153.197", + "10.59.103.10" ], "related.user": [ "borumSec" @@ -2859,9 +2859,9 @@ "sis3986.internal.lan" ], "related.ip": [ - "10.247.114.30", - "10.133.10.122", "10.19.99.129", + "10.133.10.122", + "10.247.114.30", "10.176.83.7" ], "related.user": [ @@ -2928,9 +2928,9 @@ ], "related.ip": [ "10.8.29.219", - "10.70.7.23", "10.40.177.138", - "10.64.139.17" + "10.64.139.17", + "10.70.7.23" ], "related.user": [ "rep" @@ -2995,9 +2995,9 @@ ], "related.ip": [ "10.67.173.228", + "10.180.62.222", "10.2.189.20", - "10.67.221.220", - "10.180.62.222" + "10.67.221.220" ], "related.user": [ "uptasnul" @@ -3062,10 +3062,10 @@ "uian521.www.example" ], "related.ip": [ + "10.196.176.243", "10.147.127.181", - "10.209.52.47", "10.56.134.118", - "10.196.176.243" + "10.209.52.47" ], "related.user": [ "tasu" @@ -3130,8 +3130,8 @@ ], "related.ip": [ "10.85.13.237", - "10.231.18.90", "10.248.140.59", + "10.231.18.90", "10.226.24.84" ], "related.user": [ @@ -3265,10 +3265,10 @@ "mexer3864.api.corp" ], "related.ip": [ - "10.255.145.22", "10.73.84.95", - "10.230.38.148", - "10.98.154.146" + "10.98.154.146", + "10.255.145.22", + "10.230.38.148" ], "related.user": [ "sitam" @@ -3332,8 +3332,8 @@ "oluptat6960.www5.test" ], "related.ip": [ - "10.166.142.198", "10.105.120.162", + "10.166.142.198", "10.175.181.138", "10.211.29.187" ], @@ -3400,10 +3400,10 @@ "fugiatnu2498.www.localhost" ], "related.ip": [ - "10.182.213.195", "10.122.133.162", + "10.220.202.102", "10.195.139.25", - "10.220.202.102" + "10.182.213.195" ], "related.user": [ "aquae" @@ -3468,9 +3468,9 @@ "ptat3230.domain" ], "related.ip": [ - "10.156.208.5", "10.53.72.161", "10.33.143.163", + "10.156.208.5", "10.247.144.9" ], "related.user": [ @@ -3535,10 +3535,10 @@ "exer447.internal.localhost" ], "related.ip": [ - "10.113.65.192", "10.241.143.145", "10.35.190.164", - "10.21.58.162" + "10.21.58.162", + "10.113.65.192" ], "related.user": [ "porin" @@ -3672,9 +3672,9 @@ ], "related.ip": [ "10.150.153.61", - "10.125.150.220", + "10.120.50.13", "10.22.213.196", - "10.120.50.13" + "10.125.150.220" ], "related.user": [ "inculpa" @@ -3740,9 +3740,9 @@ ], "related.ip": [ "10.85.52.249", + "10.238.171.184", "10.229.155.171", - "10.113.2.13", - "10.238.171.184" + "10.113.2.13" ], "related.user": [ "tatiset" @@ -3807,10 +3807,10 @@ "tatis7315.mail.home" ], "related.ip": [ - "10.249.174.35", - "10.198.150.185", "10.51.245.225", - "10.220.1.249" + "10.220.1.249", + "10.249.174.35", + "10.198.150.185" ], "related.user": [ "quela" @@ -3875,9 +3875,9 @@ "eosqui3723.api.localdomain" ], "related.ip": [ + "10.152.157.32", "10.190.96.181", "10.38.185.31", - "10.152.157.32", "10.251.82.195" ], "related.user": [ @@ -3942,10 +3942,10 @@ "itaedict199.mail.corp" ], "related.ip": [ - "10.230.112.179", - "10.103.102.242", + "10.190.247.194", "10.211.198.50", - "10.190.247.194" + "10.103.102.242", + "10.230.112.179" ], "related.user": [ "tDuisaut" @@ -4078,9 +4078,9 @@ ], "related.ip": [ "10.21.30.43", - "10.83.136.233", + "10.21.80.157", "10.31.86.83", - "10.21.80.157" + "10.83.136.233" ], "related.user": [ "litsed" @@ -4145,10 +4145,10 @@ "oluptas1637.home" ], "related.ip": [ - "10.195.90.73", - "10.194.197.107", "10.45.152.205", - "10.27.181.27" + "10.27.181.27", + "10.195.90.73", + "10.194.197.107" ], "related.user": [ "datatn" @@ -4213,10 +4213,10 @@ "ididu5505.api.localdomain" ], "related.ip": [ - "10.43.239.97", - "10.129.161.18", "10.222.2.132", - "10.183.90.25" + "10.129.161.18", + "10.183.90.25", + "10.43.239.97" ], "related.user": [ "aedicta" @@ -4280,10 +4280,10 @@ "mqui1099.api.corp" ], "related.ip": [ - "10.231.167.171", - "10.248.156.138", + "10.189.162.131", "10.67.129.100", - "10.189.162.131" + "10.248.156.138", + "10.231.167.171" ], "related.user": [ "sedquia" @@ -4349,9 +4349,9 @@ ], "related.ip": [ "10.63.103.30", - "10.185.107.27", + "10.6.146.184", "10.142.106.66", - "10.6.146.184" + "10.185.107.27" ], "related.user": [ "sequu" @@ -4415,10 +4415,10 @@ "tatiset4191.localdomain" ], "related.ip": [ - "10.119.179.182", + "10.214.93.200", "10.0.202.9", - "10.93.39.237", - "10.214.93.200" + "10.119.179.182", + "10.93.39.237" ], "related.user": [ "tionofd" @@ -4483,10 +4483,10 @@ "aute2433.mail.lan" ], "related.ip": [ - "10.28.145.163", + "10.123.154.140", "10.252.204.162", - "10.30.189.166", - "10.123.154.140" + "10.28.145.163", + "10.30.189.166" ], "related.user": [ "imadmin" @@ -4550,10 +4550,10 @@ "idolo6535.internal.example" ], "related.ip": [ - "10.46.162.198", - "10.79.49.3", + "10.145.128.250", "10.29.122.183", - "10.145.128.250" + "10.79.49.3", + "10.46.162.198" ], "related.user": [ "eni" @@ -4618,10 +4618,10 @@ "one7728.api.localdomain" ], "related.ip": [ + "10.142.235.217", "10.166.169.167", "10.65.174.196", - "10.177.232.136", - "10.142.235.217" + "10.177.232.136" ], "related.user": [ "olors" @@ -4686,10 +4686,10 @@ "uptatem4446.internal.localhost" ], "related.ip": [ - "10.53.188.140", + "10.29.217.44", "10.215.184.154", "10.191.78.86", - "10.29.217.44" + "10.53.188.140" ], "related.user": [ "iarc" @@ -4754,10 +4754,10 @@ "emq2514.api.localhost" ], "related.ip": [ + "10.46.222.149", "10.74.74.129", - "10.76.148.147", "10.135.77.156", - "10.46.222.149" + "10.76.148.147" ], "related.user": [ "urve" @@ -4821,10 +4821,10 @@ "agna5654.www.corp" ], "related.ip": [ - "10.130.203.37", - "10.145.49.29", "10.96.200.223", - "10.11.146.253" + "10.11.146.253", + "10.130.203.37", + "10.145.49.29" ], "related.user": [ "mvele" @@ -4889,9 +4889,9 @@ ], "related.ip": [ "10.162.78.48", - "10.48.75.140", + "10.162.2.180", "10.24.23.209", - "10.162.2.180" + "10.48.75.140" ], "related.user": [ "rumwr" @@ -4955,10 +4955,10 @@ "sequatD163.internal.example" ], "related.ip": [ - "10.151.206.38", - "10.66.92.83", + "10.97.105.115", "10.119.12.186", - "10.97.105.115" + "10.151.206.38", + "10.66.92.83" ], "related.user": [ "nproide" @@ -5090,10 +5090,10 @@ "epr3512.internal.domain" ], "related.ip": [ - "10.35.38.185", "10.111.128.11", "10.200.116.191", - "10.9.236.18" + "10.9.236.18", + "10.35.38.185" ], "related.user": [ "umfug" @@ -5157,10 +5157,10 @@ "uredol2174.home" ], "related.ip": [ - "10.191.27.182", + "10.236.67.227", "10.240.62.238", - "10.134.238.8", - "10.236.67.227" + "10.191.27.182", + "10.134.238.8" ], "related.user": [ "tlabo" @@ -5224,10 +5224,10 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.22.231.91", + "10.65.35.64", "10.109.14.142", - "10.165.66.92", - "10.65.35.64" + "10.22.231.91", + "10.165.66.92" ], "related.user": [ "perna" @@ -5291,10 +5291,10 @@ "inimav5557.www5.test" ], "related.ip": [ + "10.29.230.203", "10.71.112.86", - "10.64.161.215", "10.89.221.90", - "10.29.230.203" + "10.64.161.215" ], "related.user": [ "rnatur" @@ -5358,10 +5358,10 @@ "nonn1650.www.test" ], "related.ip": [ - "10.88.226.76", - "10.221.199.137", + "10.79.208.135", "10.140.118.182", - "10.79.208.135" + "10.221.199.137", + "10.88.226.76" ], "related.user": [ "erspic" @@ -5427,9 +5427,9 @@ ], "related.ip": [ "10.35.73.208", - "10.126.61.230", "10.189.244.22", - "10.133.48.55" + "10.133.48.55", + "10.126.61.230" ], "related.user": [ "tia" @@ -5493,8 +5493,8 @@ "suscipit587.www.localhost" ], "related.ip": [ - "10.35.65.72", "10.240.94.109", + "10.35.65.72", "10.239.194.105", "10.81.154.115" ], @@ -5561,10 +5561,10 @@ "mnisiut6146.internal.local" ], "related.ip": [ - "10.248.72.104", + "10.52.70.192", "10.38.253.213", "10.150.56.227", - "10.52.70.192" + "10.248.72.104" ], "related.user": [ "ionem" @@ -5629,10 +5629,10 @@ "borios1067.www5.home" ], "related.ip": [ + "10.218.15.164", "10.62.218.239", - "10.73.172.186", "10.203.193.134", - "10.218.15.164" + "10.73.172.186" ], "related.user": [ "reh" @@ -5696,10 +5696,10 @@ "msequ323.www.example" ], "related.ip": [ - "10.10.46.43", - "10.131.127.113", "10.136.211.234", - "10.60.20.76" + "10.131.127.113", + "10.60.20.76", + "10.10.46.43" ], "related.user": [ "nev" @@ -5766,8 +5766,8 @@ "related.ip": [ "10.233.181.250", "10.50.177.151", - "10.187.237.220", - "10.248.0.74" + "10.248.0.74", + "10.187.237.220" ], "related.user": [ "ugiatq" @@ -5833,9 +5833,9 @@ ], "related.ip": [ "10.189.43.11", + "10.248.248.120", "10.80.129.81", - "10.96.223.46", - "10.248.248.120" + "10.96.223.46" ], "related.user": [ "iatn" @@ -5902,8 +5902,8 @@ "related.ip": [ "10.91.115.139", "10.102.109.199", - "10.173.114.63", - "10.66.106.186" + "10.66.106.186", + "10.173.114.63" ], "related.user": [ "tNequ" @@ -5968,10 +5968,10 @@ "orpori3334.www.local" ], "related.ip": [ - "10.0.175.17", "10.221.223.127", - "10.198.157.122", - "10.159.155.88" + "10.0.175.17", + "10.159.155.88", + "10.198.157.122" ], "related.user": [ "iquipex" @@ -6035,10 +6035,10 @@ "equu7361.www5.localdomain" ], "related.ip": [ + "10.30.20.187", "10.189.70.237", "10.7.212.201", - "10.252.136.130", - "10.30.20.187" + "10.252.136.130" ], "related.user": [ "ugiat" @@ -6103,9 +6103,9 @@ "tse2979.internal.localhost" ], "related.ip": [ + "10.60.224.93", "10.83.105.69", "10.102.109.194", - "10.60.224.93", "10.242.121.165" ], "related.user": [ @@ -6171,10 +6171,10 @@ "uisnostr2390.mail.domain" ], "related.ip": [ - "10.219.174.45", - "10.17.20.93", + "10.181.134.69", "10.251.167.219", - "10.181.134.69" + "10.219.174.45", + "10.17.20.93" ], "related.user": [ "Uteni" @@ -6240,9 +6240,9 @@ ], "related.ip": [ "10.28.233.253", - "10.37.14.20", + "10.223.99.90", "10.30.117.82", - "10.223.99.90" + "10.37.14.20" ], "related.user": [ "numqua" @@ -6307,9 +6307,9 @@ "lites1614.www.corp" ], "related.ip": [ - "10.50.61.114", - "10.125.20.22", "10.8.32.17", + "10.125.20.22", + "10.50.61.114", "10.57.85.113" ], "related.user": [ @@ -6375,10 +6375,10 @@ "lorinrep7686.mail.corp" ], "related.ip": [ - "10.215.224.27", "10.113.78.101", - "10.181.63.82", - "10.200.28.55" + "10.215.224.27", + "10.200.28.55", + "10.181.63.82" ], "related.user": [ "ficiade" @@ -6443,9 +6443,9 @@ "nderit6272.mail.example" ], "related.ip": [ - "10.139.20.223", "10.169.95.128", "10.177.14.106", + "10.139.20.223", "10.243.43.168" ], "related.user": [ @@ -6511,10 +6511,10 @@ "ntu1279.mail.lan" ], "related.ip": [ - "10.92.168.198", - "10.18.176.44", + "10.39.100.88", "10.90.93.4", - "10.39.100.88" + "10.92.168.198", + "10.18.176.44" ], "related.user": [ "adminima" @@ -6581,8 +6581,8 @@ "related.ip": [ "10.193.43.135", "10.49.68.8", - "10.163.203.191", - "10.173.13.179" + "10.173.13.179", + "10.163.203.191" ], "related.user": [ "tlab" @@ -6646,10 +6646,10 @@ "cipitl2184.localdomain" ], "related.ip": [ - "10.240.47.113", "10.84.64.28", + "10.209.226.7", "10.31.147.51", - "10.209.226.7" + "10.240.47.113" ], "related.user": [ "ull" @@ -6714,9 +6714,9 @@ "item3647.home" ], "related.ip": [ + "10.86.1.244", "10.32.20.4", "10.52.13.192", - "10.86.1.244", "10.225.189.229" ], "related.user": [ diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 895c0723f10d..814b2b0f30dd 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json index 879cbc128a61..244c0a399b2d 100644 --- a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json @@ -367,8 +367,8 @@ "observer.vendor": "F5", "process.pid": 2289, "related.ip": [ - "10.225.160.182", - "10.204.123.107" + "10.204.123.107", + "10.225.160.182" ], "rsa.internal.messageid": "01490500", "rsa.misc.log_session_id": "eFinib", @@ -771,7 +771,13 @@ "f5.bigipapm", "forwarded" ], - "url.original": "https://www5.example.com/olli/rever.html?rsp=oluptat#metco" + "url.domain": "www5.example.com", + "url.extension": "html", + "url.fragment": "metco", + "url.original": "https://www5.example.com/olli/rever.html?rsp=oluptat#metco", + "url.path": "/olli/rever.html", + "url.query": "rsp=oluptat", + "url.scheme": "https" }, { "@timestamp": "2017-03-18T20:24:33.000Z", @@ -1476,8 +1482,8 @@ "observer.vendor": "F5", "process.pid": 2943, "related.ip": [ - "10.16.181.60", - "10.142.213.80" + "10.142.213.80", + "10.16.181.60" ], "rsa.internal.messageid": "01490549", "rsa.misc.group": "tationu", @@ -1567,8 +1573,8 @@ "observer.vendor": "F5", "process.pid": 1973, "related.ip": [ - "10.47.99.72", - "10.187.64.126" + "10.187.64.126", + "10.47.99.72" ], "rsa.internal.messageid": "01490500", "rsa.misc.category": "oremipsu", @@ -2131,7 +2137,13 @@ "f5.bigipapm", "forwarded" ], - "url.original": "https://example.net/equamn/scipi.txt?eiu=maliquam#gnama" + "url.domain": "example.net", + "url.extension": "txt", + "url.fragment": "gnama", + "url.original": "https://example.net/equamn/scipi.txt?eiu=maliquam#gnama", + "url.path": "/equamn/scipi.txt", + "url.query": "eiu=maliquam", + "url.scheme": "https" }, { "@timestamp": "2019-04-01T02:38:14.000Z", diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json index 160c14c7981b..5880bc2c97e4 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json @@ -25,8 +25,8 @@ "boNemoe4402.www.invalid" ], "related.ip": [ - "10.150.92.220", - "10.102.123.34" + "10.102.123.34", + "10.150.92.220" ], "related.user": [ "sumdo" @@ -211,8 +211,8 @@ "tinculp2940.internal.local" ], "related.ip": [ - "10.202.204.154", - "10.134.137.177" + "10.134.137.177", + "10.202.204.154" ], "related.user": [ "orsitame" @@ -335,8 +335,8 @@ "enim5316.www5.local" ], "related.ip": [ - "10.202.72.124", - "10.200.188.142" + "10.200.188.142", + "10.202.72.124" ], "related.user": [ "iusmodt" @@ -397,8 +397,8 @@ "reetdolo2770.www5.local" ], "related.ip": [ - "10.12.44.169", - "10.214.225.125" + "10.214.225.125", + "10.12.44.169" ], "related.user": [ "erep" @@ -644,8 +644,8 @@ "tatno6787.internal.localhost" ], "related.ip": [ - "10.136.252.240", - "10.65.83.160" + "10.65.83.160", + "10.136.252.240" ], "related.user": [ "ender" @@ -767,8 +767,8 @@ "ali6446.localhost" ], "related.ip": [ - "10.200.156.102", - "10.144.82.69" + "10.144.82.69", + "10.200.156.102" ], "related.user": [ "rveli" @@ -828,8 +828,8 @@ "torev7118.internal.domain" ], "related.ip": [ - "10.109.232.112", - "10.72.58.135" + "10.72.58.135", + "10.109.232.112" ], "related.user": [ "xea" @@ -890,8 +890,8 @@ "dolore6103.www5.example" ], "related.ip": [ - "10.38.22.45", - "10.72.29.73" + "10.72.29.73", + "10.38.22.45" ], "related.user": [ "onproide" @@ -952,8 +952,8 @@ "errorsi6996.www.domain" ], "related.ip": [ - "10.76.72.111", - "10.70.95.74" + "10.70.95.74", + "10.76.72.111" ], "related.user": [ "ivelits" @@ -1014,8 +1014,8 @@ "lumquido5839.api.corp" ], "related.ip": [ - "10.73.69.75", - "10.19.201.13" + "10.19.201.13", + "10.73.69.75" ], "related.user": [ "tat" @@ -1075,8 +1075,8 @@ "aperia4409.www5.invalid" ], "related.ip": [ - "10.84.105.75", - "10.78.151.178" + "10.78.151.178", + "10.84.105.75" ], "related.user": [ "iquaUten" @@ -1199,8 +1199,8 @@ "eme6710.mail.invalid" ], "related.ip": [ - "10.121.219.204", - "10.104.134.200" + "10.104.134.200", + "10.121.219.204" ], "related.user": [ "uptat" @@ -1323,8 +1323,8 @@ "umexerci1284.internal.localdomain" ], "related.ip": [ - "10.141.44.153", - "10.161.57.8" + "10.161.57.8", + "10.141.44.153" ], "related.user": [ "quisnos" @@ -1385,8 +1385,8 @@ "adol485.example" ], "related.ip": [ - "10.6.167.7", - "10.153.111.103" + "10.153.111.103", + "10.6.167.7" ], "related.user": [ "eumfug" @@ -1446,8 +1446,8 @@ "evita5008.www.localdomain" ], "related.ip": [ - "10.134.148.219", - "10.248.204.182" + "10.248.204.182", + "10.134.148.219" ], "related.user": [ "uioffi" @@ -1507,8 +1507,8 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ - "10.178.77.231", - "10.163.5.243" + "10.163.5.243", + "10.178.77.231" ], "related.user": [ "liquide" @@ -1568,8 +1568,8 @@ "ris3314.mail.invalid" ], "related.ip": [ - "10.221.89.228", - "10.177.194.18" + "10.177.194.18", + "10.221.89.228" ], "related.user": [ "aliquam" @@ -1691,8 +1691,8 @@ "non3341.mail.invalid" ], "related.ip": [ - "10.168.90.81", - "10.101.57.120" + "10.101.57.120", + "10.168.90.81" ], "related.user": [ "eporr" @@ -1815,8 +1815,8 @@ "stquido5705.api.host" ], "related.ip": [ - "10.60.129.15", - "10.248.101.25" + "10.248.101.25", + "10.60.129.15" ], "related.user": [ "evolup" @@ -1938,8 +1938,8 @@ "vita2681.www5.local" ], "related.ip": [ - "10.27.14.168", - "10.66.2.232" + "10.66.2.232", + "10.27.14.168" ], "related.user": [ "uirati" @@ -2185,8 +2185,8 @@ "eumiu765.api.lan" ], "related.ip": [ - "10.4.157.1", - "10.184.18.202" + "10.184.18.202", + "10.4.157.1" ], "related.user": [ "oditem" @@ -2309,8 +2309,8 @@ "idestlab2631.www.lan" ], "related.ip": [ - "10.27.16.118", - "10.83.177.2" + "10.83.177.2", + "10.27.16.118" ], "related.user": [ "borios" @@ -2371,8 +2371,8 @@ "inesci6789.test" ], "related.ip": [ - "10.167.227.44", - "10.38.54.72" + "10.38.54.72", + "10.167.227.44" ], "related.user": [ "riamea" @@ -2494,8 +2494,8 @@ "ima2031.api.corp" ], "related.ip": [ - "10.9.18.237", - "10.9.12.248" + "10.9.12.248", + "10.9.18.237" ], "related.user": [ "uradi" @@ -2680,8 +2680,8 @@ "gnaaliqu3935.api.test" ], "related.ip": [ - "10.142.25.100", - "10.134.18.114" + "10.134.18.114", + "10.142.25.100" ], "related.user": [ "osqui" @@ -2742,8 +2742,8 @@ "nsequat1859.internal.localhost" ], "related.ip": [ - "10.28.118.160", - "10.223.119.218" + "10.223.119.218", + "10.28.118.160" ], "related.user": [ "ntsunt" @@ -2927,8 +2927,8 @@ "rem7043.localhost" ], "related.ip": [ - "10.227.173.252", - "10.65.2.106" + "10.65.2.106", + "10.227.173.252" ], "related.user": [ "itation" @@ -3112,8 +3112,8 @@ "giatquov1918.internal.example" ], "related.ip": [ - "10.180.195.43", - "10.85.185.13" + "10.85.185.13", + "10.180.195.43" ], "related.user": [ "voluptas" @@ -3236,8 +3236,8 @@ "Sedut1775.www.domain" ], "related.ip": [ - "10.248.165.185", - "10.86.11.48" + "10.86.11.48", + "10.248.165.185" ], "related.user": [ "dquiac" @@ -3422,8 +3422,8 @@ "naaliq3710.api.local" ], "related.ip": [ - "10.120.10.211", - "10.28.82.189" + "10.28.82.189", + "10.120.10.211" ], "related.user": [ "rcit" @@ -3484,8 +3484,8 @@ "volupta3552.internal.localhost" ], "related.ip": [ - "10.31.237.225", - "10.6.38.163" + "10.6.38.163", + "10.31.237.225" ], "related.user": [ "olup" @@ -3546,8 +3546,8 @@ "onse380.internal.localdomain" ], "related.ip": [ - "10.226.5.189", - "10.125.165.144" + "10.125.165.144", + "10.226.5.189" ], "related.user": [ "mvolu" @@ -3670,8 +3670,8 @@ "oloreseo5039.test" ], "related.ip": [ - "10.28.105.124", - "10.218.0.197" + "10.218.0.197", + "10.28.105.124" ], "related.user": [ "ntNe" @@ -3732,8 +3732,8 @@ "minim459.mail.local" ], "related.ip": [ - "10.123.199.198", - "10.17.87.79" + "10.17.87.79", + "10.123.199.198" ], "related.user": [ "ratvolu" @@ -3856,8 +3856,8 @@ "aparia1179.www.localdomain" ], "related.ip": [ - "10.193.118.163", - "10.115.174.107" + "10.115.174.107", + "10.193.118.163" ], "related.user": [ "exeacomm" @@ -3918,8 +3918,8 @@ "iatqu6203.mail.corp" ], "related.ip": [ - "10.77.77.208", - "10.37.128.49" + "10.37.128.49", + "10.77.77.208" ], "related.user": [ "moles" @@ -4042,8 +4042,8 @@ "mag1506.internal.domain" ], "related.ip": [ - "10.131.126.109", - "10.182.152.242" + "10.182.152.242", + "10.131.126.109" ], "related.user": [ "dolor" @@ -4351,8 +4351,8 @@ "abor1370.www.domain" ], "related.ip": [ - "10.77.78.180", - "10.97.236.123" + "10.97.236.123", + "10.77.78.180" ], "related.user": [ "nisi" @@ -4536,8 +4536,8 @@ "venia2079.mail.example" ], "related.ip": [ - "10.65.144.51", - "10.5.11.205" + "10.5.11.205", + "10.65.144.51" ], "related.user": [ "uptat" @@ -4660,8 +4660,8 @@ "gelitsed3249.corp" ], "related.ip": [ - "10.138.210.116", - "10.225.255.211" + "10.225.255.211", + "10.138.210.116" ], "related.user": [ "fugiatn" @@ -4968,8 +4968,8 @@ "edq5397.www.test" ], "related.ip": [ - "10.73.28.165", - "10.221.206.74" + "10.221.206.74", + "10.73.28.165" ], "related.user": [ "quas" @@ -5092,8 +5092,8 @@ "rumet6923.www5.lan" ], "related.ip": [ - "10.30.246.132", - "10.208.18.210" + "10.208.18.210", + "10.30.246.132" ], "related.user": [ "veniam" @@ -5216,8 +5216,8 @@ "amc3059.local" ], "related.ip": [ - "10.181.41.154", - "10.29.109.126" + "10.29.109.126", + "10.181.41.154" ], "related.user": [ "labo" @@ -5339,8 +5339,8 @@ "liquipex1155.mail.corp" ], "related.ip": [ - "10.183.189.133", - "10.154.191.225" + "10.154.191.225", + "10.183.189.133" ], "related.user": [ "ita" @@ -5648,8 +5648,8 @@ "reprehen3513.test" ], "related.ip": [ - "10.10.86.55", - "10.61.225.196" + "10.61.225.196", + "10.10.86.55" ], "related.user": [ "eniamqu" @@ -5834,8 +5834,8 @@ "rumSecti111.www5.domain" ], "related.ip": [ - "10.87.90.49", - "10.222.245.80" + "10.222.245.80", + "10.87.90.49" ], "related.user": [ "ptatemse" @@ -6020,8 +6020,8 @@ "uptasnul2751.www5.corp" ], "related.ip": [ - "10.161.64.168", - "10.194.67.223" + "10.194.67.223", + "10.161.64.168" ], "related.user": [ "tion" @@ -6081,8 +6081,8 @@ "upt6017.api.localdomain" ], "related.ip": [ - "10.100.154.220", - "10.120.148.241" + "10.120.148.241", + "10.100.154.220" ], "related.user": [ "rsitam" diff --git a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json index 8e79962d5bc2..afb294c38149 100644 --- a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json @@ -596,8 +596,8 @@ "lamcolab3252.www.invalid" ], "related.ip": [ - "10.179.124.125", - "10.177.36.38" + "10.177.36.38", + "10.179.124.125" ], "rsa.email.email_dst": "ectio", "rsa.email.email_src": "sequine", @@ -1011,8 +1011,8 @@ "lapariat7287.internal.host" ], "related.ip": [ - "10.140.7.83", - "10.68.246.187" + "10.68.246.187", + "10.140.7.83" ], "rsa.email.email_dst": "gna", "rsa.email.email_src": "icabo", @@ -2680,12 +2680,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "taevit4968.mail.local", - "mveni5084.internal.local" + "mveni5084.internal.local", + "taevit4968.mail.local" ], "related.ip": [ - "10.144.111.42", - "10.62.61.1" + "10.62.61.1", + "10.144.111.42" ], "rsa.email.email_dst": "com", "rsa.email.email_src": "lam", @@ -3131,12 +3131,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "modi6930.internal.test", - "taevitae6868.www.corp" + "taevitae6868.www.corp", + "modi6930.internal.test" ], "related.ip": [ - "10.60.164.100", - "10.161.1.146" + "10.161.1.146", + "10.60.164.100" ], "rsa.email.email_dst": "nproiden", "rsa.email.email_src": "etconse", @@ -3235,12 +3235,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "uradip7802.mail.example", - "tetura7106.www5.corp" + "tetura7106.www5.corp", + "uradip7802.mail.example" ], "related.ip": [ - "10.44.35.57", - "10.93.239.216" + "10.93.239.216", + "10.44.35.57" ], "rsa.email.email_dst": "ciun", "rsa.email.email_src": "vento", @@ -3845,8 +3845,8 @@ "ncu3839.www.localhost" ], "related.ip": [ - "10.201.105.58", - "10.251.183.113" + "10.251.183.113", + "10.201.105.58" ], "rsa.email.email_dst": "ionemu", "rsa.email.email_src": "ent", @@ -3889,8 +3889,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.ip": [ - "10.209.203.156", - "10.132.139.98" + "10.132.139.98", + "10.209.203.156" ], "rsa.email.email_dst": "borisnis", "rsa.email.email_src": "pariat", diff --git a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml index eda458f01ded..b423b2203e4c 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json index bd66027098dc..3f078976410f 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json @@ -155,8 +155,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "roinBCS", "url.original": "https://www.example.net/orisn/cca.htm?ofdeF=metcons#roinBCS", - "url.query": "taspe", + "url.path": "/orisn/cca.htm", + "url.query": [ + "taspe", + "ofdeF=metcons" + ], + "url.scheme": "https", "user.name": "oluptas" }, { @@ -188,9 +196,9 @@ "ccaecat" ], "related.ip": [ - "10.15.159.80", + "10.94.103.117", "10.200.188.142", - "10.94.103.117" + "10.15.159.80" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -256,8 +264,8 @@ ], "related.ip": [ "10.50.112.141", - "10.27.88.95", - "10.131.233.27" + "10.131.233.27", + "10.27.88.95" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -322,13 +330,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.5670", "related.hosts": [ - "ntutl", "roinBCSe", + "ntutl", "olo7148.mail.home" ], "related.ip": [ - "10.87.212.179", - "10.157.213.15" + "10.157.213.15", + "10.87.212.179" ], "related.user": [ "rveli" @@ -383,8 +391,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "saute", "url.original": "https://api.example.com/iumto/aboreetd.gif?dun=enim#saute", - "url.query": "nsect", + "url.path": "/iumto/aboreetd.gif", + "url.query": [ + "nsect", + "dun=enim" + ], + "url.scheme": "https", "user.name": "rveli" }, { @@ -415,8 +431,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.152", "related.hosts": [ - "orain", "onse", + "orain", "agna7678.internal.host" ], "related.ip": [ @@ -431,8 +447,8 @@ "rsa.investigations.event_vcat": "mwr", "rsa.misc.OS": "imaven", "rsa.misc.action": [ - "uines", - "accept" + "accept", + "uines" ], "rsa.misc.category": "uidolo", "rsa.misc.client": "emips", @@ -476,8 +492,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "picia", "url.original": "https://www.example.net/tasuntex/sunt.txt?ume=incidi#picia", - "url.query": "nsec", + "url.path": "/tasuntex/sunt.txt", + "url.query": [ + "nsec", + "ume=incidi" + ], + "url.scheme": "https", "user.name": "aperia" }, { @@ -508,8 +532,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4059", "related.hosts": [ - "utla", "tatn", + "utla", "equep5085.mail.domain" ], "related.ip": [ @@ -569,8 +593,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "tlabore", "url.original": "https://www.example.org/inesci/serror.html?mqu=apariat#tlabore", - "url.query": "nsectet", + "url.path": "/inesci/serror.html", + "url.query": [ + "nsectet", + "mqu=apariat" + ], + "url.scheme": "https", "user.name": "nnum" }, { @@ -602,9 +634,9 @@ "labore" ], "related.ip": [ - "10.186.85.3", "10.114.16.155", - "10.176.216.90" + "10.176.216.90", + "10.186.85.3" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -668,13 +700,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3917", "related.hosts": [ - "gnido", "sperna", + "gnido", "eturadi6608.mail.host" ], "related.ip": [ - "10.23.62.94", - "10.61.163.4" + "10.61.163.4", + "10.23.62.94" ], "related.user": [ "non" @@ -684,8 +716,8 @@ "rsa.investigations.event_vcat": "oide", "rsa.misc.OS": "gel", "rsa.misc.action": [ - "cancel", - "luptatem" + "luptatem", + "cancel" ], "rsa.misc.category": "uir", "rsa.misc.client": "ratvolu", @@ -729,8 +761,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "issus", "url.original": "https://api.example.com/nimid/itatione.htm?umwr=oluptate#issus", - "url.query": "atem", + "url.path": "/nimid/itatione.htm", + "url.query": [ + "atem", + "umwr=oluptate" + ], + "url.scheme": "https", "user.name": "non" }, { @@ -766,8 +806,8 @@ "ipsumdol4488.api.localdomain" ], "related.ip": [ - "10.106.31.86", - "10.28.76.42" + "10.28.76.42", + "10.106.31.86" ], "related.user": [ "cons" @@ -822,8 +862,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "htm", + "url.fragment": "pora", "url.original": "https://example.com/tetu/stru.htm?tlabore=Exc#pora", - "url.query": "atno", + "url.path": "/tetu/stru.htm", + "url.query": [ + "atno", + "tlabore=Exc" + ], + "url.scheme": "https", "user.name": "cons" }, { @@ -855,9 +903,9 @@ "diconseq" ], "related.ip": [ - "10.106.162.153", "10.58.214.16", - "10.238.164.74" + "10.238.164.74", + "10.106.162.153" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -922,8 +970,8 @@ "tenimad" ], "related.ip": [ - "10.225.141.20", "10.110.31.190", + "10.225.141.20", "10.217.150.196" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -988,13 +1036,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3319", "related.hosts": [ - "amc", "mestq", + "amc", "cusant4946.www.domain" ], "related.ip": [ - "10.137.56.173", - "10.69.103.176" + "10.69.103.176", + "10.137.56.173" ], "related.user": [ "proide" @@ -1049,8 +1097,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "isnisi", "url.original": "https://api.example.com/isnostr/umqu.htm?emquia=inesci#isnisi", - "url.query": "uptate", + "url.path": "/isnostr/umqu.htm", + "url.query": [ + "uptate", + "emquia=inesci" + ], + "url.scheme": "https", "user.name": "proide" }, { @@ -1083,8 +1139,8 @@ ], "related.ip": [ "10.30.47.165", - "10.25.212.118", - "10.5.235.217" + "10.5.235.217", + "10.25.212.118" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1209,8 +1265,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "urerepr", "url.original": "https://example.org/tiaec/rumwrit.txt?oconsequ=edquiac#urerepr", - "url.query": "iaeco", + "url.path": "/tiaec/rumwrit.txt", + "url.query": [ + "iaeco", + "oconsequ=edquiac" + ], + "url.scheme": "https", "user.name": "tetur" }, { @@ -1243,8 +1307,8 @@ ], "related.ip": [ "10.233.120.207", - "10.51.213.42", - "10.98.194.212" + "10.98.194.212", + "10.51.213.42" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1309,9 +1373,9 @@ "tla" ], "related.ip": [ - "10.67.132.242", "10.241.132.176", - "10.245.187.229" + "10.245.187.229", + "10.67.132.242" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1391,8 +1455,8 @@ "rsa.investigations.event_vcat": "mes", "rsa.misc.OS": "riss", "rsa.misc.action": [ - "oin", - "allow" + "allow", + "oin" ], "rsa.misc.category": "idi", "rsa.misc.client": "modocon", @@ -1436,8 +1500,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "itlab", "url.original": "https://mail.example.org/ptassita/its.gif?risnis=uov#itlab", - "url.query": "hil", + "url.path": "/ptassita/its.gif", + "url.query": [ + "hil", + "risnis=uov" + ], + "url.scheme": "https", "user.name": "atv" }, { @@ -1468,8 +1540,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.760", "related.hosts": [ - "dolorsit", "rcit", + "dolorsit", "mve1890.internal.home" ], "related.ip": [ @@ -1484,8 +1556,8 @@ "rsa.investigations.event_vcat": "metcons", "rsa.misc.OS": "ehende", "rsa.misc.action": [ - "umf", - "deny" + "deny", + "umf" ], "rsa.misc.category": "emUte", "rsa.misc.client": "archite", @@ -1529,8 +1601,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "toditaut", "url.original": "https://www.example.org/redol/gnaa.htm?aliquamq=dtempori#toditaut", - "url.query": "quames", + "url.path": "/redol/gnaa.htm", + "url.query": [ + "quames", + "aliquamq=dtempori" + ], + "url.scheme": "https", "user.name": "orese" }, { @@ -1561,8 +1641,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4450", "related.hosts": [ - "saquaea", "billoi", + "saquaea", "eturad6143.www.home" ], "related.ip": [ @@ -1622,8 +1702,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "offici", "url.original": "https://example.net/aparia/tatnon.jpg?rever=ore#offici", - "url.query": "essecill", + "url.path": "/aparia/tatnon.jpg", + "url.query": [ + "essecill", + "rever=ore" + ], + "url.scheme": "https", "user.name": "enim" }, { @@ -1659,8 +1747,8 @@ "orinrep5386.www.corp" ], "related.ip": [ - "10.253.228.140", - "10.208.21.135" + "10.208.21.135", + "10.253.228.140" ], "related.user": [ "inculp" @@ -1670,8 +1758,8 @@ "rsa.investigations.event_vcat": "emagn", "rsa.misc.OS": "oditempo", "rsa.misc.action": [ - "ugitse", - "cancel" + "cancel", + "ugitse" ], "rsa.misc.category": "magnid", "rsa.misc.client": "sci", @@ -1715,8 +1803,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "equamnih", "url.original": "https://www.example.com/tali/BCS.txt?iqu=niamqu#equamnih", - "url.query": "quiineav", + "url.path": "/tali/BCS.txt", + "url.query": [ + "quiineav", + "iqu=niamqu" + ], + "url.scheme": "https", "user.name": "inculp" }, { @@ -1808,8 +1904,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "mexer", "url.original": "https://www.example.com/iame/orroquis.htm?tiumd=ntmoll#mexer", - "url.query": "tec", + "url.path": "/iame/orroquis.htm", + "url.query": [ + "tec", + "tiumd=ntmoll" + ], + "url.scheme": "https", "user.name": "olorem" }, { @@ -1840,13 +1944,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.5380", "related.hosts": [ - "uei", "onse", + "uei", "reseosqu1629.mail.lan" ], "related.ip": [ - "10.106.85.174", - "10.94.242.80" + "10.94.242.80", + "10.106.85.174" ], "related.user": [ "lmo" @@ -1901,8 +2005,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "mnisi", "url.original": "https://www.example.net/iame/laudanti.htm?stquido=rsitvolu#mnisi", - "url.query": "serror", + "url.path": "/iame/laudanti.htm", + "url.query": [ + "serror", + "stquido=rsitvolu" + ], + "url.scheme": "https", "user.name": "lmo" }, { @@ -1935,9 +2047,9 @@ "oluptat" ], "related.ip": [ - "10.117.63.181", + "10.168.20.20", "10.247.53.179", - "10.168.20.20" + "10.117.63.181" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2002,8 +2114,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2208", "related.hosts": [ - "lamcola", "duntut", + "lamcola", "tasnul4179.internal.host" ], "related.ip": [ @@ -2018,8 +2130,8 @@ "rsa.investigations.event_vcat": "illumq", "rsa.misc.OS": "idata", "rsa.misc.action": [ - "emacc", - "block" + "block", + "emacc" ], "rsa.misc.category": "ueporro", "rsa.misc.client": "veli", @@ -2063,8 +2175,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "rQuisaut", "url.original": "https://www.example.org/ess/quiad.jpg?ten=litanim#rQuisaut", - "url.query": "emp", + "url.path": "/ess/quiad.jpg", + "url.query": [ + "emp", + "ten=litanim" + ], + "url.scheme": "https", "user.name": "amqu" }, { @@ -2096,13 +2216,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3402", "related.hosts": [ - "tur", "imavenia", + "tur", "bore5546.www.local" ], "related.ip": [ - "10.44.198.184", - "10.189.82.19" + "10.189.82.19", + "10.44.198.184" ], "related.user": [ "iconsequ" @@ -2157,8 +2277,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "txt", + "url.fragment": "Finibus", "url.original": "https://example.com/xeac/llitanim.txt?oreverit=scip#Finibus", - "url.query": "atura", + "url.path": "/xeac/llitanim.txt", + "url.query": [ + "atura", + "oreverit=scip" + ], + "url.scheme": "https", "user.name": "iconsequ" }, { @@ -2250,8 +2378,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "iameaque", "url.original": "https://api.example.org/orio/gna.gif?aaliquaU=olu#iameaque", - "url.query": "tatio", + "url.path": "/orio/gna.gif", + "url.query": [ + "tatio", + "aaliquaU=olu" + ], + "url.scheme": "https", "user.name": "iosamni" }, { @@ -2343,8 +2479,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "umtota", "url.original": "https://mail.example.org/nimadmin/lumqui.txt?iquip=tinculpa#umtota", - "url.query": "iuta", + "url.path": "/nimadmin/lumqui.txt", + "url.query": [ + "iuta", + "iquip=tinculpa" + ], + "url.scheme": "https", "user.name": "eavolup" }, { @@ -2376,9 +2520,9 @@ "uio" ], "related.ip": [ - "10.37.161.101", "10.17.209.252", - "10.111.182.212" + "10.111.182.212", + "10.37.161.101" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2443,9 +2587,9 @@ "itautfu" ], "related.ip": [ - "10.158.175.98", + "10.170.196.181", "10.153.166.133", - "10.170.196.181" + "10.158.175.98" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2514,8 +2658,8 @@ "con6049.internal.lan" ], "related.ip": [ - "10.63.171.91", - "10.48.25.200" + "10.48.25.200", + "10.63.171.91" ], "related.user": [ "usanti" @@ -2570,8 +2714,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "quisn", "url.original": "https://internal.example.com/temse/caecat.jpg?emeu=tatemac#quisn", - "url.query": "mveniam", + "url.path": "/temse/caecat.jpg", + "url.query": [ + "mveniam", + "emeu=tatemac" + ], + "url.scheme": "https", "user.name": "usanti" }, { @@ -2737,9 +2889,9 @@ "eturadip" ], "related.ip": [ + "10.214.156.161", "10.145.194.12", - "10.66.90.225", - "10.214.156.161" + "10.66.90.225" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2804,8 +2956,8 @@ "iutal" ], "related.ip": [ - "10.156.208.5", "10.6.242.108", + "10.156.208.5", "10.163.36.101" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -2870,8 +3022,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4713", "related.hosts": [ - "epteurs", "data", + "epteurs", "remeum2641.www5.corp" ], "related.ip": [ @@ -2886,8 +3038,8 @@ "rsa.investigations.event_vcat": "olore", "rsa.misc.OS": "tatem", "rsa.misc.action": [ - "allow", - "itanimi" + "itanimi", + "allow" ], "rsa.misc.category": "psa", "rsa.misc.client": "ugits", @@ -2931,8 +3083,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "isni", "url.original": "https://www5.example.com/elit/sam.htm?nevolu=unt#isni", - "url.query": "onoru", + "url.path": "/elit/sam.htm", + "url.query": [ + "onoru", + "nevolu=unt" + ], + "url.scheme": "https", "user.name": "estiaec" }, { @@ -2968,8 +3128,8 @@ "itaspe3216.localdomain" ], "related.ip": [ - "10.94.177.125", - "10.116.82.108" + "10.116.82.108", + "10.94.177.125" ], "related.user": [ "ecatc" @@ -3024,8 +3184,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "iadeseru", "url.original": "https://example.org/porro/issu.htm?inculpa=ruredol#iadeseru", - "url.query": "olorsit", + "url.path": "/porro/issu.htm", + "url.query": [ + "olorsit", + "inculpa=ruredol" + ], + "url.scheme": "https", "user.name": "ecatc" }, { @@ -3062,8 +3230,8 @@ "mea6298.api.example" ], "related.ip": [ - "10.115.121.243", - "10.113.152.241" + "10.113.152.241", + "10.115.121.243" ], "related.user": [ "norumetM" @@ -3118,8 +3286,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "suntexpl", "url.original": "https://mail.example.org/uamquaer/texplica.gif?sequa=lorum#suntexpl", - "url.query": "udexerci", + "url.path": "/uamquaer/texplica.gif", + "url.query": [ + "udexerci", + "sequa=lorum" + ], + "url.scheme": "https", "user.name": "norumetM" }, { @@ -3150,8 +3326,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3804", "related.hosts": [ - "atcupi", "nder", + "atcupi", "iqu7510.internal.corp" ], "related.ip": [ @@ -3211,8 +3387,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "rsit", "url.original": "https://www5.example.org/aqu/utemvele.gif?serrorsi=tsedquia#rsit", - "url.query": "nihilmol", + "url.path": "/aqu/utemvele.gif", + "url.query": [ + "nihilmol", + "serrorsi=tsedquia" + ], + "url.scheme": "https", "user.name": "dictasun" }, { @@ -3429,8 +3613,8 @@ "deFinibu3940.internal.lan" ], "related.ip": [ - "10.124.71.88", - "10.22.248.52" + "10.22.248.52", + "10.124.71.88" ], "related.user": [ "tcons" @@ -3485,8 +3669,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "adeserun", "url.original": "https://www5.example.com/etcon/chit.txt?erspici=itinvolu#adeserun", - "url.query": "tinvolu", + "url.path": "/etcon/chit.txt", + "url.query": [ + "tinvolu", + "erspici=itinvolu" + ], + "url.scheme": "https", "user.name": "tcons" }, { @@ -3517,8 +3709,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4895", "related.hosts": [ - "ficiade", "ipexeac", + "ficiade", "tatiset4191.localdomain" ], "related.ip": [ @@ -3533,8 +3725,8 @@ "rsa.investigations.event_vcat": "iae", "rsa.misc.OS": "evelite", "rsa.misc.action": [ - "block", - "essequam" + "essequam", + "block" ], "rsa.misc.category": "tmollita", "rsa.misc.client": "uiinea", @@ -3578,8 +3770,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "nseq", "url.original": "https://api.example.org/fug/liquid.txt?ptate=lloi#nseq", - "url.query": "undeo", + "url.path": "/fug/liquid.txt", + "url.query": [ + "undeo", + "ptate=lloi" + ], + "url.scheme": "https", "user.name": "eumiure" }, { @@ -3612,8 +3812,8 @@ ], "related.ip": [ "10.200.12.126", - "10.250.231.196", - "10.14.145.107" + "10.14.145.107", + "10.250.231.196" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3678,9 +3878,9 @@ "atuse" ], "related.ip": [ - "10.21.203.112", + "10.103.36.192", "10.225.34.176", - "10.103.36.192" + "10.21.203.112" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3745,9 +3945,9 @@ "usantiu" ], "related.ip": [ - "10.5.67.140", + "10.118.111.183", "10.140.59.161", - "10.118.111.183" + "10.5.67.140" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3811,8 +4011,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4493", "related.hosts": [ - "veleumiu", "labor", + "veleumiu", "nimadmi4084.api.home" ], "related.ip": [ @@ -3872,8 +4072,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.net", + "url.extension": "html", + "url.fragment": "litess", "url.original": "https://example.net/aturQui/tquii.html?uiac=squ#litess", - "url.query": "Loremip", + "url.path": "/aturQui/tquii.html", + "url.query": [ + "Loremip", + "uiac=squ" + ], + "url.scheme": "https", "user.name": "itsed" }, { @@ -3920,8 +4128,8 @@ "rsa.investigations.event_vcat": "uep", "rsa.misc.OS": "iatisund", "rsa.misc.action": [ - "nvo", - "block" + "block", + "nvo" ], "rsa.misc.category": "tenima", "rsa.misc.client": "iuntNe", @@ -3965,8 +4173,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "aturvel", "url.original": "https://mail.example.org/mvele/teveli.htm?Nequepor=luptate#aturvel", - "url.query": "lab", + "url.path": "/mvele/teveli.htm", + "url.query": [ + "lab", + "Nequepor=luptate" + ], + "url.scheme": "https", "user.name": "rporis" }, { @@ -3998,9 +4214,9 @@ "sitv" ], "related.ip": [ - "10.251.183.113", "10.217.145.137", - "10.22.149.132" + "10.22.149.132", + "10.251.183.113" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4065,9 +4281,9 @@ "nisi" ], "related.ip": [ + "10.203.66.175", "10.183.16.252", - "10.51.60.203", - "10.203.66.175" + "10.51.60.203" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4131,8 +4347,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.409", "related.hosts": [ - "magnama", "doei", + "magnama", "ursint411.www.lan" ], "related.ip": [ @@ -4147,8 +4363,8 @@ "rsa.investigations.event_vcat": "nisiuta", "rsa.misc.OS": "olu", "rsa.misc.action": [ - "tquov", - "accept" + "accept", + "tquov" ], "rsa.misc.category": "quatD", "rsa.misc.client": "acomm", @@ -4192,8 +4408,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "etc", "url.original": "https://example.org/tem/iadeseru.jpg?olorsita=odoco#etc", - "url.query": "natu", + "url.path": "/tem/iadeseru.jpg", + "url.query": [ + "natu", + "olorsita=odoco" + ], + "url.scheme": "https", "user.name": "nimadmi" }, { @@ -4229,8 +4453,8 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.242.178.15", - "10.217.111.77" + "10.217.111.77", + "10.242.178.15" ], "related.user": [ "nimadmin" @@ -4240,8 +4464,8 @@ "rsa.investigations.event_vcat": "psaqu", "rsa.misc.OS": "nevolu", "rsa.misc.action": [ - "datatno", - "allow" + "allow", + "datatno" ], "rsa.misc.category": "ionu", "rsa.misc.client": "ugiatn", @@ -4285,8 +4509,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "tobe", "url.original": "https://mail.example.net/epteurs/usmodtem.gif?itvo=asi#tobe", - "url.query": "equepor", + "url.path": "/epteurs/usmodtem.gif", + "url.query": [ + "equepor", + "itvo=asi" + ], + "url.scheme": "https", "user.name": "nimadmin" }, { @@ -4333,8 +4565,8 @@ "rsa.investigations.event_vcat": "ora", "rsa.misc.OS": "ommod", "rsa.misc.action": [ - "cancel", - "ant" + "ant", + "cancel" ], "rsa.misc.category": "rehende", "rsa.misc.client": "rehe", @@ -4378,8 +4610,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "eav", "url.original": "https://www.example.com/iadolo/cidu.txt?aliquide=redolori#eav", - "url.query": "tiu", + "url.path": "/iadolo/cidu.txt", + "url.query": [ + "tiu", + "aliquide=redolori" + ], + "url.scheme": "https", "user.name": "msequin" }, { @@ -4410,13 +4650,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.1789", "related.hosts": [ - "dol", "ono", + "dol", "ntex5135.corp" ], "related.ip": [ - "10.234.171.117", - "10.239.194.105" + "10.239.194.105", + "10.234.171.117" ], "related.user": [ "tat" @@ -4426,8 +4666,8 @@ "rsa.investigations.event_vcat": "uia", "rsa.misc.OS": "mquae", "rsa.misc.action": [ - "deny", - "tenatus" + "tenatus", + "deny" ], "rsa.misc.category": "abo", "rsa.misc.client": "umtota", @@ -4471,8 +4711,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "boreetd", "url.original": "https://internal.example.com/isc/umdol.jpg?atn=sectet#boreetd", - "url.query": "odic", + "url.path": "/isc/umdol.jpg", + "url.query": [ + "odic", + "atn=sectet" + ], + "url.scheme": "https", "user.name": "tat" }, { @@ -4506,8 +4754,8 @@ ], "related.ip": [ "10.249.16.201", - "10.107.168.208", - "10.34.41.75" + "10.34.41.75", + "10.107.168.208" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4632,8 +4880,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "ptat", "url.original": "https://www.example.com/imadm/ugiat.txt?Nequepor=nisiu#ptat", - "url.query": "stiaec", + "url.path": "/imadm/ugiat.txt", + "url.query": [ + "stiaec", + "Nequepor=nisiu" + ], + "url.scheme": "https", "user.name": "ommodoc" }, { @@ -4669,8 +4925,8 @@ "ulamc767.internal.lan" ], "related.ip": [ - "10.112.155.228", - "10.47.191.95" + "10.47.191.95", + "10.112.155.228" ], "related.user": [ "aed" @@ -4680,8 +4936,8 @@ "rsa.investigations.event_vcat": "eiusm", "rsa.misc.OS": "emag", "rsa.misc.action": [ - "deny", - "velillu" + "velillu", + "deny" ], "rsa.misc.category": "litseddo", "rsa.misc.client": "aturE", @@ -4725,8 +4981,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "sed", "url.original": "https://mail.example.net/ionemu/nul.jpg?volupt=ori#sed", - "url.query": "radipi", + "url.path": "/ionemu/nul.jpg", + "url.query": [ + "radipi", + "volupt=ori" + ], + "url.scheme": "https", "user.name": "aed" }, { @@ -4758,8 +5022,8 @@ "metco" ], "related.ip": [ - "10.140.137.17", "10.103.169.94", + "10.140.137.17", "10.62.241.218" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4826,8 +5090,8 @@ ], "related.ip": [ "10.251.212.166", - "10.90.229.92", - "10.77.105.160" + "10.77.105.160", + "10.90.229.92" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4891,8 +5155,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4261", "related.hosts": [ - "dutp", "ipsum", + "dutp", "spici5547.internal.test" ], "related.ip": [ @@ -4907,8 +5171,8 @@ "rsa.investigations.event_vcat": "animi", "rsa.misc.OS": "tisunde", "rsa.misc.action": [ - "aut", - "cancel" + "cancel", + "aut" ], "rsa.misc.category": "lamcorpo", "rsa.misc.client": "com", @@ -4952,8 +5216,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "henderi", "url.original": "https://mail.example.org/asuntex/uovolup.html?amali=uiav#henderi", - "url.query": "eriti", + "url.path": "/asuntex/uovolup.html", + "url.query": [ + "eriti", + "amali=uiav" + ], + "url.scheme": "https", "user.name": "urmag" }, { @@ -5000,8 +5272,8 @@ "rsa.investigations.event_vcat": "uatDuisa", "rsa.misc.OS": "citation", "rsa.misc.action": [ - "utlabore", - "accept" + "accept", + "utlabore" ], "rsa.misc.category": "reeu", "rsa.misc.client": "ntut", @@ -5045,8 +5317,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "dminimve", "url.original": "https://example.org/mnisiut/porinci.htm?norum=emUten#dminimve", - "url.query": "texplica", + "url.path": "/mnisiut/porinci.htm", + "url.query": [ + "texplica", + "norum=emUten" + ], + "url.scheme": "https", "user.name": "lpaquiof" }, { @@ -5078,8 +5358,8 @@ "cons" ], "related.ip": [ - "10.246.41.77", "10.228.61.5", + "10.246.41.77", "10.157.22.21" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5145,8 +5425,8 @@ "llumdo" ], "related.ip": [ - "10.242.119.111", "10.188.131.18", + "10.242.119.111", "10.239.231.168" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5211,8 +5491,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.979", "related.hosts": [ - "lorem", "iden", + "lorem", "tru3812.mail.lan" ], "related.ip": [ @@ -5272,8 +5552,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "gif", + "url.fragment": "rai", "url.original": "https://api.example.net/nsec/smo.gif?etq=trumexe#rai", - "url.query": "ipsu", + "url.path": "/nsec/smo.gif", + "url.query": [ + "ipsu", + "etq=trumexe" + ], + "url.scheme": "https", "user.name": "ainci" }, { @@ -5351,8 +5639,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4342", "related.hosts": [ - "ten", "onsequ", + "ten", "riaturE1644.www5.example" ], "related.ip": [ @@ -5367,8 +5655,8 @@ "rsa.investigations.event_vcat": "empori", "rsa.misc.OS": "ostru", "rsa.misc.action": [ - "allow", - "quepor" + "quepor", + "allow" ], "rsa.misc.category": "cipitla", "rsa.misc.client": "exeacomm", @@ -5412,8 +5700,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "upta", "url.original": "https://internal.example.com/dolo/exeacom.txt?tlab=eufugiat#upta", - "url.query": "Lor", + "url.path": "/dolo/exeacom.txt", + "url.query": [ + "Lor", + "tlab=eufugiat" + ], + "url.scheme": "https", "user.name": "erspici" }, { @@ -5505,8 +5801,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "utod", "url.original": "https://www5.example.org/orumet/aliqu.txt?tion=sun#utod", - "url.query": "con", + "url.path": "/orumet/aliqu.txt", + "url.query": [ + "con", + "tion=sun" + ], + "url.scheme": "https", "user.name": "Sedutp" }, { @@ -5605,9 +5909,9 @@ "caecatcu" ], "related.ip": [ - "10.7.230.206", "10.249.93.150", - "10.154.151.111" + "10.154.151.111", + "10.7.230.206" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5676,8 +5980,8 @@ "itse5466.api.example" ], "related.ip": [ - "10.26.4.3", - "10.217.209.221" + "10.217.209.221", + "10.26.4.3" ], "related.user": [ "ciduntut" @@ -5732,8 +6036,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "gif", + "url.fragment": "iscin", "url.original": "https://internal.example.com/nonnumqu/isciveli.gif?wri=aute#iscin", - "url.query": "eca", + "url.path": "/nonnumqu/isciveli.gif", + "url.query": [ + "eca", + "wri=aute" + ], + "url.scheme": "https", "user.name": "ciduntut" }, { @@ -5780,8 +6092,8 @@ "rsa.investigations.event_vcat": "luptatev", "rsa.misc.OS": "emipsu", "rsa.misc.action": [ - "ido", - "accept" + "accept", + "ido" ], "rsa.misc.category": "litse", "rsa.misc.client": "evita", @@ -5825,8 +6137,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "html", + "url.fragment": "upta", "url.original": "https://www5.example.org/utlabo/scip.html?voluptas=inv#upta", - "url.query": "atnu", + "url.path": "/utlabo/scip.html", + "url.query": [ + "atnu", + "voluptas=inv" + ], + "url.scheme": "https", "user.name": "nulapar" }, { @@ -5857,8 +6177,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2052", "related.hosts": [ - "dat", "asp", + "dat", "amco1592.mail.host" ], "related.ip": [ @@ -5873,8 +6193,8 @@ "rsa.investigations.event_vcat": "atvolupt", "rsa.misc.OS": "riosam", "rsa.misc.action": [ - "ssitasp", - "deny" + "deny", + "ssitasp" ], "rsa.misc.category": "enimadmi", "rsa.misc.client": "uatDui", @@ -5918,8 +6238,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "obeata", "url.original": "https://internal.example.com/ssusci/snostrud.txt?dolo=siutaliq#obeata", - "url.query": "ptat", + "url.path": "/ssusci/snostrud.txt", + "url.query": [ + "ptat", + "dolo=siutaliq" + ], + "url.scheme": "https", "user.name": "moenimi" }, { @@ -5955,8 +6283,8 @@ "dicta7226.mail.example" ], "related.ip": [ - "10.4.244.115", - "10.53.50.77" + "10.53.50.77", + "10.4.244.115" ], "related.user": [ "idolo" @@ -6011,8 +6339,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ptasn", "url.original": "https://api.example.com/temporai/umw.jpg?mveniamq=litsed#ptasn", - "url.query": "lore", + "url.path": "/temporai/umw.jpg", + "url.query": [ + "lore", + "mveniamq=litsed" + ], + "url.scheme": "https", "user.name": "idolo" }, { @@ -6110,8 +6446,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3052", "related.hosts": [ - "xeacom", "tenima", + "xeacom", "pidatatn2627.www.localdomain" ], "related.ip": [ @@ -6126,8 +6462,8 @@ "rsa.investigations.event_vcat": "lauda", "rsa.misc.OS": "enatuser", "rsa.misc.action": [ - "rios", - "accept" + "accept", + "rios" ], "rsa.misc.category": "aUte", "rsa.misc.client": "iusm", @@ -6171,8 +6507,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "umiurere", "url.original": "https://mail.example.org/ngelit/dipiscin.gif?serro=ctet#umiurere", - "url.query": "diconseq", + "url.path": "/ngelit/dipiscin.gif", + "url.query": [ + "diconseq", + "serro=ctet" + ], + "url.scheme": "https", "user.name": "riatur" }, { @@ -6204,9 +6548,9 @@ "nimides" ], "related.ip": [ - "10.53.251.202", + "10.226.255.3", "10.123.59.69", - "10.226.255.3" + "10.53.251.202" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6271,9 +6615,9 @@ "edut" ], "related.ip": [ - "10.3.85.176", + "10.212.56.26", "10.29.141.252", - "10.212.56.26" + "10.3.85.176" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6342,8 +6686,8 @@ "emveleu4029.api.local" ], "related.ip": [ - "10.126.11.186", - "10.236.175.163" + "10.236.175.163", + "10.126.11.186" ], "related.user": [ "udantiu" @@ -6398,8 +6742,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "lestia", "url.original": "https://internal.example.net/inc/riaturEx.htm?mnihilm=itinvo#lestia", - "url.query": "icaboN", + "url.path": "/inc/riaturEx.htm", + "url.query": [ + "icaboN", + "mnihilm=itinvo" + ], + "url.scheme": "https", "user.name": "udantiu" }, { @@ -6431,9 +6783,9 @@ "mes" ], "related.ip": [ - "10.171.60.173", "10.11.150.136", - "10.83.98.220" + "10.83.98.220", + "10.171.60.173" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6498,9 +6850,9 @@ "datatno" ], "related.ip": [ - "10.238.49.73", "10.74.88.209", - "10.92.3.166" + "10.92.3.166", + "10.238.49.73" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6565,8 +6917,8 @@ "ptate" ], "related.ip": [ - "10.119.248.36", "10.187.107.47", + "10.119.248.36", "10.84.200.121" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -6632,9 +6984,9 @@ "tasu" ], "related.ip": [ - "10.135.213.17", "10.167.128.229", - "10.30.239.222" + "10.30.239.222", + "10.135.213.17" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6714,8 +7066,8 @@ "rsa.investigations.event_vcat": "iumdol", "rsa.misc.OS": "min", "rsa.misc.action": [ - "block", - "eleumiur" + "eleumiur", + "block" ], "rsa.misc.category": "ero", "rsa.misc.client": "gia", @@ -6759,8 +7111,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "rExce", "url.original": "https://api.example.net/ita/esse.txt?amquis=iatquovo#rExce", - "url.query": "ididun", + "url.path": "/ita/esse.txt", + "url.query": [ + "ididun", + "amquis=iatquovo" + ], + "url.scheme": "https", "user.name": "emq" }, { @@ -6791,8 +7151,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4195", "related.hosts": [ - "aconse", "Except", + "aconse", "rror3870.www5.local" ], "related.ip": [ @@ -6807,8 +7167,8 @@ "rsa.investigations.event_vcat": "ntexp", "rsa.misc.OS": "aconseq", "rsa.misc.action": [ - "block", - "oluptate" + "oluptate", + "block" ], "rsa.misc.category": "edqu", "rsa.misc.client": "ites", @@ -6852,8 +7212,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "rat", "url.original": "https://www5.example.net/culpa/isun.txt?cola=tura#rat", - "url.query": "orumwrit", + "url.path": "/culpa/isun.txt", + "url.query": [ + "orumwrit", + "cola=tura" + ], + "url.scheme": "https", "user.name": "caecatcu" }, { @@ -6885,9 +7253,9 @@ "emaperi" ], "related.ip": [ + "10.35.240.70", "10.224.212.88", - "10.53.82.96", - "10.35.240.70" + "10.53.82.96" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6952,9 +7320,9 @@ "oeius" ], "related.ip": [ - "10.66.149.234", "10.233.128.7", - "10.186.253.240" + "10.186.253.240", + "10.66.149.234" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7019,9 +7387,9 @@ "irat" ], "related.ip": [ + "10.173.140.201", "10.46.11.114", - "10.227.133.134", - "10.173.140.201" + "10.227.133.134" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7086,9 +7454,9 @@ "emp" ], "related.ip": [ + "10.205.18.11", "10.69.130.207", - "10.170.236.123", - "10.205.18.11" + "10.170.236.123" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7157,8 +7525,8 @@ "velill3821.mail.invalid" ], "related.ip": [ - "10.97.254.192", - "10.124.34.251" + "10.124.34.251", + "10.97.254.192" ], "related.user": [ "epor" @@ -7168,8 +7536,8 @@ "rsa.investigations.event_vcat": "lica", "rsa.misc.OS": "taedi", "rsa.misc.action": [ - "imide", - "deny" + "deny", + "imide" ], "rsa.misc.category": "iurere", "rsa.misc.client": "ollitan", @@ -7213,8 +7581,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "aborios", "url.original": "https://example.com/loremag/mqu.gif?bore=lapari#aborios", - "url.query": "sequa", + "url.path": "/loremag/mqu.gif", + "url.query": [ + "sequa", + "bore=lapari" + ], + "url.scheme": "https", "user.name": "epor" }, { @@ -7361,8 +7737,8 @@ ], "related.ip": [ "10.35.84.125", - "10.37.120.29", - "10.212.208.70" + "10.212.208.70", + "10.37.120.29" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7427,8 +7803,8 @@ "aturvel" ], "related.ip": [ - "10.143.65.84", "10.199.201.26", + "10.143.65.84", "10.207.207.106" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7494,8 +7870,8 @@ "gni" ], "related.ip": [ - "10.163.236.253", "10.41.61.88", + "10.163.236.253", "10.204.27.48" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7561,8 +7937,8 @@ "dents" ], "related.ip": [ - "10.185.44.26", "10.246.81.164", + "10.185.44.26", "10.53.110.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7628,8 +8004,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.802", "related.hosts": [ - "lam", "proid", + "lam", "cupida6106.www5.local" ], "related.ip": [ @@ -7689,8 +8065,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ido", "url.original": "https://api.example.com/aco/empo.jpg?iumdol=iusm#ido", - "url.query": "ipsamv", + "url.path": "/aco/empo.jpg", + "url.query": [ + "ipsamv", + "iumdol=iusm" + ], + "url.scheme": "https", "user.name": "aquaeab" }, { @@ -7726,8 +8110,8 @@ "unt2122.internal.local" ], "related.ip": [ - "10.38.18.72", - "10.202.250.141" + "10.202.250.141", + "10.38.18.72" ], "related.user": [ "maperia" @@ -7737,8 +8121,8 @@ "rsa.investigations.event_vcat": "rure", "rsa.misc.OS": "iquidexe", "rsa.misc.action": [ - "allow", - "volu" + "volu", + "allow" ], "rsa.misc.category": "ium", "rsa.misc.client": "liquip", @@ -7782,8 +8166,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "ect", "url.original": "https://example.org/uidolor/turve.htm?temporai=uasiarch#ect", - "url.query": "quatDui", + "url.path": "/uidolor/turve.htm", + "url.query": [ + "quatDui", + "temporai=uasiarch" + ], + "url.scheme": "https", "user.name": "maperia" }, { @@ -7814,13 +8206,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.4674", "related.hosts": [ - "oremeu", "ita", + "oremeu", "luptat2613.internal.localhost" ], "related.ip": [ - "10.182.124.88", - "10.139.144.75" + "10.139.144.75", + "10.182.124.88" ], "related.user": [ "modo" @@ -7875,8 +8267,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "odtempo", "url.original": "https://www.example.net/Maloru/lapariat.htm?tlabori=rehender#odtempo", - "url.query": "isiutali", + "url.path": "/Maloru/lapariat.htm", + "url.query": [ + "isiutali", + "tlabori=rehender" + ], + "url.scheme": "https", "user.name": "modo" }, { @@ -7912,8 +8312,8 @@ "neavo4796.internal.domain" ], "related.ip": [ - "10.188.124.185", - "10.35.10.19" + "10.35.10.19", + "10.188.124.185" ], "related.user": [ "dolo" @@ -7923,8 +8323,8 @@ "rsa.investigations.event_vcat": "olupt", "rsa.misc.OS": "rumw", "rsa.misc.action": [ - "tali", - "block" + "block", + "tali" ], "rsa.misc.category": "itsedq", "rsa.misc.client": "esciu", @@ -7968,8 +8368,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "html", + "url.fragment": "one", "url.original": "https://www5.example.com/xea/ssecill.html?quianonn=quun#one", - "url.query": "tasper", + "url.path": "/xea/ssecill.html", + "url.query": [ + "tasper", + "quianonn=quun" + ], + "url.scheme": "https", "user.name": "dolo" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json b/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json index 23436a2de5fc..1e9362659942 100644 --- a/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json +++ b/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json @@ -50,8 +50,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" @@ -107,8 +109,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" @@ -219,8 +223,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" diff --git a/x-pack/filebeat/module/google_workspace/ingest/common.yml b/x-pack/filebeat/module/google_workspace/ingest/common.yml index ec7a9f8bbcfc..d9ee4e9f86c6 100644 --- a/x-pack/filebeat/module/google_workspace/ingest/common.yml +++ b/x-pack/filebeat/module/google_workspace/ingest/common.yml @@ -1,5 +1,9 @@ description: Pipeline for parsing google_workspace logs processors: + - uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null - geoip: field: source.ip target_field: source.geo diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index aab537751715..e37b497a0ef4 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json index 749ef4301f2f..271e0db04a32 100644 --- a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json +++ b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json @@ -23,13 +23,13 @@ "radipis5408.mail.local" ], "related.ip": [ - "10.70.155.35", - "10.81.122.126" + "10.81.122.126", + "10.70.155.35" ], "related.user": [ + "tatno", "aqui", - "magn", - "tatno" + "magn" ], "rsa.counters.dclass_c1": 5910, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -117,8 +117,8 @@ ], "related.user": [ "qua", - "uradi", - "temUten" + "temUten", + "uradi" ], "rsa.counters.dclass_c1": 3626, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -170,13 +170,13 @@ "elaudant5931.internal.invalid" ], "related.ip": [ - "10.232.27.250", - "10.18.124.28" + "10.18.124.28", + "10.232.27.250" ], "related.user": [ - "modocons", + "lapariat", "mquidol", - "lapariat" + "modocons" ], "rsa.counters.dclass_c1": 6564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -234,13 +234,13 @@ "amest4147.mail.host" ], "related.ip": [ - "10.197.250.10", - "10.6.137.200" + "10.6.137.200", + "10.197.250.10" ], "related.user": [ "oluptas", - "intoc", - "occae" + "occae", + "intoc" ], "rsa.counters.event_counter": 7243, "rsa.db.database": "tNequepo", @@ -273,8 +273,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "eumfug", "url.original": "https://internal.example.net/namali/taevit.html?nsecte=itame#eumfug", - "url.query": "lit", + "url.path": "/namali/taevit.html", + "url.query": [ + "lit", + "nsecte=itame" + ], + "url.scheme": "https", "user.name": "occae" }, { @@ -308,16 +316,16 @@ ], "related.user": [ "ncidid", - "reme", - "acommod" + "acommod", + "reme" ], "rsa.counters.event_counter": 2462, "rsa.db.database": "uaUteni", "rsa.internal.event_desc": "osqui", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "accept", - "lamcolab" + "lamcolab", + "accept" ], "rsa.misc.category": "xerc", "rsa.misc.disposition": "iutali", @@ -342,8 +350,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "giatnu", "url.original": "https://example.net/tlabo/uames.gif?mpo=offi#giatnu", - "url.query": "ulapa", + "url.path": "/tlabo/uames.gif", + "url.query": [ + "ulapa", + "mpo=offi" + ], + "url.scheme": "https", "user.name": "ncidid" }, { @@ -370,13 +386,13 @@ "didunt1355.corp" ], "related.ip": [ - "10.129.149.43", - "10.211.105.204" + "10.211.105.204", + "10.129.149.43" ], "related.user": [ "labor", - "eveli", - "orema" + "orema", + "eveli" ], "rsa.counters.dclass_c1": 6855, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -432,13 +448,13 @@ "pora6854.www5.home" ], "related.ip": [ - "10.214.191.180", - "10.112.250.193" + "10.112.250.193", + "10.214.191.180" ], "related.user": [ - "ipsumdol", + "ide", "Exc", - "ide" + "ipsumdol" ], "rsa.counters.dclass_c1": 6852, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -498,8 +514,8 @@ ], "related.user": [ "ovol", - "iquipe", - "tnonpro" + "tnonpro", + "iquipe" ], "rsa.counters.dclass_c1": 3645, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -551,13 +567,13 @@ "ptasnu6684.mail.lan" ], "related.ip": [ - "10.59.138.212", - "10.74.105.218" + "10.74.105.218", + "10.59.138.212" ], "related.user": [ - "idunt", "archite", - "boree" + "boree", + "idunt" ], "rsa.counters.dclass_c1": 248, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -613,13 +629,13 @@ "rinre2977.api.corp" ], "related.ip": [ - "10.230.173.4", - "10.168.159.13" + "10.168.159.13", + "10.230.173.4" ], "related.user": [ + "inci", "atemq", - "isnostr", - "inci" + "isnostr" ], "rsa.counters.dclass_c1": 6135, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -675,12 +691,12 @@ "atevelit2450.local" ], "related.ip": [ - "10.41.21.204", - "10.49.167.57" + "10.49.167.57", + "10.41.21.204" ], "related.user": [ - "sau", "tali", + "sau", "ccaeca" ], "rsa.counters.dclass_c1": 6818, @@ -739,12 +755,12 @@ "itla658.api.localhost" ], "related.ip": [ - "10.62.147.186", - "10.216.125.252" + "10.216.125.252", + "10.62.147.186" ], "related.user": [ - "llamco", "dolore", + "llamco", "lorsita" ], "rsa.counters.event_counter": 4603, @@ -752,8 +768,8 @@ "rsa.internal.event_desc": "aquae", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "accept", - "quasia" + "quasia", + "accept" ], "rsa.misc.category": "boreetdo", "rsa.misc.disposition": "aturve", @@ -779,8 +795,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "elit", "url.original": "https://example.org/umwrit/uptate.html?ctetura=aveni#elit", - "url.query": "seosqui", + "url.path": "/umwrit/uptate.html", + "url.query": [ + "seosqui", + "ctetura=aveni" + ], + "url.scheme": "https", "user.name": "llamco" }, { @@ -848,8 +872,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "mvele", "url.original": "https://api.example.org/roidents/tem.txt?tametcon=liqua#mvele", - "url.query": "isis", + "url.path": "/roidents/tem.txt", + "url.query": [ + "isis", + "tametcon=liqua" + ], + "url.scheme": "https", "user.name": "rum" }, { @@ -879,8 +911,8 @@ "10.34.148.166" ], "related.user": [ - "icabo", "untutlab", + "icabo", "miu" ], "rsa.counters.dclass_c1": 5427, @@ -937,9 +969,9 @@ "10.134.5.40" ], "related.user": [ + "siu", "licabo", - "conse", - "siu" + "conse" ], "rsa.counters.dclass_c1": 6356, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -999,9 +1031,9 @@ "10.30.98.10" ], "related.user": [ - "olori", "velite", - "dipisci" + "dipisci", + "olori" ], "rsa.counters.dclass_c1": 7717, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1057,13 +1089,13 @@ "nimid372.api.corp" ], "related.ip": [ - "10.190.10.219", - "10.233.120.207" + "10.233.120.207", + "10.190.10.219" ], "related.user": [ - "quamnih", + "item", "accusant", - "item" + "quamnih" ], "rsa.counters.dclass_c1": 3278, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1147,13 +1179,13 @@ "maliquam2147.internal.home" ], "related.ip": [ - "10.100.98.56", - "10.248.184.200" + "10.248.184.200", + "10.100.98.56" ], "related.user": [ - "proident", "boru", - "ritati" + "ritati", + "proident" ], "rsa.counters.dclass_c1": 5923, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1213,8 +1245,8 @@ "10.82.28.220" ], "related.user": [ - "oluptat", "aecatcup", + "oluptat", "dtempo" ], "rsa.counters.dclass_c1": 3071, @@ -1275,9 +1307,9 @@ "10.167.252.183" ], "related.user": [ - "redol", + "ationul", "asnu", - "ationul" + "redol" ], "rsa.counters.dclass_c1": 6606, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1348,8 +1380,8 @@ "rsa.internal.event_desc": "iae", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "illu", - "deny" + "deny", + "illu" ], "rsa.misc.category": "quido", "rsa.misc.disposition": "emip", @@ -1375,8 +1407,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "tetur", "url.original": "https://mail.example.com/lorsi/repreh.gif?sitamet=utlabo#tetur", - "url.query": "tionula", + "url.path": "/lorsi/repreh.gif", + "url.query": [ + "tionula", + "sitamet=utlabo" + ], + "url.scheme": "https", "user.name": "undeomni" }, { @@ -1403,13 +1443,13 @@ "dolorem6882.api.local" ], "related.ip": [ - "10.29.119.245", - "10.214.3.140" + "10.214.3.140", + "10.29.119.245" ], "related.user": [ - "taliqui", "scipitl", - "edolorin" + "edolorin", + "taliqui" ], "rsa.counters.dclass_c1": 5140, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1467,8 +1507,8 @@ "temaccu5302.test" ], "related.ip": [ - "10.218.123.234", - "10.110.133.7" + "10.110.133.7", + "10.218.123.234" ], "related.user": [ "caboNem", @@ -1480,8 +1520,8 @@ "rsa.internal.event_desc": "liquid", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "vitaed", - "allow" + "allow", + "vitaed" ], "rsa.misc.category": "enim", "rsa.misc.disposition": "Finibus", @@ -1507,8 +1547,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "idolo", "url.original": "https://example.org/etconsec/elillum.htm?mporinc=onsectet#idolo", - "url.query": "atemUte", + "url.path": "/etconsec/elillum.htm", + "url.query": [ + "atemUte", + "mporinc=onsectet" + ], + "url.scheme": "https", "user.name": "pta" }, { @@ -1540,8 +1588,8 @@ ], "related.user": [ "doeiu", - "mquisn", - "litan" + "litan", + "mquisn" ], "rsa.counters.dclass_c1": 3474, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1603,9 +1651,9 @@ "10.123.166.197" ], "related.user": [ - "min", "emUte", - "liquam" + "liquam", + "min" ], "rsa.counters.event_counter": 7102, "rsa.db.database": "oluptat", @@ -1638,8 +1686,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "remipsu", "url.original": "https://internal.example.net/atnula/ditautf.jpg?iquidex=olup#remipsu", - "url.query": "tan", + "url.path": "/atnula/ditautf.jpg", + "url.query": [ + "tan", + "iquidex=olup" + ], + "url.scheme": "https", "user.name": "emUte" }, { @@ -1732,8 +1788,8 @@ "10.58.133.175" ], "related.user": [ - "nde", "oco", + "nde", "mfu" ], "rsa.counters.dclass_c1": 3795, @@ -1794,9 +1850,9 @@ "10.70.29.203" ], "related.user": [ - "pta", "mquisnos", - "veniamq" + "veniamq", + "pta" ], "rsa.counters.dclass_c1": 2358, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1852,13 +1908,13 @@ "lesti6939.api.local" ], "related.ip": [ - "10.137.85.123", - "10.165.182.111" + "10.165.182.111", + "10.137.85.123" ], "related.user": [ - "ames", + "Bonorum", "sis", - "Bonorum" + "ames" ], "rsa.counters.dclass_c1": 6401, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1944,21 +2000,21 @@ "upt6017.api.localdomain" ], "related.ip": [ - "10.173.178.109", - "10.64.184.196" + "10.64.184.196", + "10.173.178.109" ], "related.user": [ - "tam", "nesci", - "uian" + "uian", + "tam" ], "rsa.counters.event_counter": 4493, "rsa.db.database": "sin", "rsa.internal.event_desc": "orin", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "lamco", - "block" + "block", + "lamco" ], "rsa.misc.category": "enia", "rsa.misc.disposition": "iavol", @@ -1984,8 +2040,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "madmi", "url.original": "https://www.example.net/hender/ptatemU.htm?mquisnos=tnulapa#madmi", - "url.query": "tlabore", + "url.path": "/hender/ptatemU.htm", + "url.query": [ + "tlabore", + "mquisnos=tnulapa" + ], + "url.scheme": "https", "user.name": "uian" }, { @@ -2012,13 +2076,13 @@ "turQuis4046.api.test" ], "related.ip": [ - "10.90.50.149", - "10.168.225.209" + "10.168.225.209", + "10.90.50.149" ], "related.user": [ + "olupta", "olu", - "aUtenima", - "olupta" + "aUtenima" ], "rsa.counters.dclass_c1": 1127, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2074,13 +2138,13 @@ "con6049.internal.lan" ], "related.ip": [ - "10.18.150.82", - "10.59.182.36" + "10.59.182.36", + "10.18.150.82" ], "related.user": [ - "luptat", + "mtota", "qua", - "mtota" + "luptat" ], "rsa.counters.dclass_c1": 6112, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2167,9 +2231,9 @@ "10.151.240.35" ], "related.user": [ + "lam", "ama", - "ametcons", - "lam" + "ametcons" ], "rsa.counters.dclass_c1": 4325, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2221,13 +2285,13 @@ "tium3542.internal.invalid" ], "related.ip": [ - "10.242.48.203", - "10.147.142.242" + "10.147.142.242", + "10.242.48.203" ], "related.user": [ - "ese", + "quasi", "quisn", - "quasi" + "ese" ], "rsa.counters.dclass_c1": 3970, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2285,13 +2349,13 @@ "radipis3991.mail.invalid" ], "related.ip": [ - "10.213.165.165", - "10.254.10.98" + "10.254.10.98", + "10.213.165.165" ], "related.user": [ - "ttenb", + "civeli", "eufugia", - "civeli" + "ttenb" ], "rsa.counters.event_counter": 7365, "rsa.db.database": "utlabore", @@ -2325,8 +2389,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "temqui", "url.original": "https://example.net/itati/oidentsu.gif?eporroqu=aturve#temqui", - "url.query": "lup", + "url.path": "/itati/oidentsu.gif", + "url.query": [ + "lup", + "eporroqu=aturve" + ], + "url.scheme": "https", "user.name": "ttenb" }, { @@ -2383,8 +2455,8 @@ "ihi7294.www5.localhost" ], "related.ip": [ - "10.169.28.157", - "10.116.1.130" + "10.116.1.130", + "10.169.28.157" ], "related.user": [ "eturadip", @@ -2423,8 +2495,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "pteurs", "url.original": "https://api.example.org/uamqua/Neq.gif?eumiu=nim#pteurs", - "url.query": "ercitati", + "url.path": "/uamqua/Neq.gif", + "url.query": [ + "ercitati", + "eumiu=nim" + ], + "url.scheme": "https", "user.name": "eturadip" }, { @@ -2451,13 +2531,13 @@ "caecat4920.api.host" ], "related.ip": [ - "10.29.138.31", - "10.45.69.152" + "10.45.69.152", + "10.29.138.31" ], "related.user": [ - "volupta", "umq", - "tsunt" + "tsunt", + "volupta" ], "rsa.counters.dclass_c1": 744, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2513,13 +2593,13 @@ "setquas6188.internal.local" ], "related.ip": [ - "10.100.113.11", - "10.152.213.228" + "10.152.213.228", + "10.100.113.11" ], "related.user": [ "itationu", - "velillum", - "ptatev" + "ptatev", + "velillum" ], "rsa.counters.dclass_c1": 7245, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2603,8 +2683,8 @@ "nibusBo3674.www5.localhost" ], "related.ip": [ - "10.248.102.129", - "10.208.33.55" + "10.208.33.55", + "10.248.102.129" ], "related.user": [ "inimv", @@ -2669,8 +2749,8 @@ "10.203.164.132" ], "related.user": [ - "ectobea", "mporin", + "ectobea", "ibus" ], "rsa.counters.dclass_c1": 547, @@ -2727,12 +2807,12 @@ "idents7231.mail.home" ], "related.ip": [ - "10.117.81.75", - "10.151.203.60" + "10.151.203.60", + "10.117.81.75" ], "related.user": [ - "dol", "iconsequ", + "dol", "exeac" ], "rsa.counters.dclass_c1": 484, @@ -2789,13 +2869,13 @@ "tat50.mail.host" ], "related.ip": [ - "10.45.152.205", - "10.224.217.153" + "10.224.217.153", + "10.45.152.205" ], "related.user": [ + "utlabo", "eriti", - "imav", - "utlabo" + "imav" ], "rsa.counters.dclass_c1": 922, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2856,8 +2936,8 @@ "10.1.193.187" ], "related.user": [ - "adipis", "hite", + "adipis", "ugi" ], "rsa.counters.event_counter": 508, @@ -2865,8 +2945,8 @@ "rsa.internal.event_desc": "epteurs", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "taevitae", - "allow" + "allow", + "taevitae" ], "rsa.misc.category": "itse", "rsa.misc.disposition": "rever", @@ -2892,8 +2972,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "emipsumq", "url.original": "https://api.example.net/quam/saute.htm?nostru=docons#emipsumq", - "url.query": "orinr", + "url.path": "/quam/saute.htm", + "url.query": [ + "orinr", + "nostru=docons" + ], + "url.scheme": "https", "user.name": "ugi" }, { @@ -2923,9 +3011,9 @@ "10.248.244.203" ], "related.user": [ - "mquamei", "sum", - "eiusm" + "eiusm", + "mquamei" ], "rsa.counters.dclass_c1": 3058, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2982,8 +3070,8 @@ ], "related.user": [ "ine", - "consecte", - "nimv" + "nimv", + "consecte" ], "rsa.counters.dclass_c1": 2771, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3039,12 +3127,12 @@ "agnama5013.internal.example" ], "related.ip": [ - "10.201.223.119", - "10.204.223.184" + "10.204.223.184", + "10.201.223.119" ], "related.user": [ - "teni", "rcit", + "teni", "tuserror" ], "rsa.counters.dclass_c1": 4113, @@ -3101,13 +3189,13 @@ "edictas4693.home" ], "related.ip": [ - "10.200.12.126", - "10.223.56.33" + "10.223.56.33", + "10.200.12.126" ], "related.user": [ + "Nequepo", "elitsedd", - "magnido", - "Nequepo" + "magnido" ], "rsa.counters.dclass_c1": 3243, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3170,8 +3258,8 @@ ], "related.user": [ "tuserror", - "emquel", - "citation" + "citation", + "emquel" ], "rsa.counters.event_counter": 2513, "rsa.db.database": "rspiciat", @@ -3205,8 +3293,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "asia", "url.original": "https://internal.example.com/imide/uiineav.htm?lloinve=eni#asia", - "url.query": "edquiac", + "url.path": "/imide/uiineav.htm", + "url.query": [ + "edquiac", + "lloinve=eni" + ], + "url.scheme": "https", "user.name": "tuserror" }, { @@ -3232,13 +3328,13 @@ "tsun7120.home" ], "related.ip": [ - "10.191.184.105", - "10.65.174.196" + "10.65.174.196", + "10.191.184.105" ], "related.user": [ - "tione", + "iin", "uta", - "iin" + "tione" ], "rsa.counters.dclass_c1": 5836, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3292,12 +3388,12 @@ "lumquid6940.mail.localdomain" ], "related.ip": [ - "10.41.181.179", - "10.224.148.48" + "10.224.148.48", + "10.41.181.179" ], "related.user": [ - "niam", "iosamn", + "niam", "equepor" ], "rsa.counters.event_counter": 7468, @@ -3305,8 +3401,8 @@ "rsa.internal.event_desc": "ibusB", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "deny", - "rumwr" + "rumwr", + "deny" ], "rsa.misc.category": "rporis", "rsa.misc.disposition": "etco", @@ -3332,8 +3428,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iumd", "url.original": "https://api.example.com/ptatem/mporain.gif?corpo=commod#iumd", - "url.query": "ntore", + "url.path": "/ptatem/mporain.gif", + "url.query": [ + "ntore", + "corpo=commod" + ], + "url.scheme": "https", "user.name": "niam" }, { @@ -3360,13 +3464,13 @@ "amcorp7299.api.example" ], "related.ip": [ - "10.21.208.103", - "10.21.61.134" + "10.21.61.134", + "10.21.208.103" ], "related.user": [ "ostr", - "imidest", - "mipsa" + "mipsa", + "imidest" ], "rsa.counters.dclass_c1": 7766, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3486,21 +3590,21 @@ "tionevol3157.mail.invalid" ], "related.ip": [ - "10.240.62.238", - "10.191.142.143" + "10.191.142.143", + "10.240.62.238" ], "related.user": [ - "animide", + "nofde", "modtempo", - "nofde" + "animide" ], "rsa.counters.event_counter": 7580, "rsa.db.database": "Lore", "rsa.internal.event_desc": "nto", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "cancel", - "ali" + "ali", + "cancel" ], "rsa.misc.category": "sciv", "rsa.misc.disposition": "tlabo", @@ -3525,8 +3629,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "onsequa", "url.original": "https://www5.example.net/aUten/edutpers.gif?apariatu=mnisis#onsequa", - "url.query": "sunt", + "url.path": "/aUten/edutpers.gif", + "url.query": [ + "sunt", + "apariatu=mnisis" + ], + "url.scheme": "https", "user.name": "modtempo" }, { @@ -3559,8 +3671,8 @@ "10.178.79.217" ], "related.user": [ - "tqui", "inibusBo", + "tqui", "ccusan" ], "rsa.counters.event_counter": 3538, @@ -3594,8 +3706,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "iaeco", "url.original": "https://www.example.net/mvolup/pidat.jpg?ents=nsec#iaeco", - "url.query": "ommodoco", + "url.path": "/mvolup/pidat.jpg", + "url.query": [ + "ommodoco", + "ents=nsec" + ], + "url.scheme": "https", "user.name": "ccusan" }, { @@ -3687,9 +3807,9 @@ "10.186.133.184" ], "related.user": [ - "boriosa", "sci", - "acons" + "acons", + "boriosa" ], "rsa.counters.dclass_c1": 1578, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3744,9 +3864,9 @@ "10.160.147.230" ], "related.user": [ + "nimvenia", "ndeomnis", - "illoin", - "nimvenia" + "illoin" ], "rsa.counters.dclass_c1": 5988, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3860,13 +3980,13 @@ "itte6905.mail.invalid" ], "related.ip": [ - "10.249.13.159", - "10.108.130.106" + "10.108.130.106", + "10.249.13.159" ], "related.user": [ - "uisautei", "exeacomm", - "colab" + "colab", + "uisautei" ], "rsa.counters.dclass_c1": 1044, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3924,13 +4044,13 @@ "caboNemo274.www.host" ], "related.ip": [ - "10.64.94.174", - "10.39.244.49" + "10.39.244.49", + "10.64.94.174" ], "related.user": [ - "estiae", + "iunt", "Sedut", - "iunt" + "estiae" ], "rsa.counters.event_counter": 7128, "rsa.db.database": "eFinibu", @@ -3963,8 +4083,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "rautod", "url.original": "https://internal.example.org/aev/uovolup.txt?aqueip=aqueip#rautod", - "url.query": "tur", + "url.path": "/aev/uovolup.txt", + "url.query": [ + "tur", + "aqueip=aqueip" + ], + "url.scheme": "https", "user.name": "Sedut" }, { @@ -4047,13 +4175,13 @@ "qui5978.api.test" ], "related.ip": [ - "10.115.203.143", - "10.134.135.22" + "10.134.135.22", + "10.115.203.143" ], "related.user": [ - "orpori", "involu", - "utoditau" + "utoditau", + "orpori" ], "rsa.counters.dclass_c1": 7868, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4109,12 +4237,12 @@ "iamq2577.internal.corp" ], "related.ip": [ - "10.251.212.166", - "10.43.244.252" + "10.43.244.252", + "10.251.212.166" ], "related.user": [ - "inculp", "gnido", + "inculp", "uptat" ], "rsa.counters.dclass_c1": 6947, @@ -4199,13 +4327,13 @@ "usB4127.localhost" ], "related.ip": [ - "10.20.231.188", - "10.88.189.164" + "10.88.189.164", + "10.20.231.188" ], "related.user": [ - "mqu", + "uatDuisa", "tesseq", - "uatDuisa" + "mqu" ], "rsa.counters.dclass_c1": 1623, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4289,8 +4417,8 @@ "abor3266.mail.home" ], "related.ip": [ - "10.225.11.197", - "10.231.77.26" + "10.231.77.26", + "10.225.11.197" ], "related.user": [ "rehe", @@ -4353,9 +4481,9 @@ "10.106.166.105" ], "related.user": [ + "olupt", "avolup", - "usa", - "olupt" + "usa" ], "rsa.counters.dclass_c1": 2658, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4411,9 +4539,9 @@ "10.57.169.205" ], "related.user": [ - "ipsu", + "ctas", "iuta", - "ctas" + "ipsu" ], "rsa.counters.dclass_c1": 392, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4473,9 +4601,9 @@ "10.129.234.200" ], "related.user": [ + "tisundeo", "dquia", - "tevelit", - "tisundeo" + "tevelit" ], "rsa.counters.dclass_c1": 6709, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4531,13 +4659,13 @@ "ididu5928.www5.local" ], "related.ip": [ - "10.111.132.221", - "10.76.121.224" + "10.76.121.224", + "10.111.132.221" ], "related.user": [ - "scive", "ali", - "oloremi" + "oloremi", + "scive" ], "rsa.counters.dclass_c1": 6155, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4598,8 +4726,8 @@ ], "related.user": [ "enimip", - "dolo", - "ota" + "ota", + "dolo" ], "rsa.counters.dclass_c1": 469, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4655,12 +4783,12 @@ "ssusc1892.internal.host" ], "related.ip": [ - "10.179.60.167", - "10.173.13.179" + "10.173.13.179", + "10.179.60.167" ], "related.user": [ - "apar", "isn", + "apar", "ptasn" ], "rsa.counters.dclass_c1": 758, @@ -4717,13 +4845,13 @@ "iatisund424.mail.localdomain" ], "related.ip": [ - "10.42.135.34", - "10.178.190.123" + "10.178.190.123", + "10.42.135.34" ], "related.user": [ - "orsi", + "ore", "tiset", - "ore" + "orsi" ], "rsa.counters.dclass_c1": 2290, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4811,9 +4939,9 @@ "10.207.198.239" ], "related.user": [ + "aUteni", "Loremips", - "incididu", - "aUteni" + "incididu" ], "rsa.counters.dclass_c1": 3043, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4873,8 +5001,8 @@ ], "related.user": [ "nseq", - "litesseq", - "oNe" + "oNe", + "litesseq" ], "rsa.counters.dclass_c1": 3218, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4926,13 +5054,13 @@ "nibusBo1864.domain" ], "related.ip": [ - "10.253.127.130", - "10.86.180.150" + "10.86.180.150", + "10.253.127.130" ], "related.user": [ - "itasper", + "etconsec", "mnisis", - "etconsec" + "itasper" ], "rsa.counters.dclass_c1": 4564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5028,8 +5156,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "amnisiu", "url.original": "https://internal.example.com/architec/incul.txt?aborios=mco#amnisiu", - "url.query": "suntincu", + "url.path": "/architec/incul.txt", + "url.query": [ + "suntincu", + "aborios=mco" + ], + "url.scheme": "https", "user.name": "dolo" }, { @@ -5088,9 +5224,9 @@ "10.248.16.82" ], "related.user": [ - "ditautf", + "tuserror", "res", - "tuserror" + "ditautf" ], "rsa.counters.dclass_c1": 4367, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5150,9 +5286,9 @@ "10.173.19.140" ], "related.user": [ - "orsi", + "Except", "olo", - "Except" + "orsi" ], "rsa.counters.dclass_c1": 5844, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5211,8 +5347,8 @@ "10.69.5.227" ], "related.user": [ - "rumw", "doloreme", + "rumw", "ntocc" ], "rsa.counters.dclass_c1": 5201, @@ -5269,9 +5405,9 @@ "10.253.175.129" ], "related.user": [ - "ate", + "epteurs", "nrep", - "epteurs" + "ate" ], "rsa.counters.dclass_c1": 6260, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5333,9 +5469,9 @@ "10.89.26.170" ], "related.user": [ + "orumetMa", "atus", - "aboris", - "orumetMa" + "aboris" ], "rsa.counters.event_counter": 5863, "rsa.db.database": "inventor", @@ -5368,8 +5504,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "edut", "url.original": "https://www.example.com/sit/ugi.gif?sitametc=rur#edut", - "url.query": "sitametc", + "url.path": "/sit/ugi.gif", + "url.query": [ + "sitametc", + "sitametc=rur" + ], + "url.scheme": "https", "user.name": "aboris" }, { @@ -5402,9 +5546,9 @@ "10.81.108.232" ], "related.user": [ - "neavolup", + "uaturve", "aco", - "uaturve" + "neavolup" ], "rsa.counters.event_counter": 5098, "rsa.db.database": "lapa", @@ -5438,8 +5582,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "amrem", "url.original": "https://internal.example.net/obeatae/sedqui.jpg?nulap=onseq#amrem", - "url.query": "plicab", + "url.path": "/obeatae/sedqui.jpg", + "url.query": [ + "plicab", + "nulap=onseq" + ], + "url.scheme": "https", "user.name": "aco" }, { @@ -5468,12 +5620,12 @@ "par3605.internal.localdomain" ], "related.ip": [ - "10.223.10.28", - "10.230.48.97" + "10.230.48.97", + "10.223.10.28" ], "related.user": [ - "untex", "erit", + "untex", "usmodte" ], "rsa.counters.event_counter": 4029, @@ -5481,8 +5633,8 @@ "rsa.internal.event_desc": "itatiset", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "tconse", - "deny" + "deny", + "tconse" ], "rsa.misc.category": "uaerat", "rsa.misc.disposition": "met", @@ -5508,8 +5660,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "itinvo", "url.original": "https://mail.example.com/iaturE/inc.htm?uisaut=mnihilm#itinvo", - "url.query": "lestia", + "url.path": "/iaturE/inc.htm", + "url.query": [ + "lestia", + "uisaut=mnihilm" + ], + "url.scheme": "https", "user.name": "erit" }, { @@ -5536,13 +5696,13 @@ "isau4356.www.home" ], "related.ip": [ - "10.161.212.150", - "10.115.42.231" + "10.115.42.231", + "10.161.212.150" ], "related.user": [ - "sequamn", "tasnul", - "res" + "res", + "sequamn" ], "rsa.counters.dclass_c1": 4846, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5600,13 +5760,13 @@ "labo3477.www5.domain" ], "related.ip": [ - "10.247.108.144", - "10.226.75.20" + "10.226.75.20", + "10.247.108.144" ], "related.user": [ - "fugia", "maccusan", - "tema" + "tema", + "fugia" ], "rsa.counters.event_counter": 3711, "rsa.db.database": "psa", @@ -5640,8 +5800,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "eip", "url.original": "https://www5.example.org/elaud/temsequ.htm?dolo=iciatisu#eip", - "url.query": "iquaUte", + "url.path": "/elaud/temsequ.htm", + "url.query": [ + "iquaUte", + "dolo=iciatisu" + ], + "url.scheme": "https", "user.name": "tema" }, { @@ -5671,9 +5839,9 @@ "10.192.15.65" ], "related.user": [ - "rExcep", "illumd", - "nimides" + "nimides", + "rExcep" ], "rsa.counters.dclass_c1": 4173, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5727,12 +5895,12 @@ "duntutl3396.api.host" ], "related.ip": [ - "10.116.76.161", - "10.197.254.133" + "10.197.254.133", + "10.116.76.161" ], "related.user": [ - "idu", "ide", + "idu", "trudex" ], "rsa.counters.event_counter": 2608, @@ -5766,8 +5934,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "ema", "url.original": "https://mail.example.net/teturad/nimide.htm?ueporroq=writ#ema", - "url.query": "ioffici", + "url.path": "/teturad/nimide.htm", + "url.query": [ + "ioffici", + "ueporroq=writ" + ], + "url.scheme": "https", "user.name": "ide" }, { @@ -5794,8 +5970,8 @@ "colabo6686.internal.invalid" ], "related.ip": [ - "10.28.77.79", - "10.144.14.15" + "10.144.14.15", + "10.28.77.79" ], "related.user": [ "rspic", @@ -5855,13 +6031,13 @@ "tsunti1164.www.example" ], "related.ip": [ - "10.18.15.43", - "10.248.177.182" + "10.248.177.182", + "10.18.15.43" ], "related.user": [ - "quaturve", "quei", - "caecat" + "caecat", + "quaturve" ], "rsa.counters.dclass_c1": 983, "rsa.counters.dclass_c1_str": "Affected Rows", diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index b04d4b5d67fc..aa8e3f976928 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -232,6 +232,10 @@ processors: ignore_missing: true if: ctx?.json?.evidence?.url != null +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS Host Mapping ## ###################### diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index ae33c77d6d5e..5103d12de9df 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -221,7 +221,10 @@ processors: target_field: url.full ignore_missing: true if: ctx?.json?.entities?.url != null - +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS User Mapping ## ###################### diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 8a25a657a0a9..22f32138c275 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json index 7e500a270946..66df7548ac2b 100644 --- a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json +++ b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json @@ -323,7 +323,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.net/uam/untutl.jpg?llu=uptassi#tamremap" + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "tamremap", + "url.original": "https://mail.example.net/uam/untutl.jpg?llu=uptassi#tamremap", + "url.path": "/uam/untutl.jpg", + "url.query": "llu=uptassi", + "url.scheme": "https" }, { "event.action": "Script mitigation", @@ -365,8 +371,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.155.162.162", - "10.66.171.247" + "10.66.171.247", + "10.155.162.162" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -379,7 +385,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.org/seq/olorema.jpg?quid=fug#uatDuis" + "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "uatDuis", + "url.original": "https://www5.example.org/seq/olorema.jpg?quid=fug#uatDuis", + "url.path": "/seq/olorema.jpg", + "url.query": "quid=fug", + "url.scheme": "https" }, { "event.action": "Fault Occured", @@ -458,7 +470,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://example.org/isiu/nimadmi.gif?ari=equun#suntinc" + "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "suntinc", + "url.original": "https://example.org/isiu/nimadmi.gif?ari=equun#suntinc", + "url.path": "/isiu/nimadmi.gif", + "url.query": "ari=equun", + "url.scheme": "https" }, { "event.code": "Hardware", @@ -848,7 +866,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/nse/sinto.gif?CSed=lupt#psaquae" + "url.domain": "www.example.net", + "url.extension": "gif", + "url.fragment": "psaquae", + "url.original": "https://www.example.net/nse/sinto.gif?CSed=lupt#psaquae", + "url.path": "/nse/sinto.gif", + "url.query": "CSed=lupt", + "url.scheme": "https" }, { "event.code": "BGP", @@ -980,7 +1004,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.net/corp/modtemp.jpg?oluptas=tNequepo#lup" + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "lup", + "url.original": "https://mail.example.net/corp/modtemp.jpg?oluptas=tNequepo#lup", + "url.path": "/corp/modtemp.jpg", + "url.query": "oluptas=tNequepo", + "url.scheme": "https" }, { "event.code": "Change_Log", @@ -1036,7 +1066,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://example.net/temqu/edol.jpg?ipi=reseos#pariatu" + "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "pariatu", + "url.original": "https://example.net/temqu/edol.jpg?ipi=reseos#pariatu", + "url.path": "/temqu/edol.jpg", + "url.query": "ipi=reseos", + "url.scheme": "https" }, { "destination.ip": [ @@ -1147,8 +1183,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.83.23.104", - "10.163.161.165" + "10.163.161.165", + "10.83.23.104" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1161,7 +1197,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.org/atem/gnido.txt?tmollita=fde#nsecte" + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "nsecte", + "url.original": "https://www5.example.org/atem/gnido.txt?tmollita=fde#nsecte", + "url.path": "/atem/gnido.txt", + "url.query": "tmollita=fde", + "url.scheme": "https" }, { "destination.ip": [ @@ -1402,7 +1444,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://api.example.com/eufugi/pici.html?ccaecat=tquiin#tse" + "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "tse", + "url.original": "https://api.example.com/eufugi/pici.html?ccaecat=tquiin#tse", + "url.path": "/eufugi/pici.html", + "url.query": "ccaecat=tquiin", + "url.scheme": "https" }, { "event.code": "Change_Log", @@ -1752,8 +1800,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.28.226.128", - "10.122.76.148" + "10.122.76.148", + "10.28.226.128" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1766,7 +1814,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.org/idunt/luptat.txt?ica=lillum#remips" + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "remips", + "url.original": "https://mail.example.org/idunt/luptat.txt?ica=lillum#remips", + "url.path": "/idunt/luptat.txt", + "url.query": "ica=lillum", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1790,7 +1844,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://internal.example.com/ineavol/iosa.html?usc=rem#amvolupt" + "url.domain": "internal.example.com", + "url.extension": "html", + "url.fragment": "amvolupt", + "url.original": "https://internal.example.com/ineavol/iosa.html?usc=rem#amvolupt", + "url.path": "/ineavol/iosa.html", + "url.query": "usc=rem", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1814,7 +1874,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://internal.example.org/quaUten/nisiut.txt?teturad=perspici#itation" + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "itation", + "url.original": "https://internal.example.org/quaUten/nisiut.txt?teturad=perspici#itation", + "url.path": "/quaUten/nisiut.txt", + "url.query": "teturad=perspici", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1838,7 +1904,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.org/nofd/dipisci.txt?ilmol=eri#quunt" + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "quunt", + "url.original": "https://mail.example.org/nofd/dipisci.txt?ilmol=eri#quunt", + "url.path": "/nofd/dipisci.txt", + "url.query": "ilmol=eri", + "url.scheme": "https" }, { "destination.ip": [ @@ -1870,7 +1942,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.org/ptateve/enderi.html?toccaec=fugi#labo" + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "labo", + "url.original": "https://www.example.org/ptateve/enderi.html?toccaec=fugi#labo", + "url.path": "/ptateve/enderi.html", + "url.query": "toccaec=fugi", + "url.scheme": "https" }, { "destination.ip": [ @@ -1902,7 +1980,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo" + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "umetMalo", + "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo", + "url.path": "/tanimi/rumSecti.jpg", + "url.query": "emporain=ntiumto", + "url.scheme": "https" }, { "event.code": "configuration", @@ -2169,8 +2253,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.151.129.181", - "10.55.156.64" + "10.55.156.64", + "10.151.129.181" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -2183,7 +2267,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/itanim/nesciun.txt?mollita=tatem#iae" + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "iae", + "url.original": "https://www.example.net/itanim/nesciun.txt?mollita=tatem#iae", + "url.path": "/itanim/nesciun.txt", + "url.query": "mollita=tatem", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -2207,7 +2297,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/rever/ore.jpg?oluptat=metco#acom" + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "acom", + "url.original": "https://www.example.net/rever/ore.jpg?oluptat=metco#acom", + "url.path": "/rever/ore.jpg", + "url.query": "oluptat=metco", + "url.scheme": "https" }, { "event.code": "Host", @@ -2314,7 +2410,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://api.example.org/eosquira/pta.htm?econs=lmolesti#apariatu" + "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "apariatu", + "url.original": "https://api.example.org/eosquira/pta.htm?econs=lmolesti#apariatu", + "url.path": "/eosquira/pta.htm", + "url.query": "econs=lmolesti", + "url.scheme": "https" }, { "event.action": "Script mitigation", @@ -2362,7 +2464,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.com/atcupida/tessequa.htm?dolores=equamnih#taliqui" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "taliqui", + "url.original": "https://www5.example.com/atcupida/tessequa.htm?dolores=equamnih#taliqui", + "url.path": "/atcupida/tessequa.htm", + "url.query": "dolores=equamnih", + "url.scheme": "https" }, { "event.code": "Change_Log", diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 493713469254..c2d1a74c0306 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -7,6 +7,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json index 4a6f14974fc4..41256959a5c5 100644 --- a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json @@ -64,7 +64,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -142,7 +146,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -220,7 +228,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "aspx", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -298,7 +310,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "aspx", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -377,7 +393,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -455,7 +475,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -533,7 +557,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -612,7 +640,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -690,7 +722,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -768,7 +804,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -846,7 +886,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json index 75f16b9867e3..c440645b3604 100644 --- a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json @@ -80,7 +80,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -176,7 +180,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -268,7 +276,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -364,7 +376,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -460,7 +476,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL%20CREDIT%20CARD%20NUMBERS.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL CREDIT CARD NUMBERS.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -556,7 +576,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", + "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", @@ -652,7 +676,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", + "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json b/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json index 941adb1e1ac6..7996a2d808df 100644 --- a/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json @@ -791,7 +791,11 @@ "tags": [ "forwarded" ], + "url.domain": "example.net", + "url.extension": "com/sharepoint", "url.original": "https://example.net/testsiem2.onmicrosoft.com/sharepoint", + "url.path": "/testsiem2.onmicrosoft.com/sharepoint", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json b/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json index 372b29d8c2ca..89a70bc165f5 100644 --- a/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json @@ -249,7 +249,10 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem4.sharepoint.com", "url.original": "https://testsiem4.sharepoint.com/sites/users/Shared Documents/General", + "url.path": "/sites/users/Shared Documents/General", + "url.scheme": "https", "user.domain": "testsiem4.onmicrosoft.com", "user.email": "root@testsiem4.onmicrosoft.com", "user.id": "root@testsiem4.onmicrosoft.com", @@ -1547,7 +1550,10 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem4.sharepoint.com", "url.original": "https://testsiem4.sharepoint.com/sites/users/Shared Documents/General", + "url.path": "/sites/users/Shared Documents/General", + "url.scheme": "https", "user.domain": "testsiem4.onmicrosoft.com", "user.email": "root@testsiem4.onmicrosoft.com", "user.id": "root@testsiem4.onmicrosoft.com", diff --git a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml index 3c41092be69f..66d787b27ce3 100644 --- a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json index e0caf6d7abbf..bf35e8659161 100644 --- a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json @@ -70,8 +70,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "block", - "ugiatnu" + "ugiatnu", + "block" ], "rsa.misc.comments": "colabo", "rsa.misc.content_type": "sedd", @@ -103,7 +103,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "iciatisu", "url.original": "https://mail.example.net/tati/utaliqu.html?iquaUten=santium#iciatisu", + "url.path": "/tati/utaliqu.html", + "url.query": "iquaUten=santium", + "url.scheme": "https", "user.name": "sunt" }, { @@ -163,8 +169,8 @@ "10.106.239.55" ], "related.user": [ - "eaq", - "itquiin" + "itquiin", + "eaq" ], "rsa.identity.logon_type": "stquidol", "rsa.internal.event_desc": "bor", @@ -974,8 +980,8 @@ "10.232.108.32" ], "related.user": [ - "rsp", - "llum" + "llum", + "rsp" ], "rsa.identity.logon_type": "ntut", "rsa.internal.event_desc": "ittenb", @@ -1038,8 +1044,8 @@ ], "related.user": [ "tcustatuscode=eumiu", - "tio", - "pteurs" + "pteurs", + "tio" ], "rsa.db.index": "eavolupt", "rsa.identity.logon_type": "ursintoc", @@ -1048,8 +1054,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "iuntN", - "deny" + "deny", + "iuntN" ], "rsa.misc.comments": "onorume", "rsa.misc.content_type": "lapa", @@ -1084,7 +1090,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "ssequamn", "url.original": "https://mail.example.net/velillu/ecatcupi.txt?rsitamet=leumiur#ssequamn", + "url.path": "/velillu/ecatcupi.txt", + "url.query": "rsitamet=leumiur", + "url.scheme": "https", "user.name": "tio" }, { @@ -1237,7 +1249,13 @@ "sophos.utm", "forwarded" ], - "url.original": "https://www5.example.com/dantium/ors.htm?sinto=edi#eumiure" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "eumiure", + "url.original": "https://www5.example.com/dantium/ors.htm?sinto=edi#eumiure", + "url.path": "/dantium/ors.htm", + "url.query": "sinto=edi", + "url.scheme": "https" }, { "@timestamp": "2017-05-29T07:37:24.000Z", @@ -1610,8 +1628,8 @@ "10.244.96.61" ], "related.user": [ - "itsedqui", - "iumt" + "iumt", + "itsedqui" ], "rsa.identity.logon_type": "psamvolu", "rsa.internal.event_desc": "orroqui", @@ -1851,8 +1869,8 @@ "tenbyCi4371.www5.localdomain" ], "related.ip": [ - "10.214.167.164", - "10.98.126.206" + "10.98.126.206", + "10.214.167.164" ], "related.user": [ "amremapstatuscode=dolorsit", @@ -1899,7 +1917,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "enbyCi", "url.original": "https://api.example.org/iono/secillum.txt?apariat=tse#enbyCi", + "url.path": "/iono/secillum.txt", + "url.query": "apariat=tse", + "url.scheme": "https", "user.name": "hen" }, { @@ -2029,14 +2053,14 @@ "iusmo901.www.home" ], "related.ip": [ - "10.2.24.156", - "10.92.93.236" + "10.92.93.236", + "10.2.24.156" ], "related.user": [ - "dolorsistatuscode=acc", - "Sedutper", + "ulpaq", "ntoccae", - "ulpaq" + "Sedutper", + "dolorsistatuscode=acc" ], "rsa.db.index": "snisiut", "rsa.identity.logon_type": "umdol", @@ -2082,8 +2106,16 @@ "sophos.utm", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "tru", "url.original": "https://www.example.org/imipsamv/doeiu.jpg?nderit=ficia#tru", - "url.query": "exerc", + "url.path": "/imipsamv/doeiu.jpg", + "url.query": [ + "exerc", + "nderit=ficia" + ], + "url.scheme": "https", "user.name": "Sedutper" }, { @@ -2124,8 +2156,8 @@ "10.180.169.49" ], "related.user": [ - "atatno", "tasu", + "atatno", "iscivelistatuscode=urve" ], "rsa.db.index": "amrem", @@ -2135,8 +2167,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "mquisn", - "cancel" + "cancel", + "mquisn" ], "rsa.misc.comments": "architec", "rsa.misc.content_type": "atisetqu", @@ -2168,7 +2200,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "adm", "url.original": "https://mail.example.net/isqua/deF.html?iameaq=orainci#adm", + "url.path": "/isqua/deF.html", + "url.query": "iameaq=orainci", + "url.scheme": "https", "user.name": "atatno" }, { @@ -2314,13 +2352,13 @@ "obea2960.mail.corp" ], "related.ip": [ - "10.45.12.53", - "10.33.138.154" + "10.33.138.154", + "10.45.12.53" ], "related.user": [ - "eturadip", + "umqustatuscode=ntexpli", "porincid", - "umqustatuscode=ntexpli" + "eturadip" ], "rsa.db.index": "dolor", "rsa.identity.logon_type": "eturadi", @@ -2362,7 +2400,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "oditautf", "url.original": "https://www5.example.org/etcons/aecatc.jpg?ditem=tut#oditautf", + "url.path": "/etcons/aecatc.jpg", + "url.query": "ditem=tut", + "url.scheme": "https", "user.name": "eturadip" }, { @@ -2481,8 +2525,8 @@ "10.32.85.21" ], "related.user": [ - "antium", - "etconsec" + "etconsec", + "antium" ], "rsa.identity.logon_type": "umiurere", "rsa.internal.event_desc": "serro", @@ -2634,8 +2678,8 @@ "related.user": [ "Loremi", "inimastatuscode=emipsum", - "reetd", - "rExce" + "rExce", + "reetd" ], "rsa.db.index": "apa", "rsa.identity.logon_type": "sedquia", @@ -2644,8 +2688,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "odte", - "cancel" + "cancel", + "odte" ], "rsa.misc.comments": "emquia", "rsa.misc.content_type": "sauteir", @@ -2678,8 +2722,16 @@ "sophos.utm", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "agnaaliq", "url.original": "https://internal.example.com/ritat/dipi.jpg?aliquide=aliqui#agnaaliq", - "url.query": "iatquovo", + "url.path": "/ritat/dipi.jpg", + "url.query": [ + "iatquovo", + "aliquide=aliqui" + ], + "url.scheme": "https", "user.name": "reetd" }, { @@ -2707,8 +2759,8 @@ "itametc1599.api.test" ], "related.ip": [ - "10.115.166.48", - "10.133.45.45" + "10.133.45.45", + "10.115.166.48" ], "rsa.internal.event_desc": "Authentication", "rsa.internal.messageid": "ulogd", @@ -3463,8 +3515,8 @@ "imv1805.api.host" ], "related.ip": [ - "10.248.62.55", - "10.96.243.231" + "10.96.243.231", + "10.248.62.55" ], "rsa.internal.event_desc": "ICMP", "rsa.internal.messageid": "ulogd", diff --git a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml index 845a93c945ed..6eb60e460bbc 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml @@ -192,6 +192,29 @@ processors: target_field: url.domain ignore_missing: true if: "ctx.sophos?.xg?.src_domainname != null" +- uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null +- script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } +- remove: + field: _temp_ + ignore_missing: true ############################ ## ECS User Agent Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml index c659264d633e..64c3e52bf03f 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml @@ -120,6 +120,33 @@ processors: target_field: url.original ignore_missing: true if: "ctx.sophos?.xg?.url != null" +- uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null && ctx?.url?.original != ctx?.destination?.ip +- script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } +- remove: + field: _temp_ + ignore_missing: true +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true ############################# ## ECS Network/Geo Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml index 0b5f92c1e699..7b4853ce5bf4 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml @@ -122,16 +122,19 @@ processors: ##################### ## ECS URL Mapping ## ##################### +- uri_parts: + field: sophos.xg.url + ignore_failure: true + if: "ctx.sophos?.xg?.url != null" - rename: field: sophos.xg.url target_field: url.full ignore_missing: true if: "ctx.sophos?.xg?.url != null" -- rename: - field: sophos.xg.domain - target_field: url.domain - ignore_missing: true - if: "ctx.sophos?.xg?.domain != null" +- set: + field: url.domain + value: "{{sophos.xg.domain}}" + ignore_empty_value: true ############################ ## ECS User Agent Mapping ## @@ -268,7 +271,32 @@ processors: - sophos.xg.src_port - sophos.xg.recv_bytes - sophos.xg.sent_bytes + - sophos.xg.domain ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml index e0ea89c41dc7..6462ac5d6bf2 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml @@ -118,16 +118,19 @@ processors: ##################### ## ECS URL Mapping ## ##################### +- uri_parts: + field: sophos.xg.url + ignore_failure: true + if: "ctx.sophos?.xg?.url != null" - rename: field: sophos.xg.url target_field: url.full ignore_missing: true if: "ctx.sophos?.xg?.url != null" -- rename: - field: sophos.xg.domain - target_field: url.domain - ignore_missing: true - if: "ctx.sophos?.xg?.domain != null" +- set: + field: url.domain + value: "{{sophos.xg.domain}}" + ignore_empty_value: true ############################ ## ECS User Agent Mapping ## @@ -279,7 +282,32 @@ processors: - sophos.xg.bytesrcv - sophos.xg.bytessent - sophos.xg.httpstatus + - sophos.xg.domain ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json index 4afefcee9b49..8ab666cc94bc 100644 --- a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json @@ -77,7 +77,10 @@ "forwarded" ], "url.domain": "sophostest.com", + "url.extension": "pdf", "url.original": "http://sophostest.com/Sandstorm/SBTestFile1.pdf", + "url.path": "/Sandstorm/SBTestFile1.pdf", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" }, { @@ -158,7 +161,10 @@ "forwarded" ], "url.domain": "sophostest.com", + "url.extension": "html", "url.original": "http://sophostest.com/eicar/index.html", + "url.path": "/eicar/index.html", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" }, { diff --git a/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json index a0230cb1dc49..969ca99d7935 100644 --- a/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json @@ -140,7 +140,11 @@ "sophos-xg", "forwarded" ], - "url.original": "http://sophostest.com/callhome/index.html" + "url.domain": "sophostest.com", + "url.extension": "html", + "url.original": "http://sophostest.com/callhome/index.html", + "url.path": "/callhome/index.html", + "url.scheme": "http" }, { "@timestamp": "2020-05-18T14:38:35.000-02:00", @@ -211,7 +215,11 @@ "sophos-xg", "forwarded" ], - "url.original": "http://sophostest.com/callhome/index.html" + "url.domain": "sophostest.com", + "url.extension": "html", + "url.original": "http://sophostest.com/callhome/index.html", + "url.path": "/callhome/index.html", + "url.scheme": "http" }, { "@timestamp": "2018-06-05T08:49:00.000-02:00", diff --git a/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json index c8bb6001058b..3194c309b5b7 100644 --- a/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json @@ -71,7 +71,10 @@ "forwarded" ], "url.domain": "r8---sn-ci5gup-qxas.googlevideo.com", - "url.full": "https://r8---sn-ci5gup-qxas.googlevideo.com/" + "url.full": "https://r8---sn-ci5gup-qxas.googlevideo.com/", + "url.original": "https://r8---sn-ci5gup-qxas.googlevideo.com/", + "url.path": "/", + "url.scheme": "https" }, { "@timestamp": "2017-02-01T18:20:21.000-02:00", @@ -152,7 +155,10 @@ "forwarded" ], "url.domain": "hanuman.com", - "url.full": "http://hanuman.com/" + "url.full": "http://hanuman.com/", + "url.original": "http://hanuman.com/", + "url.path": "/", + "url.scheme": "http" }, { "@timestamp": "2017-02-01T18:13:29.000-02:00", @@ -309,7 +315,10 @@ "forwarded" ], "url.domain": "his-eur1-neur1.servicebus.windows.net", - "url.full": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket" + "url.full": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket", + "url.original": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket", + "url.path": "/$servicebus/websocket", + "url.scheme": "https" }, { "@timestamp": "2020-05-18T14:38:52.000-02:00", @@ -386,7 +395,10 @@ "forwarded" ], "url.domain": "40.90.137.127", - "url.full": "https://40.90.137.127/" + "url.full": "https://40.90.137.127/", + "url.original": "https://40.90.137.127/", + "url.path": "/", + "url.scheme": "https" }, { "@timestamp": "2020-05-18T14:38:53.000-02:00", @@ -462,7 +474,11 @@ "forwarded" ], "url.domain": "update.eset.com", + "url.extension": "signed", "url.full": "http://update.eset.com/eset_upd/ep7/dll/update.ver.signed", + "url.original": "http://update.eset.com/eset_upd/ep7/dll/update.ver.signed", + "url.path": "/eset_upd/ep7/dll/update.ver.signed", + "url.scheme": "http", "user_agent.original": "EFSW Update (Windows; U; 64bit; BPC 7.1.12010.0; OS: 10.0.17763 SP 0.0 NT; TDB 45511; CL 1.1.1; x64s; APP efsw; PX 1; PUA 1; CD 1; RA 1; PEV 0; UNS 1; UBR 1158; HVCI 0; SHA256 1; WU 3; HWF: 01009DAA-757A-D666-EFD2-92DD0D501284; PLOC de_de; PCODE 211.0.0; " }, { @@ -592,7 +608,10 @@ "forwarded" ], "url.domain": "www.google.com", - "url.full": "http://www.google.com/" + "url.full": "http://www.google.com/", + "url.original": "http://www.google.com/", + "url.path": "/", + "url.scheme": "http" }, { "@timestamp": "2016-12-02T18:50:22.000-02:00", @@ -669,6 +688,10 @@ "forwarded" ], "url.domain": "www.google.ca", - "url.full": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw" + "url.full": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.original": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.path": "/", + "url.query": "gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.scheme": "http" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json index 055f255a15a1..9950baa2a62a 100644 --- a/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json @@ -81,6 +81,8 @@ "forwarded" ], "url.full": "/mapi/nspi/", + "url.original": "/mapi/nspi/", + "url.path": "/mapi/nspi/", "user_agent.original": "Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.4954; Pro)" }, { @@ -166,6 +168,8 @@ "forwarded" ], "url.full": "/mapi/nspi/", + "url.original": "/mapi/nspi/", + "url.path": "/mapi/nspi/", "user_agent.original": "Microsoft Office/16.0 (Windows NT 6.2; Microsoft Outlook 16.0.4954; Pro)" }, { @@ -236,6 +240,8 @@ "forwarded" ], "url.full": "/", + "url.original": "/", + "url.path": "/", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" }, { @@ -308,7 +314,10 @@ "sophos-xg", "forwarded" ], + "url.extension": "zip", "url.full": "/download/eicarcom2.zip", + "url.original": "/download/eicarcom2.zip", + "url.path": "/download/eicarcom2.zip", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" }, { @@ -392,6 +401,8 @@ "sophos-xg", "forwarded" ], - "url.full": "/" + "url.full": "/", + "url.original": "/", + "url.path": "/" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index d2bf90581b81..f9f6ddaa0276 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -10,6 +10,34 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - remove: + field: url.scheme + ignore_missing: true + if: ctx?.url?.scheme == ctx?.url?.domain # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index bcced9c85a3e..f70c9ad8ae49 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -26,8 +26,8 @@ "login.yahoo.com" ], "related.ip": [ - "209.73.177.115", - "10.105.21.199" + "10.105.21.199", + "209.73.177.115" ], "related.user": [ "badeyek" @@ -37,8 +37,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "CONNECT" + "CONNECT", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -97,8 +97,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -109,8 +109,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -134,7 +134,9 @@ ], "url.domain": "www.goonernews.com", "url.original": "http://www.goonernews.com/", + "url.path": "/", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -170,8 +172,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -182,8 +184,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -206,8 +208,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "css", "url.original": "http://www.goonernews.com/styles.css", + "url.path": "/styles.css", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -241,8 +246,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -265,8 +270,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "css", "url.original": "http://www.goonernews.com/styles.css", + "url.path": "/styles.css", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -324,8 +332,11 @@ "forwarded" ], "url.domain": "www.google-analytics.com", + "url.extension": "js", "url.original": "http://www.google-analytics.com/urchin.js", + "url.path": "/urchin.js", "url.registered_domain": "google-analytics.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -360,8 +371,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -397,7 +408,9 @@ ], "url.domain": "www.goonernews.com", "url.original": "http://www.goonernews.com/", + "url.path": "/", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -442,8 +455,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -466,8 +479,12 @@ "forwarded" ], "url.domain": "www.google-analytics.com", + "url.extension": "gif", "url.original": "http://www.google-analytics.com/__utm.gif?", + "url.path": "/__utm.gif", + "url.query": "", "url.registered_domain": "google-analytics.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -503,8 +520,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -539,8 +556,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/graphics/newslogo.gif", + "url.path": "/graphics/newslogo.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -588,8 +608,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -612,8 +632,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "jpg", "url.original": "http://www.goonernews.com/shop/arsenal_shop_ad.jpg", + "url.path": "/shop/arsenal_shop_ad.jpg", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -649,8 +672,8 @@ "www.goonernews.com" ], "related.ip": [ - "10.105.21.199", - "207.58.145.61" + "207.58.145.61", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -661,8 +684,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -685,8 +708,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FUS.gif", + "url.path": "/flags/FUS.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -720,8 +746,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -744,8 +770,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FGB.gif", + "url.path": "/flags/FGB.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -781,8 +810,8 @@ "as.casalemedia.com" ], "related.ip": [ - "10.105.21.199", - "209.85.16.38" + "209.85.16.38", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -818,7 +847,10 @@ ], "url.domain": "as.casalemedia.com", "url.original": "http://as.casalemedia.com/s?", + "url.path": "/s", + "url.query": "", "url.registered_domain": "casalemedia.com", + "url.scheme": "http", "url.subdomain": "as", "url.top_level_domain": "com", "user.name": "badeyek" @@ -848,8 +880,8 @@ "us.bc.yahoo.com" ], "related.ip": [ - "68.142.213.132", - "10.105.21.199" + "10.105.21.199", + "68.142.213.132" ], "related.user": [ "badeyek" @@ -859,8 +891,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_MISS" + "TCP_MISS", + "CONNECT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -917,8 +949,8 @@ "impgb.tradedoubler.com" ], "related.ip": [ - "10.105.21.199", - "217.212.240.172" + "217.212.240.172", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -929,8 +961,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "302", @@ -954,7 +986,9 @@ ], "url.domain": "impgb.tradedoubler.com", "url.original": "http://impgb.tradedoubler.com/imp/img/16349696/992098", + "url.path": "/imp/img/16349696/992098", "url.registered_domain": "tradedoubler.com", + "url.scheme": "http", "url.subdomain": "impgb", "url.top_level_domain": "com", "user.name": "badeyek" @@ -990,8 +1024,8 @@ "4.adbrite.com" ], "related.ip": [ - "10.105.21.199", - "206.169.136.22" + "206.169.136.22", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -1002,8 +1036,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -1026,8 +1060,12 @@ "forwarded" ], "url.domain": "4.adbrite.com", + "url.extension": "php", "url.original": "http://4.adbrite.com/mb/text_group.php?", + "url.path": "/mb/text_group.php", + "url.query": "", "url.registered_domain": "adbrite.com", + "url.scheme": "http", "url.subdomain": "4", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1061,8 +1099,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -1085,8 +1123,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FFR.gif", + "url.path": "/flags/FFR.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1134,8 +1175,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -1158,8 +1199,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FAU.gif", + "url.path": "/flags/FAU.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1195,8 +1239,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -1207,8 +1251,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -1231,8 +1275,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/graphics/spacer.gif", + "url.path": "/graphics/spacer.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1263,8 +1310,8 @@ "4.adbrite.com" ], "related.ip": [ - "64.127.126.178", - "10.105.21.199" + "10.105.21.199", + "64.127.126.178" ], "related.user": [ "badeyek" @@ -1299,8 +1346,12 @@ "forwarded" ], "url.domain": "4.adbrite.com", + "url.extension": "php", "url.original": "http://4.adbrite.com/mb/text_group.php?", + "url.path": "/mb/text_group.php", + "url.query": "", "url.registered_domain": "adbrite.com", + "url.scheme": "http", "url.subdomain": "4", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1336,8 +1387,8 @@ "ff.connextra.com" ], "related.ip": [ - "10.105.21.199", - "213.160.98.161" + "213.160.98.161", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -1348,8 +1399,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "302", @@ -1373,7 +1424,10 @@ ], "url.domain": "ff.connextra.com", "url.original": "http://ff.connextra.com/Ladbrokes/selector/image?", + "url.path": "/Ladbrokes/selector/image", + "url.query": "", "url.registered_domain": "connextra.com", + "url.scheme": "http", "url.subdomain": "ff", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1446,7 +1500,10 @@ ], "url.domain": "dd.connextra.com", "url.original": "http://dd.connextra.com/servlet/controller?", + "url.path": "/servlet/controller", + "url.query": "", "url.registered_domain": "connextra.com", + "url.scheme": "http", "url.subdomain": "dd", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1503,7 +1560,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "-" }, @@ -1534,8 +1593,8 @@ "login.yahoo.com" ], "related.ip": [ - "209.73.177.115", - "10.105.21.199" + "10.105.21.199", + "209.73.177.115" ], "related.user": [ "badeyek" @@ -1604,8 +1663,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -1628,8 +1687,11 @@ "forwarded" ], "url.domain": "update.messenger.yahoo.com", + "url.extension": "html", "url.original": "http://update.messenger.yahoo.com/msgrcli7.html", + "url.path": "/msgrcli7.html", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "update.messenger", "url.top_level_domain": "com", "user.name": "-" @@ -1697,7 +1759,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -1768,7 +1832,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -1803,8 +1869,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -1815,8 +1881,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -1838,8 +1904,11 @@ "forwarded" ], "url.domain": "hi5.com", + "url.extension": "css", "url.original": "http://hi5.com/friend/styles/homepage.css", + "url.path": "/friend/styles/homepage.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -1870,8 +1939,8 @@ "shttp.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.155.194.239" + "216.155.194.239", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -1881,8 +1950,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -1906,7 +1975,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -1964,8 +2035,11 @@ "forwarded" ], "url.domain": "rms.adobe.com", + "url.extension": "xml", "url.original": "http://rms.adobe.com/read/0600/win_/ENU/read0600win_ENUadbe0000.xml", + "url.path": "/read/0600/win_/ENU/read0600win_ENUadbe0000.xml", "url.registered_domain": "adobe.com", + "url.scheme": "http", "url.subdomain": "rms", "url.top_level_domain": "com", "user.name": "-" @@ -2023,8 +2097,11 @@ "forwarded" ], "url.domain": "images.hi5.com", + "url.extension": "css", "url.original": "http://images.hi5.com/styles/style.css", + "url.path": "/styles/style.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.subdomain": "images", "url.top_level_domain": "com", "user.name": "nazsoau" @@ -2058,8 +2135,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "304", @@ -2082,8 +2159,11 @@ "forwarded" ], "url.domain": "images.hi5.com", + "url.extension": "css", "url.original": "http://images.hi5.com/friend/styles/buttons_en_us.css", + "url.path": "/friend/styles/buttons_en_us.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.subdomain": "images", "url.top_level_domain": "com", "user.name": "nazsoau" @@ -2118,8 +2198,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -2154,7 +2234,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -2189,8 +2271,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -2201,8 +2283,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -2224,8 +2306,11 @@ "forwarded" ], "url.domain": "hi5.com", + "url.extension": "css", "url.original": "http://hi5.com/friend/styles/headernav.css", + "url.path": "/friend/styles/headernav.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -2292,7 +2377,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2322,8 +2409,8 @@ "insider.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.194.14" + "68.142.194.14", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2359,7 +2446,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/?", + "url.path": "/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2392,8 +2482,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2428,8 +2518,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/playmessenger.asp", + "url.path": "/radio/play/playmessenger.asp", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2497,7 +2590,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2565,7 +2660,10 @@ ], "url.domain": "address.yahoo.com", "url.original": "http://address.yahoo.com/yab/us?", + "url.path": "/yab/us", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "address", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2637,8 +2735,11 @@ "forwarded" ], "url.domain": "fxfeeds.mozilla.org", + "url.extension": "xml", "url.original": "http://fxfeeds.mozilla.org/rss20.xml", + "url.path": "/rss20.xml", "url.registered_domain": "mozilla.org", + "url.scheme": "http", "url.subdomain": "fxfeeds", "url.top_level_domain": "org", "user.name": "badeyek" @@ -2682,8 +2783,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2707,7 +2808,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/ycontent/?", + "url.path": "/ycontent/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2749,8 +2853,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2774,7 +2878,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/ycontent/?", + "url.path": "/ycontent/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2806,8 +2913,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "CONNECT" + "CONNECT", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -2864,8 +2971,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_DENIED" + "TCP_DENIED", + "POST" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -2888,8 +2995,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appinstru.asp", + "url.path": "/apps/agent/submgr/appinstru.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -2946,8 +3056,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appsync.asp", + "url.path": "/apps/agent/submgr/appsync.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -2979,8 +3092,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_DENIED" + "TCP_DENIED", + "CONNECT" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3049,8 +3162,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3073,8 +3186,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_stations.gif", + "url.path": "/radio/clientdata/538/images/btn_stations.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3164,8 +3280,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -3200,8 +3316,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_stations_over.gif", + "url.path": "/radio/clientdata/538/images/btn_stations_over.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3244,8 +3363,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -3269,7 +3388,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3338,8 +3459,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_left.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_left.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3372,8 +3496,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -3384,8 +3508,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3408,8 +3532,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/launchcast_radio.gif", + "url.path": "/radio/clientdata/538/images/launchcast_radio.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3442,8 +3569,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "POST" + "POST", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3466,8 +3593,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appinstru.asp", + "url.path": "/apps/agent/submgr/appinstru.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3524,8 +3654,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appsync.asp", + "url.path": "/apps/agent/submgr/appsync.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3570,8 +3703,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3594,8 +3727,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_right.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_right.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3640,8 +3776,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3664,8 +3800,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_center.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_center.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3710,8 +3849,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3734,8 +3873,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_controls_off.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_controls_off.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3780,8 +3922,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3804,8 +3946,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", + "url.path": "/radio/common_radio/resources/images/t.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3863,8 +4008,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_off_state_station.gif", + "url.path": "/radio/clientdata/538/images/btn_off_state_station.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3922,8 +4070,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_controls_fill.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_controls_fill.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3957,8 +4108,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -3981,8 +4132,11 @@ "forwarded" ], "url.domain": "us.i1.yimg.com", + "url.extension": "gif", "url.original": "http://us.i1.yimg.com/us.yimg.com/i/us/toolbar50x50.gif", + "url.path": "/us.yimg.com/i/us/toolbar50x50.gif", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.i1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4051,8 +4205,11 @@ "forwarded" ], "url.domain": "newsrss.bbc.co.uk", + "url.extension": "xml", "url.original": "http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml", + "url.path": "/rss/newsonline_world_edition/front_page/rss.xml", "url.registered_domain": "bbc.co.uk", + "url.scheme": "http", "url.subdomain": "newsrss", "url.top_level_domain": "co.uk", "user.name": "badeyek" @@ -4085,8 +4242,8 @@ "insider.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.231.252" + "68.142.231.252", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4097,8 +4254,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -4121,8 +4278,11 @@ "forwarded" ], "url.domain": "insider.msg.yahoo.com", + "url.extension": "php", "url.original": "http://insider.msg.yahoo.com/ycontent/beacon.php", + "url.path": "/ycontent/beacon.php", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4180,8 +4340,11 @@ "forwarded" ], "url.domain": "us.ent1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.ent1.yimg.com/images.launch.yahoo.com/000/032/457/32457654.jpg", + "url.path": "/images.launch.yahoo.com/000/032/457/32457654.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.ent1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4215,8 +4378,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "200", @@ -4239,8 +4402,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060906/thumb.71d29ded334347c48ac88433d033c9a9.pakistan_bin_laden_nyol440.jpg", + "url.path": "/us.yimg.com/p/ap/20060906/thumb.71d29ded334347c48ac88433d033c9a9.pakistan_bin_laden_nyol440.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4273,8 +4439,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4308,8 +4474,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/authplay.asp", + "url.path": "/radio/play/authplay.asp", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4357,8 +4526,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "200", @@ -4381,8 +4550,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060908/thumb.443f57762d7349669f609fbf0c97a5f1.academy_awards_host_cacp101.jpg", + "url.path": "/us.yimg.com/p/ap/20060908/thumb.443f57762d7349669f609fbf0c97a5f1.academy_awards_host_cacp101.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4415,8 +4587,8 @@ "radio.music.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4451,8 +4623,12 @@ "forwarded" ], "url.domain": "radio.music.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.music.yahoo.com/radio/player/ymsgr/initstationfeed.asp?", + "url.path": "/radio/player/ymsgr/initstationfeed.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.music", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4521,8 +4697,12 @@ "forwarded" ], "url.domain": "radio.music.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.music.yahoo.com/radio/player/ymsgr/initstationfeed.asp?", + "url.path": "/radio/player/ymsgr/initstationfeed.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.music", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4555,8 +4735,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4567,8 +4747,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -4591,8 +4771,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/authplay.asp?", + "url.path": "/radio/play/authplay.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4651,7 +4835,11 @@ ], "url.domain": "natrocket.kmip.net", "url.original": "http://natrocket.kmip.net:5288/iesocks?", + "url.path": "/iesocks", + "url.port": 5288, + "url.query": "", "url.registered_domain": "kmip.net", + "url.scheme": "http", "url.subdomain": "natrocket", "url.top_level_domain": "net", "user.name": "-" @@ -4685,8 +4873,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -4710,7 +4898,11 @@ ], "url.domain": "natrocket.kmip.net", "url.original": "http://natrocket.kmip.net:5288/return?", + "url.path": "/return", + "url.port": 5288, + "url.query": "", "url.registered_domain": "kmip.net", + "url.scheme": "http", "url.subdomain": "natrocket", "url.top_level_domain": "net", "user.name": "-" @@ -4746,8 +4938,8 @@ "us.news1.yimg.com" ], "related.ip": [ - "213.160.98.159", - "10.105.33.214" + "10.105.33.214", + "213.160.98.159" ], "related.user": [ "adeolaegbedokun" @@ -4782,8 +4974,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060907/thumb.1caf18e56db54eafb16da58356eb3382.amazon_com_online_video_watw101.jpg", + "url.path": "/us.yimg.com/p/ap/20060907/thumb.1caf18e56db54eafb16da58356eb3382.amazon_com_online_video_watw101.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4828,8 +5023,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "302", @@ -4852,8 +5047,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/515/starter.asp?", + "url.path": "/radio/clientdata/515/starter.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4886,8 +5085,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4922,8 +5121,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/player/default.asp?", + "url.path": "/radio/player/default.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4971,8 +5174,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "application/x-shockwave-flash", "rsa.misc.result_code": "200", @@ -4995,8 +5198,12 @@ "forwarded" ], "url.domain": "us.a2.yimg.com", + "url.extension": "swf", "url.original": "http://us.a2.yimg.com/us.yimg.com/a/ya/yahoo_messenger/081106_lrec_msgr_interophitchhiker.swf?", + "url.path": "/us.yimg.com/a/ya/yahoo_messenger/081106_lrec_msgr_interophitchhiker.swf", + "url.query": "", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.a2", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5029,8 +5236,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -5065,8 +5272,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/player/stickwall.asp?", + "url.path": "/radio/player/stickwall.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5097,8 +5308,8 @@ "us.bc.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.213.132" + "68.142.213.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5109,8 +5320,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -5134,7 +5345,10 @@ ], "url.domain": "us.bc.yahoo.com", "url.original": "http://us.bc.yahoo.com/b?", + "url.path": "/b", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "us.bc", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5201,8 +5415,12 @@ "forwarded" ], "url.domain": "insider.msg.yahoo.com", + "url.extension": "php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw", "url.original": "http://insider.msg.yahoo.com/ycontent/beacon.php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw?", + "url.path": "/ycontent/beacon.php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5243,8 +5461,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "CONNECT" + "CONNECT", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -5302,8 +5520,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "304", @@ -5326,8 +5544,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "js", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", + "url.path": "/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5363,8 +5584,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5399,8 +5620,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "css", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", + "url.path": "/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5448,8 +5672,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5472,8 +5696,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5505,8 +5732,8 @@ "login.yahoo.com" ], "related.ip": [ - "10.105.21.199", - "209.73.177.115" + "209.73.177.115", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -5516,8 +5743,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_MISS" + "TCP_MISS", + "CONNECT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -5577,8 +5804,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "213.160.98.167", - "10.105.33.214" + "10.105.33.214", + "213.160.98.167" ], "related.user": [ "adeolaegbedokun" @@ -5589,8 +5816,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -5613,8 +5840,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", + "url.path": "/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5650,8 +5880,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5662,8 +5892,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5686,8 +5916,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5721,8 +5954,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_IMS_HIT" + "TCP_IMS_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5745,8 +5978,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", + "url.path": "/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5818,8 +6054,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5891,8 +6130,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", + "url.path": "/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5928,8 +6170,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.167" + "213.160.98.167", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5940,8 +6182,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5964,8 +6206,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5999,8 +6244,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "GET" + "GET", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -6024,7 +6269,9 @@ ], "url.domain": "www.google.com", "url.original": "http://www.google.com/supported_domains", + "url.path": "/supported_domains", "url.registered_domain": "google.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "-" @@ -6082,8 +6329,12 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/en-us/agent5/chknews.asp?", + "url.path": "/apps/agent/en-us/agent5/chknews.asp", + "url.query": "", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -6114,8 +6365,8 @@ "launch.adserver.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.109.125.112" + "216.109.125.112", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -6126,8 +6377,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -6151,7 +6402,10 @@ ], "url.domain": "launch.adserver.yahoo.com", "url.original": "http://launch.adserver.yahoo.com/l?", + "url.path": "/l", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "launch.adserver", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -6221,7 +6475,10 @@ ], "url.domain": "uk.f250.mail.yahoo.com", "url.original": "http://uk.f250.mail.yahoo.com/dc/launch?", + "url.path": "/dc/launch", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "uk.f250.mail", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6314,8 +6571,8 @@ "us.js2.yimg.com" ], "related.ip": [ - "213.160.98.169", - "10.105.21.199" + "10.105.21.199", + "213.160.98.169" ], "related.user": [ "badeyek" @@ -6326,8 +6583,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_SWAPFAIL_MISS", - "GET" + "GET", + "TCP_SWAPFAIL_MISS" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "200", @@ -6350,8 +6607,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/77cf3e56414f974dfd8616f56f0f632c_1.js", + "url.path": "/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/77cf3e56414f974dfd8616f56f0f632c_1.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6409,8 +6669,11 @@ "forwarded" ], "url.domain": "us.js1.yimg.com", + "url.extension": "css", "url.original": "http://us.js1.yimg.com/us.yimg.com/lib/hdr/ygma5.css", + "url.path": "/us.yimg.com/lib/hdr/ygma5.css", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js1", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6446,8 +6709,8 @@ "us.js2.yimg.com" ], "related.ip": [ - "213.160.98.169", - "10.105.21.199" + "10.105.21.199", + "213.160.98.169" ], "related.user": [ "badeyek" @@ -6482,8 +6745,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/f7fc76100697c9c2d25dd0ec35e563b0_1.js", + "url.path": "/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/f7fc76100697c9c2d25dd0ec35e563b0_1.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6541,8 +6807,11 @@ "forwarded" ], "url.domain": "us.js1.yimg.com", + "url.extension": "js", "url.original": "http://us.js1.yimg.com/us.yimg.com/lib/pim/r/medici/13_15/mail/ac.js", + "url.path": "/us.yimg.com/lib/pim/r/medici/13_15/mail/ac.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js1", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6576,8 +6845,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "200", @@ -6600,8 +6869,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/common/utils/2/yahoo_2.0.0-b4.js", + "url.path": "/us.js.yimg.com/lib/common/utils/2/yahoo_2.0.0-b4.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6635,8 +6907,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -6659,8 +6931,11 @@ "forwarded" ], "url.domain": "us.i1.yimg.com", + "url.extension": "gif", "url.original": "http://us.i1.yimg.com/us.yimg.com/i/us/pim/dclient/d/img/liam_ball_1.gif", + "url.path": "/us.yimg.com/i/us/pim/dclient/d/img/liam_ball_1.gif", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.i1", "url.top_level_domain": "com", "user.name": "badeyek" diff --git a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json index 69a23c8a5717..070d7b72259a 100644 --- a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json @@ -18,8 +18,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "example.net" + "example.net", + "example.org" ], "related.ip": [ "10.234.224.44", @@ -55,9 +55,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "min", "url.original": "https://example.org/exercita/der.htm?odoco=ria#min", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/exercita/der.htm" + ], + "url.query": "odoco=ria", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "tation", "user_agent.device.name": "Samsung SM-A715F", @@ -87,8 +94,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www.example.org" + "www.example.org", + "example.com" ], "related.ip": [ "10.102.123.34", @@ -125,9 +132,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "nidolor", "url.original": "https://www.example.org/enderitq/sperna.txt?billoi=oreetdol#nidolor", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/enderitq/sperna.txt" + ], + "url.query": "billoi=oreetdol", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "doeiu", @@ -158,8 +172,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "example.net" + "example.net", + "internal.example.com" ], "related.ip": [ "10.142.172.64", @@ -196,9 +210,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "iin", "url.original": "https://internal.example.com/aqu/utper.jpg?eFinib=omm#iin", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/aqu/utper.jpg" + ], + "url.query": "eFinib=omm", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "tia", @@ -228,8 +249,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.org" + "api.example.org", + "mail.example.com" ], "related.ip": [ "10.44.134.153", @@ -240,8 +261,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "GET" + "GET", + "allow" ], "rsa.misc.content_type": "taev", "rsa.misc.result_code": "quiavo", @@ -267,9 +288,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "con", "url.original": "https://api.example.org/ceroinBC/ratvolup.gif?iatu=ionofde#con", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ceroinBC/ratvolup.gif" + ], + "url.query": "iatu=ionofde", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user_agent.device.name": "5024D_RU", @@ -337,9 +365,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "eos", "url.original": "https://api.example.org/isetq/estqui.gif?magn=equuntu#eos", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/isetq/estqui.gif" + ], + "url.query": "magn=equuntu", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "onev", @@ -374,8 +409,8 @@ "api.example.com" ], "related.ip": [ - "10.175.107.139", - "10.12.195.60" + "10.12.195.60", + "10.175.107.139" ], "related.user": [ "mrema" @@ -385,8 +420,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "HEAD", - "accept" + "accept", + "HEAD" ], "rsa.misc.content_type": "aboris", "rsa.misc.result_code": "natura", @@ -411,9 +446,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "ect", "url.original": "https://www.example.org/inesci/rsitvolu.txt?pori=occ#ect", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/inesci/rsitvolu.txt" + ], + "url.query": "pori=occ", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "mrema", @@ -482,9 +524,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "atat", "url.original": "https://www5.example.com/ari/eataevit.txt?iam=mqua#atat", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/ari/eataevit.txt" + ], + "url.query": "iam=mqua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "tsed", @@ -515,8 +564,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "internal.example.net" + "internal.example.net", + "example.org" ], "related.ip": [ "10.116.120.216", @@ -527,8 +576,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "accept" + "accept", + "PROPFIND" ], "rsa.misc.content_type": "mol", "rsa.misc.result_code": "apariat", @@ -552,9 +601,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "orsitame", "url.original": "https://example.org/tatno/imav.htm?ofdeF=tion#orsitame", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/tatno/imav.htm" + ], + "url.query": "ofdeF=tion", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "umdo", "user_agent.device.name": "Notepad_K10", @@ -588,8 +644,8 @@ "example.org" ], "related.ip": [ - "10.90.131.186", - "10.30.216.41" + "10.30.216.41", + "10.90.131.186" ], "related.user": [ "saute" @@ -624,9 +680,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "iumto", "url.original": "https://example.org/uaera/sitas.txt?aedic=atquovo#iumto", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/uaera/sitas.txt" + ], + "url.query": "aedic=atquovo", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "saute", "user_agent.device.name": "STK-L21", @@ -696,9 +759,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "nsequu", "url.original": "https://mail.example.org/edquiano/loru.htm?end=enia#nsequu", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/edquiano/loru.htm" + ], + "url.query": "end=enia", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "Nemoeni", @@ -732,16 +802,16 @@ "www.example.org" ], "related.ip": [ - "10.158.185.163", - "10.71.34.9" + "10.71.34.9", + "10.158.185.163" ], "related.user": [ "aliq" ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "allow", - "UNLOCK" + "UNLOCK", + "allow" ], "rsa.misc.content_type": "adm", "rsa.misc.result_code": "quam", @@ -766,9 +836,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "xercitat", "url.original": "https://www.example.org/iqui/etc.txt?tatiset=eprehen#xercitat", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/iqui/etc.txt" + ], + "url.query": "tatiset=eprehen", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "aliq", @@ -799,8 +876,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "internal.example.net" + "internal.example.net", + "example.net" ], "related.ip": [ "10.210.74.24", @@ -811,8 +888,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "OPTIONS", - "accept" + "accept", + "OPTIONS" ], "rsa.misc.content_type": "emips", "rsa.misc.result_code": "onse", @@ -837,9 +914,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "txt", + "url.fragment": "ugiatnu", "url.original": "https://internal.example.net/ommod/sequatur.txt?tlabo=suntexp#ugiatnu", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/ommod/sequatur.txt" + ], + "url.query": "tlabo=suntexp", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "uines", @@ -866,8 +950,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "www.example.net" + "www.example.net", + "api.example.org" ], "related.ip": [ "10.206.136.206", @@ -904,9 +988,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "eporroqu", "url.original": "https://api.example.org/oriosamn/deFinibu.gif?iciatisu=rehender#eporroqu", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/oriosamn/deFinibu.gif" + ], + "url.query": "iciatisu=rehender", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "xeac", @@ -937,8 +1028,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.org", - "www.example.com" + "www.example.com", + "mail.example.org" ], "related.ip": [ "10.200.199.166", @@ -975,9 +1066,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "civeli", "url.original": "https://mail.example.org/rehend/tio.html?numqu=qui#civeli", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/rehend/tio.html" + ], + "url.query": "numqu=qui", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "ipitla", @@ -1008,8 +1106,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "www5.example.com" + "www5.example.com", + "www.example.org" ], "related.ip": [ "10.122.46.71", @@ -1020,8 +1118,8 @@ ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "NONE", - "allow" + "allow", + "NONE" ], "rsa.misc.content_type": "aaliquaU", "rsa.misc.result_code": "mpori", @@ -1046,9 +1144,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "iamquis", "url.original": "https://www.example.org/eav/ionevo.txt?siar=orev#iamquis", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/eav/ionevo.txt" + ], + "url.query": "siar=orev", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "aturve", @@ -1083,16 +1188,16 @@ "api.example.com" ], "related.ip": [ - "10.249.213.83", - "10.164.250.63" + "10.164.250.63", + "10.249.213.83" ], "related.user": [ "itame" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "accept" + "accept", + "PROPFIND" ], "rsa.misc.content_type": "asun", "rsa.misc.result_code": "lit", @@ -1117,9 +1222,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "leumiu", "url.original": "https://internal.example.net/wri/bor.jpg?hitect=dol#leumiu", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/wri/bor.jpg" + ], + "url.query": "hitect=dol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "itame", @@ -1150,12 +1262,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.net" + "api.example.net", + "www5.example.net" ], "related.ip": [ - "10.236.248.65", - "10.61.242.75" + "10.61.242.75", + "10.236.248.65" ], "related.user": [ "iquidex" @@ -1191,9 +1303,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "eriamea", "url.original": "https://api.example.net/equat/doloreme.htm?ione=ihilmole#eriamea", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/equat/doloreme.htm" + ], + "url.query": "ione=ihilmole", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "iquidex", @@ -1224,20 +1343,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www5.example.com" + "www5.example.com", + "internal.example.net" ], "related.ip": [ - "10.13.59.31", - "10.214.7.83" + "10.214.7.83", + "10.13.59.31" ], "related.user": [ "etdol" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "block", - "PUT" + "PUT", + "block" ], "rsa.misc.content_type": "eprehend", "rsa.misc.result_code": "boN", @@ -1262,9 +1381,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "xerc", "url.original": "https://www5.example.com/estia/tper.gif?volupt=osqui#xerc", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/estia/tper.gif" + ], + "url.query": "volupt=osqui", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "etdol", @@ -1310,8 +1436,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "GET" + "GET", + "accept" ], "rsa.misc.content_type": "tcons", "rsa.misc.result_code": "tsu", @@ -1336,9 +1462,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "isnis", "url.original": "https://internal.example.net/sin/rvel.htm?nimid=itatione#isnis", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/sin/rvel.htm" + ], + "url.query": "nimid=itatione", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "isnisiu", @@ -1369,8 +1502,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "mail.example.net" + "mail.example.net", + "api.example.org" ], "related.ip": [ "10.90.86.89", @@ -1407,9 +1540,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "equat", "url.original": "https://api.example.org/abo/veniamqu.gif?aliquide=ofde#equat", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/abo/veniamqu.gif" + ], + "url.query": "aliquide=ofde", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "lapar", @@ -1440,20 +1580,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.org" + "api.example.org", + "www5.example.net" ], "related.ip": [ - "10.14.211.43", - "10.14.48.16" + "10.14.48.16", + "10.14.211.43" ], "related.user": [ "volupt" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "cancel" + "cancel", + "PROPFIND" ], "rsa.misc.content_type": "Utenima", "rsa.misc.result_code": "uiinea", @@ -1478,9 +1618,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "icabo", "url.original": "https://api.example.org/autfu/gnaaliq.jpg?olupta=litse#icabo", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/autfu/gnaaliq.jpg" + ], + "url.query": "olupta=litse", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "volupt", @@ -1515,8 +1662,8 @@ "example.com" ], "related.ip": [ - "10.47.25.230", - "10.93.123.174" + "10.93.123.174", + "10.47.25.230" ], "related.user": [ "reetdolo" @@ -1525,8 +1672,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "iusmodi", "rsa.misc.result_code": "etcons", @@ -1551,9 +1698,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "ntocc", "url.original": "https://internal.example.net/ptatemq/luptatev.html?Nequepo=ipsumd#ntocc", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/ptatemq/luptatev.html" + ], + "url.query": "Nequepo=ipsumd", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "reetdolo", @@ -1584,8 +1738,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "www5.example.com" + "www5.example.com", + "www5.example.net" ], "related.ip": [ "10.233.48.103", @@ -1622,9 +1776,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "cons", "url.original": "https://www5.example.net/quiavol/rrorsi.gif?iatisu=sec#cons", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/quiavol/rrorsi.gif" + ], + "url.query": "iatisu=sec", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "tlab", @@ -1655,12 +1816,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.net" + "api.example.net", + "mail.example.com" ], "related.ip": [ - "10.93.220.10", - "10.27.58.92" + "10.27.58.92", + "10.93.220.10" ], "related.user": [ "qui" @@ -1693,9 +1854,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "edquia", "url.original": "https://api.example.net/orain/tiumt.jpg?litessec=itas#edquia", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/orain/tiumt.jpg" + ], + "url.query": "litessec=itas", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "qui", @@ -1730,8 +1898,8 @@ "example.net" ], "related.ip": [ - "10.213.144.249", - "10.135.217.12" + "10.135.217.12", + "10.213.144.249" ], "related.user": [ "ntexplic" @@ -1764,9 +1932,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "untut", "url.original": "https://www.example.net/str/idolore.txt?eetdolo=cteturad#untut", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/str/idolore.txt" + ], + "url.query": "eetdolo=cteturad", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "ntexplic", @@ -1801,8 +1976,8 @@ "internal.example.com" ], "related.ip": [ - "10.13.226.57", - "10.233.239.112" + "10.233.239.112", + "10.13.226.57" ], "related.user": [ "mquelau" @@ -1835,9 +2010,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "mestq", "url.original": "https://mail.example.net/velitse/oditem.html?torever=oremi#mestq", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/velitse/oditem.html" + ], + "url.query": "torever=oremi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "mquelau", @@ -1872,8 +2054,8 @@ "example.com" ], "related.ip": [ - "10.161.203.252", - "10.21.169.127" + "10.21.169.127", + "10.161.203.252" ], "related.user": [ "ice" @@ -1908,9 +2090,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "uisa", "url.original": "https://internal.example.org/isnisi/ritatise.gif?tamet=quatur#uisa", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/isnisi/ritatise.gif" + ], + "url.query": "tamet=quatur", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "ice", @@ -1941,20 +2130,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.net", - "api.example.com" + "api.example.com", + "www.example.net" ], "related.ip": [ - "10.17.215.111", - "10.69.139.26" + "10.69.139.26", + "10.17.215.111" ], "related.user": [ "edqui" ], "rsa.internal.messageid": "LOCK", "rsa.misc.action": [ - "LOCK", - "block" + "block", + "LOCK" ], "rsa.misc.content_type": "volupta", "rsa.misc.result_code": "veli", @@ -1979,9 +2168,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "ano", "url.original": "https://www.example.net/ianon/tsed.htm?ameiusm=proide#ano", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/ianon/tsed.htm" + ], + "url.query": "ameiusm=proide", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "edqui", @@ -2024,8 +2220,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "accept", - "COPY" + "COPY", + "accept" ], "rsa.misc.content_type": "onulam", "rsa.misc.result_code": "ugiat", @@ -2050,9 +2246,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "apariat", "url.original": "https://www5.example.org/ncididun/umSe.jpg?ise=itau#apariat", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/ncididun/umSe.jpg" + ], + "url.query": "ise=itau", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "onsecte", @@ -2095,8 +2298,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "block", - "UNLOCK" + "UNLOCK", + "block" ], "rsa.misc.content_type": "emUteni", "rsa.misc.result_code": "utlab", @@ -2120,9 +2323,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "tem", "url.original": "https://example.org/uatur/adminimv.gif?exeacom=roidents#tem", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/uatur/adminimv.gif" + ], + "url.query": "exeacom=roidents", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "isis", "user_agent.device.name": "Generic Smartphone", @@ -2154,16 +2364,16 @@ "mail.example.com" ], "related.ip": [ - "10.26.96.202", - "10.119.90.128" + "10.119.90.128", + "10.26.96.202" ], "related.user": [ "oraincid" ], "rsa.internal.messageid": "ICP_QUERY", "rsa.misc.action": [ - "ICP_QUERY", - "cancel" + "cancel", + "ICP_QUERY" ], "rsa.misc.content_type": "ntsunt", "rsa.misc.result_code": "amcorp", @@ -2188,9 +2398,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "reprehe", "url.original": "https://api.example.com/ven/rQu.html?doloreme=dun#reprehe", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ven/rQu.html" + ], + "url.query": "doloreme=dun", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "oraincid", @@ -2221,12 +2438,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.net" + "api.example.net", + "mail.example.com" ], "related.ip": [ - "10.0.98.205", - "10.76.110.144" + "10.76.110.144", + "10.0.98.205" ], "related.user": [ "upt" @@ -2262,9 +2479,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "ate", "url.original": "https://api.example.net/eseru/quamest.html?qua=rsita#ate", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/eseru/quamest.html" + ], + "url.query": "qua=rsita", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "upt", @@ -2299,16 +2523,16 @@ "www.example.org" ], "related.ip": [ - "10.224.11.165", - "10.135.46.242" + "10.135.46.242", + "10.224.11.165" ], "related.user": [ "xeaco" ], "rsa.internal.messageid": "MOVE", "rsa.misc.action": [ - "MOVE", - "deny" + "deny", + "MOVE" ], "rsa.misc.content_type": "elites", "rsa.misc.result_code": "oremi", @@ -2333,9 +2557,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "isiu", "url.original": "https://internal.example.org/mvolu/conse.txt?aincidu=nimadmin#isiu", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/mvolu/conse.txt" + ], + "url.query": "aincidu=nimadmin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "xeaco", @@ -2377,8 +2608,8 @@ ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "TRACE", - "accept" + "accept", + "TRACE" ], "rsa.misc.content_type": "plica", "rsa.misc.result_code": "cidunt", @@ -2403,9 +2634,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "html", + "url.fragment": "ilmol", "url.original": "https://internal.example.org/isciv/rroqu.html?uisa=tametco#ilmol", - "url.path": "https://internal.example.org", + "url.path": [ + "https://internal.example.org", + "/isciv/rroqu.html" + ], + "url.query": "uisa=tametco", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "autodit", @@ -2440,16 +2678,16 @@ "example.org" ], "related.ip": [ - "10.93.39.140", - "10.150.245.88" + "10.150.245.88", + "10.93.39.140" ], "related.user": [ "reetd" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "cancel", - "COPY" + "COPY", + "cancel" ], "rsa.misc.content_type": "iusmodte", "rsa.misc.result_code": "ntutlabo", @@ -2474,9 +2712,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "pid", "url.original": "https://www5.example.net/uaeratv/isa.txt?periam=dqu#pid", - "url.path": "https://example.org", + "url.path": [ + "https://example.org", + "/uaeratv/isa.txt" + ], + "url.query": "periam=dqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "reetd", @@ -2507,12 +2752,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "www5.example.com" + "www5.example.com", + "mail.example.net" ], "related.ip": [ - "10.61.92.2", - "10.73.207.70" + "10.73.207.70", + "10.61.92.2" ], "related.user": [ "atu" @@ -2545,9 +2790,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "remagn", "url.original": "https://www5.example.com/orroq/vitaedic.txt?orisni=ons#remagn", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/orroq/vitaedic.txt" + ], + "url.query": "orisni=ons", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "atu", @@ -2577,12 +2829,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "internal.example.com" + "internal.example.com", + "example.net" ], "related.ip": [ - "10.84.32.178", - "10.50.124.116" + "10.50.124.116", + "10.84.32.178" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", @@ -2615,9 +2867,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "ruredo", "url.original": "https://example.net/mven/olorsit.gif?oremag=illu#ruredo", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/mven/olorsit.gif" + ], + "url.query": "oremag=illu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -2646,8 +2905,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "www5.example.net" + "www5.example.net", + "api.example.org" ], "related.ip": [ "10.173.222.131", @@ -2658,8 +2917,8 @@ ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "accept", - "TRACE" + "TRACE", + "accept" ], "rsa.misc.content_type": "boNem", "rsa.misc.result_code": "aUt", @@ -2684,9 +2943,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "html", + "url.fragment": "oin", "url.original": "https://www5.example.net/rum/eataevi.html?ulla=iqu#oin", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/rum/eataevi.html" + ], + "url.query": "ulla=iqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "Duisa", @@ -2717,12 +2983,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "mail.example.net" + "mail.example.net", + "www5.example.net" ], "related.ip": [ - "10.0.157.225", - "10.11.83.126" + "10.11.83.126", + "10.0.157.225" ], "related.user": [ "atu" @@ -2755,9 +3021,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "tinvolup", "url.original": "https://mail.example.net/osquir/mod.txt?fugitse=imad#tinvolup", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/osquir/mod.txt" + ], + "url.query": "fugitse=imad", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "atu", @@ -2788,20 +3061,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.com", - "api.example.com" + "api.example.com", + "www5.example.com" ], "related.ip": [ - "10.228.77.21", - "10.92.237.93" + "10.92.237.93", + "10.228.77.21" ], "related.user": [ "onse" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "cancel" + "cancel", + "PUT" ], "rsa.misc.content_type": "mod", "rsa.misc.result_code": "gnaa", @@ -2826,9 +3099,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "txt", + "url.fragment": "mve", "url.original": "https://api.example.com/asper/umq.txt?itasper=uae#mve", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/asper/umq.txt" + ], + "url.query": "itasper=uae", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "onse", @@ -2859,12 +3139,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www5.example.net" + "www5.example.net", + "api.example.com" ], "related.ip": [ - "10.102.215.23", - "10.20.28.92" + "10.20.28.92", + "10.102.215.23" ], "related.user": [ "ntexpl" @@ -2899,9 +3179,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "jpg", + "url.fragment": "veniamqu", "url.original": "https://www5.example.net/quatD/isqua.jpg?oloreseo=iruredol#veniamqu", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/quatD/isqua.jpg" + ], + "url.query": "oloreseo=iruredol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "ntexpl", @@ -2934,16 +3221,16 @@ "api.example.net" ], "related.ip": [ - "10.45.28.159", - "10.17.87.79" + "10.17.87.79", + "10.45.28.159" ], "related.user": [ "tionula" ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "NONE", - "block" + "block", + "NONE" ], "rsa.misc.content_type": "uamei", "rsa.misc.result_code": "ecatcupi", @@ -2968,9 +3255,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "repreh", "url.original": "https://api.example.net/ation/luptas.html?iatqu=lorsi#repreh", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/ation/luptas.html" + ], + "url.query": "iatqu=lorsi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "tionula", @@ -3001,8 +3295,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "api.example.com" + "api.example.com", + "internal.example.com" ], "related.ip": [ "10.177.238.45", @@ -3039,9 +3333,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "eumi", "url.original": "https://internal.example.com/mpo/unte.jpg?ueipsa=scipitl#eumi", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/mpo/unte.jpg" + ], + "url.query": "ueipsa=scipitl", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "rsp", @@ -3072,8 +3373,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "example.com" + "example.com", + "www.example.org" ], "related.ip": [ "10.101.85.169", @@ -3113,9 +3414,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "olup", "url.original": "https://www.example.org/uov/quaeab.jpg?moles=dipiscin#olup", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uov/quaeab.jpg" + ], + "url.query": "moles=dipiscin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "liquid", @@ -3146,12 +3454,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "www5.example.org" + "www5.example.org", + "example.net" ], "related.ip": [ - "10.24.54.129", - "10.231.7.209" + "10.231.7.209", + "10.24.54.129" ], "related.user": [ "eavol" @@ -3161,8 +3469,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "HEAD", - "block" + "block", + "HEAD" ], "rsa.misc.content_type": "scipitl", "rsa.misc.result_code": "temaccu", @@ -3186,9 +3494,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "snos", "url.original": "https://example.net/edolo/ugiatquo.jpg?eosquira=pta#snos", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/edolo/ugiatquo.jpg" + ], + "url.query": "eosquira=pta", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user.name": "eavol", "user_agent.device.name": "Asus X01BDA", @@ -3222,8 +3537,8 @@ "www.example.com" ], "related.ip": [ - "10.121.163.5", - "10.77.129.175" + "10.77.129.175", + "10.121.163.5" ], "related.user": [ "BCS" @@ -3232,8 +3547,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "CONNECT" + "CONNECT", + "allow" ], "rsa.misc.content_type": "incidid", "rsa.misc.result_code": "ugiatquo", @@ -3258,9 +3573,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "nrepreh", "url.original": "https://api.example.org/isci/dolor.htm?orinrep=quiavol#nrepreh", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/isci/dolor.htm" + ], + "url.query": "orinrep=quiavol", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "BCS", @@ -3329,9 +3651,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "culpaqui", "url.original": "https://mail.example.com/roide/tem.gif?rerepre=nculpaq#culpaqui", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/roide/tem.gif" + ], + "url.query": "rerepre=nculpaq", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "obea", @@ -3374,8 +3703,8 @@ ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "block", - "NONE" + "NONE", + "block" ], "rsa.misc.content_type": "tobeatae", "rsa.misc.result_code": "tion", @@ -3400,9 +3729,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "edquia", "url.original": "https://internal.example.net/rautod/olest.jpg?lapar=ritati#edquia", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/rautod/olest.jpg" + ], + "url.query": "lapar=ritati", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "amvolu", @@ -3471,9 +3807,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "iame", "url.original": "https://api.example.org/iusmodt/enim.txt?aquio=ersp#iame", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/iusmodt/enim.txt" + ], + "url.query": "aquio=ersp", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "mexer", @@ -3508,8 +3851,8 @@ "example.org" ], "related.ip": [ - "10.45.54.107", - "10.45.114.111" + "10.45.114.111", + "10.45.54.107" ], "related.user": [ "nse" @@ -3518,8 +3861,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "accept" + "accept", + "POST" ], "rsa.misc.content_type": "mdolors", "rsa.misc.result_code": "edictasu", @@ -3543,9 +3886,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "eetdo", "url.original": "https://example.org/abillo/undeom.html?oraincid=quaer#eetdo", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/abillo/undeom.html" + ], + "url.query": "oraincid=quaer", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "nse", "user_agent.device.name": "Samsung SM-A715F", @@ -3575,8 +3925,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "mail.example.net" + "mail.example.net", + "www5.example.net" ], "related.ip": [ "10.205.28.24", @@ -3613,9 +3963,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "sse", "url.original": "https://mail.example.net/uam/orumSec.jpg?isnisiu=suntincu#sse", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/uam/orumSec.jpg" + ], + "url.query": "isnisiu=suntincu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "untutlab", @@ -3661,8 +4018,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "HEAD" + "HEAD", + "deny" ], "rsa.misc.content_type": "elites", "rsa.misc.result_code": "itseddoe", @@ -3687,9 +4044,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "iamea", "url.original": "https://www5.example.org/umiurer/rere.txt?mnisi=usmo#iamea", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/umiurer/rere.txt" + ], + "url.query": "mnisi=usmo", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "odoco", @@ -3716,20 +4080,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "internal.example.net" + "internal.example.net", + "internal.example.org" ], "related.ip": [ - "10.81.140.173", - "10.88.172.222" + "10.88.172.222", + "10.81.140.173" ], "related.user": [ "etdol" ], "rsa.internal.messageid": "MOVE", "rsa.misc.action": [ - "cancel", - "MOVE" + "MOVE", + "cancel" ], "rsa.misc.content_type": "tura", "rsa.misc.result_code": "nturmag", @@ -3754,9 +4118,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "remipsu", "url.original": "https://internal.example.net/atnula/ditautf.jpg?iquidex=olup#remipsu", - "url.path": "https://internal.example.org", + "url.path": [ + "https://internal.example.org", + "/atnula/ditautf.jpg" + ], + "url.query": "iquidex=olup", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "etdol", @@ -3787,8 +4158,8 @@ "example.com" ], "related.ip": [ - "10.247.53.179", - "10.162.129.196" + "10.162.129.196", + "10.247.53.179" ], "related.user": [ "identsu" @@ -3821,9 +4192,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "xeacommo", "url.original": "https://api.example.net/uscip/umS.txt?quiacons=uisa#xeacommo", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uscip/umS.txt" + ], + "url.query": "quiacons=uisa", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "identsu", @@ -3854,16 +4232,16 @@ "www5.example.net" ], "related.ip": [ - "10.110.86.230", - "10.172.148.223" + "10.172.148.223", + "10.110.86.230" ], "related.user": [ "enimadm" ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "accept", - "UNLOCK" + "UNLOCK", + "accept" ], "rsa.misc.content_type": "atc", "rsa.misc.result_code": "upta", @@ -3888,9 +4266,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "temporai", "url.original": "https://mail.example.com/mrema/ullamc.txt?eufug=roquisq#temporai", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/mrema/ullamc.txt" + ], + "url.query": "eufug=roquisq", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "enimadm", @@ -3962,9 +4347,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "orsitam", "url.original": "https://www5.example.com/isau/itinvol.txt?saquaea=ons#orsitam", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/isau/itinvol.txt" + ], + "url.query": "saquaea=ons", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "riame", @@ -4033,9 +4425,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "onemulla", "url.original": "https://api.example.net/veli/venia.htm?etdolor=uat#onemulla", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/veli/venia.htm" + ], + "url.query": "etdolor=uat", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "asp", @@ -4066,12 +4465,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "internal.example.net" + "internal.example.net", + "www5.example.net" ], "related.ip": [ - "10.89.73.240", - "10.41.156.88" + "10.41.156.88", + "10.89.73.240" ], "related.user": [ "ntorever" @@ -4104,9 +4503,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "expli", "url.original": "https://internal.example.net/ainci/osqu.jpg?sus=imavenia#expli", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/ainci/osqu.jpg" + ], + "url.query": "sus=imavenia", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "ntorever", @@ -4137,8 +4543,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www.example.org" + "www.example.org", + "internal.example.net" ], "related.ip": [ "10.54.44.231", @@ -4151,8 +4557,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "allow" + "allow", + "CONNECT" ], "rsa.misc.content_type": "oreverit", "rsa.misc.result_code": "abor", @@ -4177,9 +4583,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "seos", "url.original": "https://www.example.org/runtm/eturadip.htm?psumd=oloree#seos", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/runtm/eturadip.htm" + ], + "url.query": "psumd=oloree", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "mcorpo", @@ -4206,8 +4619,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "www.example.net" + "www.example.net", + "internal.example.org" ], "related.ip": [ "10.181.177.74", @@ -4244,9 +4657,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "html", + "url.fragment": "umwri", "url.original": "https://internal.example.org/liquipex/uisnos.html?ventor=lupt#umwri", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/liquipex/uisnos.html" + ], + "url.query": "ventor=lupt", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "nvo", @@ -4281,8 +4701,8 @@ "api.example.net" ], "related.ip": [ - "10.83.130.95", - "10.76.220.3" + "10.76.220.3", + "10.83.130.95" ], "related.user": [ "userror" @@ -4318,9 +4738,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "veniamq", "url.original": "https://api.example.org/mquisnos/lore.txt?siar=isn#veniamq", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/mquisnos/lore.txt" + ], + "url.query": "siar=isn", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "userror", @@ -4351,12 +4778,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www.example.com" + "www.example.com", + "api.example.com" ], "related.ip": [ - "10.166.160.217", - "10.219.245.58" + "10.219.245.58", + "10.166.160.217" ], "related.user": [ "radip" @@ -4389,9 +4816,16 @@ "forwarded" ], "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "uptatem", "url.original": "https://www.example.com/quas/occaeca.htm?ender=dico#uptatem", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/quas/occaeca.htm" + ], + "url.query": "ender=dico", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "radip", @@ -4420,8 +4854,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.org" + "www5.example.org", + "example.com" ], "related.ip": [ "10.121.121.153", @@ -4432,8 +4866,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "cancel", - "UNLOCK" + "UNLOCK", + "cancel" ], "rsa.misc.content_type": "magnam", "rsa.misc.result_code": "modoc", @@ -4458,9 +4892,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "sit", "url.original": "https://www5.example.org/uidolo/umdolore.jpg?oquisq=abori#sit", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uidolo/umdolore.jpg" + ], + "url.query": "oquisq=abori", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "tatio", @@ -4529,9 +4970,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "tisu", "url.original": "https://internal.example.net/eniamqu/inimav.htm?imadm=uta#tisu", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/eniamqu/inimav.htm" + ], + "url.query": "imadm=uta", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "aturv", @@ -4562,8 +5010,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "mail.example.com" + "mail.example.com", + "mail.example.net" ], "related.ip": [ "10.72.99.69", @@ -4600,9 +5048,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "giatquo", "url.original": "https://mail.example.net/sintocca/mipsumqu.htm?tnulapar=ico#giatquo", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/sintocca/mipsumqu.htm" + ], + "url.query": "tnulapar=ico", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "uatu", @@ -4633,20 +5088,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "internal.example.net" + "internal.example.net", + "api.example.org" ], "related.ip": [ - "10.245.240.47", - "10.142.130.227" + "10.142.130.227", + "10.245.240.47" ], "related.user": [ "odic" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "allow", - "DELETE" + "DELETE", + "allow" ], "rsa.misc.content_type": "scivelit", "rsa.misc.result_code": "liquaUte", @@ -4671,9 +5126,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "eleumi", "url.original": "https://api.example.org/rep/remap.html?siarc=fdeFin#eleumi", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/rep/remap.html" + ], + "url.query": "siarc=fdeFin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "odic", @@ -4708,8 +5170,8 @@ "api.example.net" ], "related.ip": [ - "10.62.188.193", - "10.61.110.7" + "10.61.110.7", + "10.62.188.193" ], "related.user": [ "quaU" @@ -4742,9 +5204,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "osa", "url.original": "https://api.example.net/eturad/tDuis.htm?enimadmi=tateveli#osa", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/eturad/tDuis.htm" + ], + "url.query": "enimadmi=tateveli", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "quaU", @@ -4771,8 +5240,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "www.example.net" + "www.example.net", + "mail.example.net" ], "related.ip": [ "10.68.198.188", @@ -4783,8 +5252,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "COPY", - "block" + "block", + "COPY" ], "rsa.misc.content_type": "animid", "rsa.misc.result_code": "inea", @@ -4809,9 +5278,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "madmi", "url.original": "https://www.example.net/hender/ptatemU.htm?mquisnos=tnulapa#madmi", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/hender/ptatemU.htm" + ], + "url.query": "mquisnos=tnulapa", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "onsectet", @@ -4856,8 +5332,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "teturad", "rsa.misc.result_code": "avolu", @@ -4882,9 +5358,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "oid", "url.original": "https://www.example.org/iduntutl/rsitam.htm?ntor=oinBCSed#oid", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/iduntutl/rsitam.htm" + ], + "url.query": "ntor=oinBCSed", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "midestl", @@ -4917,16 +5400,16 @@ "api.example.org" ], "related.ip": [ - "10.62.10.137", - "10.32.98.109" + "10.32.98.109", + "10.62.10.137" ], "related.user": [ "deomnisi" ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "cancel" + "cancel", + "PURGE" ], "rsa.misc.content_type": "laboree", "rsa.misc.result_code": "oll", @@ -4950,9 +5433,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "dat", "url.original": "https://example.org/itessequ/porissu.html?uip=ectobea#dat", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/itessequ/porissu.html" + ], + "url.query": "uip=ectobea", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "deomnisi", "user_agent.device.name": "LG-$2", @@ -4986,8 +5476,8 @@ "www5.example.com" ], "related.ip": [ - "10.176.62.146", - "10.255.40.12" + "10.255.40.12", + "10.176.62.146" ], "related.user": [ "oeiusmo" @@ -5020,9 +5510,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "nimide", "url.original": "https://api.example.net/enimad/tis.txt?mipsumq=ident#nimide", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/enimad/tis.txt" + ], + "url.query": "mipsumq=ident", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "oeiusmo", @@ -5056,8 +5553,8 @@ "internal.example.com" ], "related.ip": [ - "10.194.198.46", - "10.88.98.31" + "10.88.98.31", + "10.194.198.46" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", @@ -5091,9 +5588,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "nse", "url.original": "https://api.example.org/taspe/yCiceroi.htm?cti=ommodoc#nse", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/taspe/yCiceroi.htm" + ], + "url.query": "cti=ommodoc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user_agent.device.name": "iPhone", @@ -5123,8 +5627,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.com" + "www5.example.com", + "example.com" ], "related.ip": [ "10.1.27.133", @@ -5135,8 +5639,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "block", - "OPTIONS" + "OPTIONS", + "block" ], "rsa.misc.content_type": "ntutlab", "rsa.misc.result_code": "olore", @@ -5160,9 +5664,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "iarchite", "url.original": "https://example.com/beat/rro.jpg?uisau=qua#iarchite", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/beat/rro.jpg" + ], + "url.query": "uisau=qua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "tationu", "user_agent.device.name": "Lenovo A2016a40 ", @@ -5192,12 +5703,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "www5.example.org" + "www5.example.org", + "internal.example.com" ], "related.ip": [ - "10.11.73.145", - "10.70.244.155" + "10.70.244.155", + "10.11.73.145" ], "related.user": [ "caboNemo" @@ -5206,8 +5717,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "POST" + "POST", + "accept" ], "rsa.misc.content_type": "ons", "rsa.misc.result_code": "temaccus", @@ -5232,9 +5743,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "dexea", "url.original": "https://www5.example.org/loremq/turmagni.txt?emUtenim=ende#dexea", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/loremq/turmagni.txt" + ], + "url.query": "emUtenim=ende", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "caboNemo", @@ -5303,9 +5821,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "ecte", "url.original": "https://www5.example.net/deomnisi/ddoe.txt?oremi=ectobeat#ecte", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/deomnisi/ddoe.txt" + ], + "url.query": "oremi=ectobeat", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "cillumdo", @@ -5374,9 +5899,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "lors", "url.original": "https://api.example.net/tiset/sci.jpg?rauto=doloreeu#lors", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/tiset/sci.jpg" + ], + "url.query": "rauto=doloreeu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "roquisq", @@ -5407,8 +5939,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "mail.example.com" + "mail.example.com", + "api.example.net" ], "related.ip": [ "10.191.220.1", @@ -5419,8 +5951,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "deny" + "deny", + "DELETE" ], "rsa.misc.content_type": "tdolo", "rsa.misc.result_code": "ntu", @@ -5445,9 +5977,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "tconsect", "url.original": "https://api.example.net/tDuisau/aturve.htm?tper=pisciv#tconsect", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/tDuisau/aturve.htm" + ], + "url.query": "tper=pisciv", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "isi", @@ -5482,8 +6021,8 @@ "www5.example.org" ], "related.ip": [ - "10.254.10.98", - "10.109.88.27" + "10.109.88.27", + "10.254.10.98" ], "related.user": [ "aparia" @@ -5516,9 +6055,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "scip", "url.original": "https://internal.example.com/oluptate/todi.jpg?tdolo=ident#scip", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/oluptate/todi.jpg" + ], + "url.query": "tdolo=ident", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "aparia", @@ -5587,9 +6133,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "html", + "url.fragment": "utlabore", "url.original": "https://mail.example.com/radipis/lore.html?civeli=eufugia#utlabore", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/radipis/lore.html" + ], + "url.query": "civeli=eufugia", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "onemul", @@ -5620,8 +6173,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "internal.example.org" + "internal.example.org", + "example.com" ], "related.ip": [ "10.18.199.203", @@ -5632,8 +6185,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "allow", - "PROPFIND" + "PROPFIND", + "allow" ], "rsa.misc.content_type": "llum", "rsa.misc.result_code": "ntut", @@ -5658,9 +6211,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "jpg", + "url.fragment": "emp", "url.original": "https://internal.example.org/olupta/tio.jpg?idestl=litani#emp", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/olupta/tio.jpg" + ], + "url.query": "idestl=litani", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "ittenb", @@ -5695,8 +6255,8 @@ "www5.example.org" ], "related.ip": [ - "10.1.220.47", - "10.73.80.251" + "10.73.80.251", + "10.1.220.47" ], "related.user": [ "ercitati" @@ -5729,9 +6289,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "Neq", "url.original": "https://www5.example.com/sequines/cto.gif?temaccu=uamqua#Neq", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/sequines/cto.gif" + ], + "url.query": "temaccu=uamqua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "ercitati", @@ -5766,16 +6333,16 @@ "api.example.org" ], "related.ip": [ - "10.153.109.61", - "10.22.34.206" + "10.22.34.206", + "10.153.109.61" ], "related.user": [ "mve" ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "block" + "block", + "PURGE" ], "rsa.misc.content_type": "velites", "rsa.misc.result_code": "uasiarch", @@ -5800,9 +6367,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "gif", + "url.fragment": "onorume", "url.original": "https://www.example.net/periam/ain.gif?iquipex=mqu#onorume", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/periam/ain.gif" + ], + "url.query": "iquipex=mqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "mve", @@ -5873,9 +6447,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "Duis", "url.original": "https://www.example.net/ecatc/quovolu.jpg?dexe=nemul#Duis", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ecatc/quovolu.jpg" + ], + "url.query": "dexe=nemul", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "ipsa", @@ -5910,16 +6491,16 @@ "www5.example.com" ], "related.ip": [ - "10.128.84.27", - "10.97.33.56" + "10.97.33.56", + "10.128.84.27" ], "related.user": [ "ptate" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "block", - "COPY" + "COPY", + "block" ], "rsa.misc.content_type": "utla", "rsa.misc.result_code": "umfu", @@ -5943,9 +6524,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "cipitla", "url.original": "https://example.com/tqui/ssequ.gif?emse=emqui#cipitla", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/tqui/ssequ.gif" + ], + "url.query": "emse=emqui", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "ptate", "user_agent.device.name": "Asus X01BDA", @@ -6015,9 +6603,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "quisnost", "url.original": "https://example.com/caboN/imipsam.jpg?catcupid=ritquiin#quisnost", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/caboN/imipsam.jpg" + ], + "url.query": "catcupid=ritquiin", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "ore", "user_agent.device.name": "Samsung SM-A260G", @@ -6047,8 +6642,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "internal.example.com" + "internal.example.com", + "www.example.org" ], "related.ip": [ "10.33.112.100", @@ -6085,9 +6680,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "osqu", "url.original": "https://www.example.org/oinvento/ali.htm?utaliqui=isciv#osqu", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/oinvento/ali.htm" + ], + "url.query": "utaliqui=isciv", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "enimad", @@ -6132,8 +6734,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "cancel", - "CONNECT" + "CONNECT", + "cancel" ], "rsa.misc.content_type": "urau", "rsa.misc.result_code": "lla", @@ -6158,9 +6760,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "rch", "url.original": "https://api.example.net/quido/llo.htm?tpersp=assi#rch", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/quido/llo.htm" + ], + "url.query": "tpersp=assi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "oremeu", @@ -6206,8 +6815,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "HEAD" + "HEAD", + "allow" ], "rsa.misc.content_type": "tevel", "rsa.misc.result_code": "taevitae", @@ -6232,9 +6841,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "deriti", "url.original": "https://www.example.net/tvolu/imve.txt?gnaaliq=quam#deriti", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/tvolu/imve.txt" + ], + "url.query": "gnaaliq=quam", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "mSecti", @@ -6265,20 +6881,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "www5.example.com" + "www5.example.com", + "api.example.net" ], "related.ip": [ - "10.82.148.126", - "10.60.56.205" + "10.60.56.205", + "10.82.148.126" ], "related.user": [ "ita" ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "block", - "NONE" + "NONE", + "block" ], "rsa.misc.content_type": "nder", "rsa.misc.result_code": "nihilmol", @@ -6303,9 +6919,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "tquii", "url.original": "https://www5.example.com/ugitsed/dminimve.htm?onse=uiac#tquii", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/ugitsed/dminimve.htm" + ], + "url.query": "onse=uiac", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "ita", @@ -6336,20 +6959,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.com" + "api.example.com", + "www5.example.net" ], "related.ip": [ - "10.245.251.98", - "10.6.11.124" + "10.6.11.124", + "10.245.251.98" ], "related.user": [ "tvolu" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "accept" + "accept", + "DELETE" ], "rsa.misc.content_type": "onsequ", "rsa.misc.result_code": "strud", @@ -6374,9 +6997,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "usan", "url.original": "https://api.example.com/ntium/ide.htm?tamrema=isautem#usan", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/ntium/ide.htm" + ], + "url.query": "tamrema=isautem", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "tvolu", @@ -6411,8 +7041,8 @@ "www5.example.org" ], "related.ip": [ - "10.99.55.115", - "10.145.25.55" + "10.145.25.55", + "10.99.55.115" ], "related.user": [ "lumd" @@ -6421,8 +7051,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "iamquisn", "rsa.misc.result_code": "lorem", @@ -6447,9 +7077,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "utaliqui", "url.original": "https://www5.example.org/iad/ngelits.jpg?mporin=orissusc#utaliqui", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/iad/ngelits.jpg" + ], + "url.query": "mporin=orissusc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "lumd", @@ -6484,8 +7121,8 @@ "internal.example.com" ], "related.ip": [ - "10.187.86.64", - "10.6.88.105" + "10.6.88.105", + "10.187.86.64" ], "related.user": [ "rem" @@ -6518,9 +7155,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "eprehe", "url.original": "https://mail.example.com/iatnulap/roi.htm?uine=loreeu#eprehe", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/iatnulap/roi.htm" + ], + "url.query": "uine=loreeu", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "rem", @@ -6551,12 +7195,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "mail.example.org" + "mail.example.org", + "example.com" ], "related.ip": [ - "10.252.146.132", - "10.163.9.35" + "10.163.9.35", + "10.252.146.132" ], "related.user": [ "umq" @@ -6591,9 +7235,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "illoin", "url.original": "https://mail.example.org/turv/use.jpg?mtot=macc#illoin", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/turv/use.jpg" + ], + "url.query": "mtot=macc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "umq", @@ -6624,20 +7275,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.org" + "api.example.org", + "mail.example.com" ], "related.ip": [ - "10.235.160.245", - "10.249.101.177" + "10.249.101.177", + "10.235.160.245" ], "related.user": [ "upta" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "deny", - "DELETE" + "DELETE", + "deny" ], "rsa.misc.content_type": "uameiu", "rsa.misc.result_code": "porinc", @@ -6662,9 +7313,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "ineavol", "url.original": "https://mail.example.com/umdol/rerepr.txt?emipsumq=orinr#ineavol", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/umdol/rerepr.txt" + ], + "url.query": "emipsumq=orinr", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "upta", @@ -6699,16 +7357,16 @@ "mail.example.org" ], "related.ip": [ - "10.73.218.58", - "10.140.170.171" + "10.140.170.171", + "10.73.218.58" ], "related.user": [ "tinv" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "block", - "TRACE" + "TRACE", + "block" ], "rsa.misc.content_type": "umq", "rsa.misc.result_code": "nse", @@ -6733,9 +7391,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "nevolup", "url.original": "https://internal.example.com/rautod/onorumet.htm?mvo=agnidol#nevolup", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/rautod/onorumet.htm" + ], + "url.query": "mvo=agnidol", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "tinv", @@ -6803,9 +7468,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "asiar", "url.original": "https://example.net/dun/xce.jpg?nsequat=mvol#asiar", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/dun/xce.jpg" + ], + "url.query": "nsequat=mvol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user.name": "squamest", "user_agent.device.name": "Generic Smartphone", @@ -6837,8 +7509,8 @@ "www.example.org" ], "related.ip": [ - "10.83.154.75", - "10.37.33.179" + "10.37.33.179", + "10.83.154.75" ], "related.user": [ "eatae" @@ -6871,9 +7543,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "sequu", "url.original": "https://api.example.com/teiru/mquamei.jpg?pta=uradi#sequu", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/teiru/mquamei.jpg" + ], + "url.query": "pta=uradi", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "eatae", @@ -6916,8 +7595,8 @@ ], "rsa.internal.messageid": "MKOL", "rsa.misc.action": [ - "deny", - "MKOL" + "MKOL", + "deny" ], "rsa.misc.content_type": "itse", "rsa.misc.result_code": "qui", @@ -6942,9 +7621,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "umdolo", "url.original": "https://www5.example.net/dtempor/rroquisq.gif?liquid=uidex#umdolo", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/dtempor/rroquisq.gif" + ], + "url.query": "liquid=uidex", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "usmod", @@ -6975,8 +7661,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "api.example.com" + "api.example.com", + "example.com" ], "related.ip": [ "10.221.86.133", @@ -6989,8 +7675,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "POST" + "POST", + "deny" ], "rsa.misc.content_type": "rerepr", "rsa.misc.result_code": "mcorpor", @@ -7015,9 +7701,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "rchite", "url.original": "https://api.example.com/ore/adeser.htm?pre=aute#rchite", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/ore/adeser.htm" + ], + "url.query": "pre=aute", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "ptasnul", @@ -7052,16 +7745,16 @@ "api.example.com" ], "related.ip": [ - "10.195.4.70", - "10.229.39.190" + "10.229.39.190", + "10.195.4.70" ], "related.user": [ "edictas" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "deny" + "deny", + "PUT" ], "rsa.misc.content_type": "exeaco", "rsa.misc.result_code": "rmagnido", @@ -7086,9 +7779,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "aer", "url.original": "https://api.example.com/liqu/dolor.htm?ess=umdo#aer", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/liqu/dolor.htm" + ], + "url.query": "ess=umdo", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "edictas", diff --git a/x-pack/filebeat/module/suricata/eve/ingest/http.yml b/x-pack/filebeat/module/suricata/eve/ingest/http.yml new file mode 100644 index 000000000000..de8c1710ed75 --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/http.yml @@ -0,0 +1,83 @@ +--- +description: Pipeline for Suricata DNS Events + +processors: + - set: + value: "{{suricata.eve.http.http_method}}" + field: http.request.method + if: "ctx?.suricata?.eve?.http?.http_method != null" + - rename: + field: suricata.eve.http.status + target_field: http.response.status_code + ignore_missing: true + - rename: + field: suricata.eve.http.hostname + target_field: destination.domain + ignore_missing: true + # - grok: + # field: suricata.eve.http.url + # patterns: + # - '%{PATH:url.path}(?:\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?' + # ignore_missing: true + # pattern_definitions: + # PATH: '[^?#]*' + # QUERY: '[^#]*' + # ANY: '.*' + - rename: + field: suricata.eve.http.url + target_field: url.original + ignore_missing: true + - rename: + field: suricata.eve.http.http_port + target_field: url.port + ignore_missing: true + + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - set: + field: url.domain + value: '{{destination.domain}}' + ignore_empty_value: true + if: ctx?.url?.domain == null && ctx?.destination?.domain != null + - remove: + field: _temp_ + ignore_missing: true + - rename: + field: suricata.eve.http.http_refer + target_field: http.request.referrer + ignore_missing: true + - rename: + field: suricata.eve.http.length + target_field: http.response.body.bytes + ignore_missing: true + - user_agent: + field: suricata.eve.http.http_user_agent + ignore_missing: true + +on_failure: + - append: + field: error.message + value: >- + error in HTTP pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index a24a5df3ff67..e957d177f38f 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -152,6 +152,11 @@ processors: - pipeline: if: ctx?.network?.protocol == "tls" name: '{< IngestPipeline "tls" >}' + ## HTTP + - pipeline: + if: >- + ctx?.network?.protocol == "http" + name: '{< IngestPipeline "http" >}' ## Flow - append: if: ctx?.suricata?.eve?.flow?.state == "new" @@ -163,24 +168,6 @@ processors: field: event.type value: - end - - - - set: - value: "{{suricata.eve.http.http_method}}" - field: http.request.method - if: "ctx?.suricata?.eve?.http?.http_method != null" - - rename: - field: suricata.eve.http.status - target_field: http.response.status_code - ignore_missing: true - - append: - if: ctx.suricata?.eve?.http?.hostname != null - value: '{{suricata.eve.http.hostname}}' - field: destination.domain - allow_duplicates: false - - remove: - field: suricata.eve.http.hostname - ignore_failure: true - script: lang: painless tag: suricata_deduplicate_dest_domain @@ -194,36 +181,6 @@ processors: ctx.destination.domain = domain; } ignore_failure: true - - set: - if: ctx?.network?.protocol == 'http' - field: url.domain - value: '{{destination.domain}}' - ignore_empty_value: true - - grok: - field: suricata.eve.http.url - patterns: - - '%{PATH:url.path}(?:\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?' - ignore_missing: true - pattern_definitions: - PATH: '[^?#]*' - QUERY: '[^#]*' - ANY: '.*' - - rename: - field: suricata.eve.http.url - target_field: url.original - ignore_missing: true - - rename: - field: suricata.eve.http.http_port - target_field: url.port - ignore_missing: true - - rename: - field: suricata.eve.http.http_refer - target_field: http.request.referrer - ignore_missing: true - - rename: - field: suricata.eve.http.length - target_field: http.response.body.bytes - ignore_missing: true - rename: field: suricata.eve.fileinfo.filename target_field: file.path @@ -342,9 +299,6 @@ processors: field: suricata.eve.proto target_field: network.transport ignore_missing: true - - user_agent: - field: suricata.eve.http.http_user_agent - ignore_missing: true - geoip: if: ctx?.source?.geo == null field: source.ip diff --git a/x-pack/filebeat/module/suricata/eve/manifest.yml b/x-pack/filebeat/module/suricata/eve/manifest.yml index 8d7f87d83735..a712ab663364 100644 --- a/x-pack/filebeat/module/suricata/eve/manifest.yml +++ b/x-pack/filebeat/module/suricata/eve/manifest.yml @@ -19,6 +19,7 @@ ingest_pipeline: - ingest/dns-answer-v1.yml - ingest/dns-answer-v2.yml - ingest/tls.yml + - ingest/http.yml input: config/eve.yml requires.processors: diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json index e8f77f9033a7..b294e0cbdca8 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json @@ -78,6 +78,7 @@ "suricata" ], "url.domain": "testmynids.org", + "url.extension": "html", "url.original": "/uid/index.html", "url.path": "/uid/index.html", "user_agent.device.name": "Other", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log b/x-pack/filebeat/module/suricata/eve/test/eve-small.log index 9cc157a9e757..28104b077ecc 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log @@ -1,7 +1,7 @@ {"timestamp":"2018-07-05T15:01:09.820360-0400","flow_id":298824096901438,"in_iface":"en0","event_type":"ssh","src_ip":"192.168.86.85","src_port":55406,"dest_ip":"192.168.253.112","dest_port":22,"proto":"TCP","ssh":{"client":{"proto_version":"2.0","software_version":"OpenSSH_7.6"},"server":{"proto_version":"2.0","software_version":"libssh_0.7.0"}}} {"timestamp":"2018-07-05T15:07:20.910626-0400","flow_id":904992230150281,"in_iface":"en0","event_type":"alert","src_ip":"192.168.86.85","src_port":55641,"dest_ip":"192.168.156.70","dest_port":443,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2024833,"rev":3,"signature":"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)","category":"Potential Corporate Privacy Violation","severity":1},"tls":{"session_resumed":true,"sni":"l2.io","version":"TLS 1.2"},"app_proto":"tls","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":793,"bytes_toclient":343,"start":"2018-07-05T15:07:19.659593-0400"}} {"timestamp":"2018-07-05T15:43:47.690014-0400","flow_id":2115002772430095,"in_iface":"en0","event_type":"http","src_ip":"192.168.86.85","src_port":56119,"dest_ip":"192.168.86.28","dest_port":63963,"proto":"TCP","tx_id":0,"http":{"hostname":"192.168.86.28","url":"\/dd.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"text\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1155}} -{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.28","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} +{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.85","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} {"timestamp":"2018-07-05T15:51:20.213418-0400","flow_id":1684780223079543,"in_iface":"en0","event_type":"dns","src_ip":"192.168.86.1","src_port":53,"dest_ip":"192.168.86.85","dest_port":39464,"proto":"UDP","dns":{"type":"answer","id":12308,"rcode":"NOERROR","rrname":"clients.l.google.com","rrtype":"A","ttl":299,"rdata":"172.217.13.110"}} {"timestamp":"2018-07-05T15:51:23.009510-0400","event_type":"stats","stats":{"uptime":5400,"capture":{"kernel_packets":430313,"kernel_drops":0,"kernel_ifdrops":0},"decoder":{"pkts":430313,"bytes":335138381,"invalid":2,"ipv4":425873,"ipv6":3785,"ethernet":430313,"raw":0,"null":0,"sll":0,"tcp":370093,"udp":58337,"sctp":0,"icmpv4":186,"icmpv6":1019,"ppp":0,"pppoe":0,"gre":0,"vlan":0,"vlan_qinq":0,"ieee8021ah":0,"teredo":1,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":778,"max_pkt_size":1514,"erspan":0,"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"dce":{"pkt_too_small":0}},"flow":{"memcap":0,"tcp":1113,"udp":1881,"icmpv4":0,"icmpv6":677,"spare":10000,"emerg_mode_entered":0,"emerg_mode_over":0,"tcp_reuse":0,"memuse":11537312},"defrag":{"ipv4":{"fragments":0,"reassembled":0,"timeouts":0},"ipv6":{"fragments":0,"reassembled":0,"timeouts":0},"max_frag_hits":0},"tcp":{"sessions":842,"ssn_memcap_drop":0,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"no_flow":0,"syn":1138,"synack":656,"rst":1165,"segment_memcap_drop":0,"stream_depth_reached":63,"reassembly_gap":0,"overlap":5979,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"insert_list_fail":0,"memuse":4587520,"reassembly_memuse":768000},"detect":{"alert":2},"app_layer":{"flow":{"http":22,"ftp":0,"smtp":0,"tls":560,"ssh":4,"imap":0,"msn":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"failed_tcp":2,"dcerpc_udp":0,"dns_udp":762,"failed_udp":1119},"tx":{"http":25,"ftp":0,"smtp":0,"tls":0,"ssh":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"dcerpc_udp":0,"dns_udp":762}},"flow_mgr":{"closed_pruned":729,"new_pruned":1879,"est_pruned":975,"bypassed_pruned":0,"flows_checked":8,"flows_notimeout":8,"flows_timeout":0,"flows_timeout_inuse":0,"flows_removed":0,"rows_checked":65536,"rows_skipped":65530,"rows_empty":0,"rows_busy":0,"rows_maxlen":2},"file_store":{"open_files":0},"dns":{"memuse":7749,"memcap_state":0,"memcap_global":0},"http":{"memuse":17861,"memcap":0}}} {"timestamp":"2018-07-05T15:51:50.666597-0400","flow_id":89751777876473,"in_iface":"en0","event_type":"tls","src_ip":"192.168.86.85","src_port":56187,"dest_ip":"17.142.164.13","dest_port":443,"proto":"TCP","tls":{"subject":"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US","issuerdn":"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US","serial":"5C:9C:E1:09:78:87:F8:07","fingerprint":"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47","sni":"p33-btmmdns.icloud.com.","version":"TLS 1.2","notbefore":"2017-02-27T17:54:31","notafter":"2019-03-29T17:54:31"}} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 50125bc3f3c2..cfd6fa5ff832 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -151,6 +151,7 @@ "suricata" ], "url.domain": "192.168.86.28", + "url.extension": "xml", "url.original": "/dd.xml", "url.path": "/dd.xml", "user_agent.device.name": "Mac", @@ -164,7 +165,7 @@ { "@timestamp": "2018-07-05T19:44:33.222Z", "destination.address": "192.168.86.85", - "destination.domain": "192.168.86.28", + "destination.domain": "192.168.86.85", "destination.ip": "192.168.86.85", "destination.port": 56118, "event.category": [ @@ -173,7 +174,7 @@ "event.dataset": "suricata.eve", "event.kind": "event", "event.module": "suricata", - "event.original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.28\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}", + "event.original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.85\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}", "file.path": "/ssdp/device-desc.xml", "file.size": 1071, "fileset.name": "eve", @@ -186,7 +187,7 @@ "network.protocol": "http", "network.transport": "tcp", "related.hosts": [ - "192.168.86.28" + "192.168.86.85" ], "related.ip": [ "192.168.86.28", @@ -211,7 +212,8 @@ "tags": [ "suricata" ], - "url.domain": "192.168.86.28", + "url.domain": "192.168.86.85", + "url.extension": "xml", "url.original": "/ssdp/device-desc.xml", "url.path": "/ssdp/device-desc.xml", "user_agent.device.name": "Mac", @@ -582,9 +584,11 @@ "suricata" ], "url.domain": "ctldl.windowsupdate.com", + "url.extension": "cab", "url.original": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111", - "url.path": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", + "url.path": "/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", "url.query": "111111111111", + "url.scheme": "http", "user_agent.device.name": "Other", "user_agent.name": "Microsoft-CryptoAPI", "user_agent.original": "Microsoft-CryptoAPI/10.0", diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index d8408c511333..045d5afe760b 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -20,6 +20,11 @@ processors: - remove: field: zeek.sip.seq ignore_missing: true +- grok: + field: url.full + patterns: + - '%{WORD:url.scheme}:(%{USERNAME:url.username}@)?%{IPORHOST:url.domain}(:%{POSINT:url.port:long})?' + ignore_missing: true - geoip: field: destination.ip target_field: destination.geo @@ -80,6 +85,30 @@ processors: field: event.outcome value: success if: "ctx?.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code < 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json index 71061cd293bc..a9e15731ebc3 100644 --- a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json @@ -42,7 +42,10 @@ "tags": [ "zeek.sip" ], + "url.domain": "newyork.voip.ms", "url.full": "sip:newyork.voip.ms:5060", + "url.port": 5060, + "url.scheme": "sip", "zeek.session_id": "CPRLCB4eWHdjP852Bk", "zeek.sip.call_id": "8694cd7e-976e4fc3-d76f6e38@172.16.133.19", "zeek.sip.request.body_length": 0, @@ -120,7 +123,11 @@ "tags": [ "zeek.sip" ], + "url.domain": "bestel.com", "url.full": "sip:francisco@bestel.com:55060", + "url.port": 55060, + "url.scheme": "sip", + "url.username": "francisco", "zeek.session_id": "ComJz236lSOcuOmix3", "zeek.sip.call_id": "12013223@200.57.7.195", "zeek.sip.request.body_length": 229, @@ -201,7 +208,9 @@ "tags": [ "zeek.sip" ], + "url.domain": "Verso.com", "url.full": "sip:Verso.com", + "url.scheme": "sip", "zeek.session_id": "CJZDWgixtwqXctWEg", "zeek.sip.call_id": "46E1C3CB36304F84A020CF6DD3F96461@Verso.com", "zeek.sip.request.body_length": 0, diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml index 9291add35933..62f05fa073f2 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml @@ -38,14 +38,22 @@ processors: ignore_missing: true - rename: field: zoom.meeting.join_url - target_field: url.full + target_field: url.original ignore_missing: true - rename: field: zoom.registrant.join_url - target_field: url.full + target_field: url.original ignore_missing: true - if: ctx?.url?.full == null - + if: ctx?.original?.full == null +- uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null +- set: + field: url.full + copy_from: url.original + ignore_failure: true + if: ctx?.url?.original != null # # Set user.* from participant, if any. # diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml b/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml index b836cd9c96c4..e5fca522f334 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml @@ -19,6 +19,10 @@ processors: field: zoom.object target_field: zoom.phone ignore_missing: true +- uri_parts: + field: zoom.phone.download_url + ignore_failure: true + if: ctx?.zoom?.phone?.download_url != null - rename: field: zoom.phone.download_url target_field: url.full diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml b/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml index 715f46bcbd93..f9e5b1c30422 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml @@ -35,6 +35,10 @@ processors: field: zoom.object target_field: zoom.recording ignore_missing: true +- uri_parts: + field: zoom.recording.share_url + ignore_failure: true + if: ctx?.zoom?.recording?.share_url != null - rename: field: zoom.recording.share_url target_field: url.full diff --git a/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json index 123de911c51b..723a37a326fd 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json @@ -99,7 +99,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/j/00000000", + "url.original": "https://zoom.us/j/00000000", + "url.path": "/j/00000000", + "url.scheme": "https", "user.email": "someemail@email.com", "user.id": "BBBBBBBBBB", "zoom.account_id": "AAAAAAAAAAA", @@ -254,7 +258,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/w/someendpointhere", + "url.original": "https://zoom.us/w/someendpointhere", + "url.path": "/w/someendpointhere", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.meeting.host_id": "uLobbbbbbbbbb_qQsQ", diff --git a/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json index 507943735b29..ee24b279b3c1 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json @@ -320,7 +320,12 @@ "zoom-webhook", "forwarded" ], + "url.domain": "testurl.com", + "url.extension": "mp4", "url.full": "https://testurl.com/file.mp4", + "url.original": "https://testurl.com/file.mp4", + "url.path": "/file.mp4", + "url.scheme": "https", "zoom.account_id": "test", "zoom.phone.callee.extension_type": "2", "zoom.phone.callee.id": "1234", diff --git a/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json index f9be7349ab4f..a0296121e1a8 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json @@ -175,7 +175,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -254,7 +258,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -293,7 +301,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -332,7 +344,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -371,7 +387,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index b6105a0fddd8..abf179e94b35 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -9,7 +9,31 @@ processors: # User agent - user_agent: field: user_agent.original - ignore_missing: true + ignore_missing: + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json index 54841870df77..5c4bd9fc36cb 100644 --- a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json +++ b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json @@ -26,8 +26,8 @@ "rci737.www5.example" ], "related.ip": [ - "10.206.191.17", - "10.176.10.114" + "10.176.10.114", + "10.206.191.17" ], "related.user": [ "sumdo" @@ -41,8 +41,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntium", "rsa.misc.action": [ - "Blocked", - "pisciv" + "pisciv", + "Blocked" ], "rsa.misc.category": "umq", "rsa.misc.filter": "oremi", @@ -65,7 +65,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "amremap", "url.original": "https://api.example.com/ivelitse/ritin.htm?utl=vol#amremap", + "url.path": "/ivelitse/ritin.htm", + "url.query": "utl=vol", + "url.scheme": "https", "user.name": "sumdo", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -139,7 +145,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "mquia", "url.original": "https://internal.example.net/isiutal/moenimi.jpg?gnaali=enatus#mquia", + "url.path": "/isiutal/moenimi.jpg", + "url.query": "gnaali=enatus", + "url.scheme": "https", "user.name": "eataevi", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -215,7 +227,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "iquaUte", "url.original": "https://example.com/taspe/mvolu.gif?atcup=snos#iquaUte", + "url.path": "/taspe/mvolu.gif", + "url.query": "atcup=snos", + "url.scheme": "https", "user.name": "tenima", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -291,7 +309,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "uid", "url.original": "https://api.example.org/doloreeu/pori.jpg?itati=mfu#uid", + "url.path": "/doloreeu/pori.jpg", + "url.query": "itati=mfu", + "url.scheme": "https", "user.name": "equun", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -328,8 +352,8 @@ "ore2933.www.test" ], "related.ip": [ - "10.136.153.149", - "10.61.78.108" + "10.61.78.108", + "10.136.153.149" ], "related.user": [ "ercit" @@ -367,7 +391,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iinea", "url.original": "https://api.example.com/ele/tenbyCic.gif?porainc=amquisno#iinea", + "url.path": "/ele/tenbyCic.gif", + "url.query": "porainc=amquisno", + "url.scheme": "https", "user.name": "ercit", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -404,8 +434,8 @@ "ollit4105.mail.localdomain" ], "related.ip": [ - "10.183.16.166", - "10.66.250.92" + "10.66.250.92", + "10.183.16.166" ], "related.user": [ "tessec" @@ -419,8 +449,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "avol", "rsa.misc.action": [ - "Allowed", - "ist" + "ist", + "Allowed" ], "rsa.misc.category": "lorema", "rsa.misc.filter": "sun", @@ -443,7 +473,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "aboreetd", "url.original": "https://mail.example.org/sitas/ehenderi.jpg?atquovo=iumto#aboreetd", + "url.path": "/sitas/ehenderi.jpg", + "url.query": "atquovo=iumto", + "url.scheme": "https", "user.name": "tessec", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -519,7 +555,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "ctetura", "url.original": "https://mail.example.net/aborumSe/luptat.txt?antiumto=strude#ctetura", + "url.path": "/aborumSe/luptat.txt", + "url.query": "antiumto=strude", + "url.scheme": "https", "user.name": "xercitat", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -595,7 +637,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "jpg", + "url.fragment": "des", "url.original": "https://www5.example.net/ntutla/equa.jpg?civeli=errorsi#des", + "url.path": "/ntutla/equa.jpg", + "url.query": "civeli=errorsi", + "url.scheme": "https", "user.name": "erc", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -632,8 +680,8 @@ "aperia4409.www5.invalid" ], "related.ip": [ - "10.78.151.178", - "10.25.192.202" + "10.25.192.202", + "10.78.151.178" ], "related.user": [ "quip" @@ -671,7 +719,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "oriosamn", "url.original": "https://api.example.net/atvol/umiur.txt?tati=utaliqu#oriosamn", + "url.path": "/atvol/umiur.txt", + "url.query": "tati=utaliqu", + "url.scheme": "https", "user.name": "quip", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -708,8 +762,8 @@ "sitvolup368.internal.host" ], "related.ip": [ - "10.71.170.37", - "10.135.225.244" + "10.135.225.244", + "10.71.170.37" ], "related.user": [ "atu" @@ -723,8 +777,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ihilm", "rsa.misc.action": [ - "Allowed", - "psaquae" + "psaquae", + "Allowed" ], "rsa.misc.category": "eFinib", "rsa.misc.filter": "inesci", @@ -747,7 +801,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "uipe", "url.original": "https://mail.example.net/equep/iavolu.gif?aqu=rpo#uipe", + "url.path": "/equep/iavolu.gif", + "url.query": "aqu=rpo", + "url.scheme": "https", "user.name": "atu", "user_agent.device.name": "POCOPHONE F1", "user_agent.name": "Chrome Mobile", @@ -784,8 +844,8 @@ "ite2026.www.invalid" ], "related.ip": [ - "10.223.247.86", - "10.19.145.131" + "10.19.145.131", + "10.223.247.86" ], "related.user": [ "tNequepo" @@ -823,7 +883,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "namali", "url.original": "https://example.org/bor/occa.htm?dol=leumiu#namali", + "url.path": "/bor/occa.htm", + "url.query": "dol=leumiu", + "url.scheme": "https", "user.name": "tNequepo", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -899,7 +965,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "reseos", "url.original": "https://internal.example.net/oru/temqu.htm?etMalor=ipi#reseos", + "url.path": "/oru/temqu.htm", + "url.query": "etMalor=ipi", + "url.scheme": "https", "user.name": "ihilmo", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -951,8 +1023,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "catc", "rsa.misc.action": [ - "Allowed", - "veni" + "veni", + "Allowed" ], "rsa.misc.category": "sBono", "rsa.misc.filter": "isnisiu", @@ -975,7 +1047,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "osamn", "url.original": "https://mail.example.net/itatione/isnis.html?oluptate=issus#osamn", + "url.path": "/itatione/isnis.html", + "url.query": "oluptate=issus", + "url.scheme": "https", "user.name": "ratvolu", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1012,8 +1090,8 @@ "piscin6866.internal.host" ], "related.ip": [ - "10.135.160.125", - "10.0.55.9" + "10.0.55.9", + "10.135.160.125" ], "related.user": [ "volupta" @@ -1051,7 +1129,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "rau", "url.original": "https://www.example.org/eporr/xeacomm.html?aturQui=utlabor#rau", + "url.path": "/eporr/xeacomm.html", + "url.query": "aturQui=utlabor", + "url.scheme": "https", "user.name": "volupta", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -1103,8 +1187,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "nnum", "rsa.misc.action": [ - "Allowed", - "ntoccae" + "ntoccae", + "Allowed" ], "rsa.misc.category": "tium", "rsa.misc.filter": "uteirure", @@ -1127,7 +1211,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "ntocc", "url.original": "https://internal.example.net/ptatemq/luptatev.html?Nequepo=ipsumd#ntocc", + "url.path": "/ptatemq/luptatev.html", + "url.query": "Nequepo=ipsumd", + "url.scheme": "https", "user.name": "saute", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1203,7 +1293,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "ipsumd", "url.original": "https://www5.example.com/tateve/itinvol.txt?tenatus=cipitlab#ipsumd", + "url.path": "/tateve/itinvol.txt", + "url.query": "tenatus=cipitlab", + "url.scheme": "https", "user.name": "inibusB", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1279,7 +1375,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "taedicta", "url.original": "https://api.example.net/tquiin/tse.jpg?ovol=ptasn#taedicta", + "url.path": "/tquiin/tse.jpg", + "url.query": "ovol=ptasn", + "url.scheme": "https", "user.name": "exercita", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -1355,7 +1457,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "amcol", "url.original": "https://mail.example.org/olor/ineavo.gif?mquelau=iadolor#amcol", + "url.path": "/olor/ineavo.gif", + "url.query": "mquelau=iadolor", + "url.scheme": "https", "user.name": "str", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -1407,8 +1515,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "oNemoeni", "rsa.misc.action": [ - "Blocked", - "nre" + "nre", + "Blocked" ], "rsa.misc.category": "labo", "rsa.misc.filter": "tutlab", @@ -1431,7 +1539,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "seq", "url.original": "https://www.example.com/its/ender.gif?oles=edic#seq", + "url.path": "/its/ender.gif", + "url.query": "oles=edic", + "url.scheme": "https", "user.name": "aturve", "user_agent.device.name": "Samsung SM-S337TL", "user_agent.name": "Chrome Mobile", @@ -1468,8 +1582,8 @@ "eacommod1930.internal.lan" ], "related.ip": [ - "10.229.83.165", - "10.29.155.171" + "10.29.155.171", + "10.229.83.165" ], "related.user": [ "ulapar" @@ -1507,7 +1621,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "psum", "url.original": "https://www5.example.org/oeni/tdol.gif?llamco=nea#psum", + "url.path": "/oeni/tdol.gif", + "url.query": "llamco=nea", + "url.scheme": "https", "user.name": "ulapar", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -1559,8 +1679,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uaUten", "rsa.misc.action": [ - "amcorp", - "Blocked" + "Blocked", + "amcorp" ], "rsa.misc.category": "umdolor", "rsa.misc.filter": "velillu", @@ -1583,7 +1703,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "lor", "url.original": "https://www.example.com/uasiar/utlab.htm?loremqu=dantium#lor", + "url.path": "/uasiar/utlab.htm", + "url.query": "loremqu=dantium", + "url.scheme": "https", "user.name": "lor", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -1635,8 +1761,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tdol", "rsa.misc.action": [ - "Allowed", - "nte" + "nte", + "Allowed" ], "rsa.misc.category": "adeseru", "rsa.misc.filter": "mac", @@ -1659,7 +1785,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "quamest", "url.original": "https://api.example.org/icabo/gna.html?urerepr=eseru#quamest", + "url.path": "/icabo/gna.html", + "url.query": "urerepr=eseru", + "url.scheme": "https", "user.name": "snost", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -1735,7 +1867,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "onsequ", "url.original": "https://api.example.org/oremi/elites.html?iosa=boNemoe#onsequ", + "url.path": "/oremi/elites.html", + "url.query": "iosa=boNemoe", + "url.scheme": "https", "user.name": "olori", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1787,8 +1925,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ulpa", "rsa.misc.action": [ - "Allowed", - "gnaal" + "gnaal", + "Allowed" ], "rsa.misc.category": "nte", "rsa.misc.filter": "pid", @@ -1811,7 +1949,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "dqu", "url.original": "https://example.com/luptatem/uaeratv.gif?dat=periam#dqu", + "url.path": "/luptatem/uaeratv.gif", + "url.query": "dat=periam", + "url.scheme": "https", "user.name": "fugi", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -1848,8 +1992,8 @@ "sitam5077.internal.host" ], "related.ip": [ - "10.179.210.218", - "10.32.39.220" + "10.32.39.220", + "10.179.210.218" ], "related.user": [ "boreetdo" @@ -1887,7 +2031,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "umetMalo", "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo", + "url.path": "/tanimi/rumSecti.jpg", + "url.query": "emporain=ntiumto", + "url.scheme": "https", "user.name": "boreetdo", "user_agent.device.name": "Samsung SM-A715F", "user_agent.name": "Facebook", @@ -1939,8 +2089,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntNeq", "rsa.misc.action": [ - "dtempo", - "Blocked" + "Blocked", + "dtempo" ], "rsa.misc.category": "ipsu", "rsa.misc.filter": "iqu", @@ -1963,7 +2113,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "ulla", "url.original": "https://api.example.com/ori/tconsect.html?ercit=eporroq#ulla", + "url.path": "/ori/tconsect.html", + "url.query": "ercit=eporroq", + "url.scheme": "https", "user.name": "agnaaliq", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -2000,8 +2156,8 @@ "lloin4019.www.localhost" ], "related.ip": [ - "10.130.241.232", - "10.238.224.49" + "10.238.224.49", + "10.130.241.232" ], "related.user": [ "onse" @@ -2015,8 +2171,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mnisiut", "rsa.misc.action": [ - "mod", - "Allowed" + "Allowed", + "mod" ], "rsa.misc.category": "uiinea", "rsa.misc.filter": "aturQu", @@ -2039,7 +2195,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "utfug", "url.original": "https://api.example.org/rure/asiarchi.txt?loremeu=aturve#utfug", + "url.path": "/rure/asiarchi.txt", + "url.query": "loremeu=aturve", + "url.scheme": "https", "user.name": "onse", "user_agent.device.name": "POCOPHONE F1", "user_agent.name": "Chrome Mobile", @@ -2076,8 +2238,8 @@ "tamet6317.www.host" ], "related.ip": [ - "10.2.67.127", - "10.115.53.31" + "10.115.53.31", + "10.2.67.127" ], "related.user": [ "Cic" @@ -2091,8 +2253,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "quatD", "rsa.misc.action": [ - "Allowed", - "tatem" + "tatem", + "Allowed" ], "rsa.misc.category": "aincidun", "rsa.misc.filter": "uela", @@ -2115,7 +2277,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "htm", + "url.fragment": "labore", "url.original": "https://example.com/emUte/molestia.htm?orroqu=elitsed#labore", + "url.path": "/emUte/molestia.htm", + "url.query": "orroqu=elitsed", + "url.scheme": "https", "user.name": "Cic", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -2191,7 +2359,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "tionula", "url.original": "https://mail.example.net/repreh/plic.jpg?utlabo=tetur#tionula", + "url.path": "/repreh/plic.jpg", + "url.query": "utlabo=tetur", + "url.scheme": "https", "user.name": "ueipsa", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -2228,8 +2402,8 @@ "utaliqu4248.www.localhost" ], "related.ip": [ - "10.18.226.72", - "10.101.85.169" + "10.101.85.169", + "10.18.226.72" ], "related.user": [ "rroqu" @@ -2267,7 +2441,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ema", "url.original": "https://api.example.com/tcu/iatqu.jpg?quovo=urExcep#ema", + "url.path": "/tcu/iatqu.jpg", + "url.query": "quovo=urExcep", + "url.scheme": "https", "user.name": "rroqu", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -2304,8 +2484,8 @@ "mdolore473.internal.test" ], "related.ip": [ - "10.87.100.240", - "10.242.182.193" + "10.242.182.193", + "10.87.100.240" ], "related.user": [ "stenatus" @@ -2319,8 +2499,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mag", "rsa.misc.action": [ - "Allowed", - "tali" + "tali", + "Allowed" ], "rsa.misc.category": "oconse", "rsa.misc.filter": "npr", @@ -2343,7 +2523,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "lupta", "url.original": "https://www5.example.com/apariatu/lorsita.gif?msequ=uat#lupta", + "url.path": "/apariatu/lorsita.gif", + "url.query": "msequ=uat", + "url.scheme": "https", "user.name": "stenatus", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -2419,7 +2605,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "tesseq", "url.original": "https://internal.example.net/ende/abor.jpg?riameaqu=ame#tesseq", + "url.path": "/ende/abor.jpg", + "url.query": "riameaqu=ame", + "url.scheme": "https", "user.name": "itasp", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -2456,8 +2648,8 @@ "lapar1599.www.lan" ], "related.ip": [ - "10.106.77.138", - "10.193.66.155" + "10.193.66.155", + "10.106.77.138" ], "related.user": [ "iusmodt" @@ -2495,7 +2687,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "txt", + "url.fragment": "mvolupta", "url.original": "https://example.com/ame/amvolu.txt?equaturv=lamc#mvolupta", + "url.path": "/ame/amvolu.txt", + "url.query": "equaturv=lamc", + "url.scheme": "https", "user.name": "iusmodt", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -2532,8 +2730,8 @@ "aquioff3853.www.localdomain" ], "related.ip": [ - "10.236.230.136", - "10.54.159.1" + "10.54.159.1", + "10.236.230.136" ], "related.user": [ "mUteni" @@ -2571,7 +2769,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "nisi", "url.original": "https://mail.example.org/uisnostr/reetdol.txt?ugi=niamquis#nisi", + "url.path": "/uisnostr/reetdol.txt", + "url.query": "ugi=niamquis", + "url.scheme": "https", "user.name": "mUteni", "user_agent.device.name": "STK-L21", "user_agent.name": "Chrome Mobile", @@ -2608,8 +2812,8 @@ "ura675.mail.localdomain" ], "related.ip": [ - "10.49.242.174", - "10.131.246.134" + "10.131.246.134", + "10.49.242.174" ], "related.user": [ "umdolo" @@ -2623,8 +2827,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tvolup", "rsa.misc.action": [ - "utemvel", - "Allowed" + "Allowed", + "utemvel" ], "rsa.misc.category": "untutlab", "rsa.misc.filter": "dol", @@ -2647,7 +2851,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "stiaecon", "url.original": "https://api.example.com/radipis/cive.gif?orumSec=nisiuta#stiaecon", + "url.path": "/radipis/cive.gif", + "url.query": "orumSec=nisiuta", + "url.scheme": "https", "user.name": "umdolo", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -2699,8 +2909,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ende", "rsa.misc.action": [ - "Blocked", - "doconse" + "doconse", + "Blocked" ], "rsa.misc.category": "uovolupt", "rsa.misc.filter": "litesse", @@ -2723,7 +2933,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "eursinto", "url.original": "https://mail.example.org/oin/itseddoe.html?citati=uamei#eursinto", + "url.path": "/oin/itseddoe.html", + "url.query": "citati=uamei", + "url.scheme": "https", "user.name": "olori", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -2760,8 +2976,8 @@ "eaque6543.api.domain" ], "related.ip": [ - "10.138.188.201", - "10.128.184.241" + "10.128.184.241", + "10.138.188.201" ], "related.user": [ "etur" @@ -2775,8 +2991,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "issu", "rsa.misc.action": [ - "sed", - "Allowed" + "Allowed", + "sed" ], "rsa.misc.category": "atur", "rsa.misc.filter": "iciadese", @@ -2799,7 +3015,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "html", + "url.fragment": "umS", "url.original": "https://mail.example.com/eseruntm/lpaquiof.html?magnaal=uscip#umS", + "url.path": "/eseruntm/lpaquiof.html", + "url.query": "magnaal=uscip", + "url.scheme": "https", "user.name": "etur", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -2851,8 +3073,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ese", "rsa.misc.action": [ - "litanim", - "Allowed" + "Allowed", + "litanim" ], "rsa.misc.category": "idata", "rsa.misc.filter": "urerepre", @@ -2875,7 +3097,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.net", + "url.extension": "html", + "url.fragment": "ine", "url.original": "https://example.net/snulap/enimadm.html?writte=sitvo#ine", + "url.path": "/snulap/enimadm.html", + "url.query": "writte=sitvo", + "url.scheme": "https", "user.name": "isau", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -2951,7 +3179,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "ents", "url.original": "https://internal.example.org/etcon/onsequu.gif?Bonoru=madminim#ents", + "url.path": "/etcon/onsequu.gif", + "url.query": "Bonoru=madminim", + "url.scheme": "https", "user.name": "eiusmo", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -2988,8 +3222,8 @@ "pariatur7238.www5.invalid" ], "related.ip": [ - "10.202.224.79", - "10.33.144.10" + "10.33.144.10", + "10.202.224.79" ], "related.user": [ "rios" @@ -3027,7 +3261,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "seos", "url.original": "https://www.example.org/rur/itse.gif?pisciv=fugiatqu#seos", + "url.path": "/rur/itse.gif", + "url.query": "pisciv=fugiatqu", + "url.scheme": "https", "user.name": "rios", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -3079,8 +3319,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Loremip", "rsa.misc.action": [ - "Allowed", - "quid" + "quid", + "Allowed" ], "rsa.misc.category": "mini", "rsa.misc.filter": "uisnos", @@ -3103,7 +3343,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "liquipex", "url.original": "https://mail.example.com/qui/equeporr.jpg?itsedd=texpli#liquipex", + "url.path": "/qui/equeporr.jpg", + "url.query": "itsedd=texpli", + "url.scheme": "https", "user.name": "CSe", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -3140,8 +3386,8 @@ "mquisnos7453.home" ], "related.ip": [ - "10.134.128.27", - "10.118.177.136" + "10.118.177.136", + "10.134.128.27" ], "related.user": [ "Utenima" @@ -3179,7 +3425,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "eacommo", "url.original": "https://api.example.net/lup/iumtotam.html?ipitlabo=userror#eacommo", + "url.path": "/lup/iumtotam.html", + "url.query": "ipitlabo=userror", + "url.scheme": "https", "user.name": "Utenima", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -3231,8 +3483,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "amni", "rsa.misc.action": [ - "Allowed", - "edutp" + "edutp", + "Allowed" ], "rsa.misc.category": "ames", "rsa.misc.filter": "dmi", @@ -3255,7 +3507,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "umdolore", "url.original": "https://example.org/onproide/uamnih.htm?tatisetq=uidolo#umdolore", + "url.path": "/onproide/uamnih.htm", + "url.query": "tatisetq=uidolo", + "url.scheme": "https", "user.name": "reet", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -3331,7 +3589,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "emip", "url.original": "https://www5.example.org/obeataev/umf.htm?moll=quaeabil#emip", + "url.path": "/obeataev/umf.htm", + "url.query": "moll=quaeabil", + "url.scheme": "https", "user.name": "orissus", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -3368,8 +3632,8 @@ "etdolore4227.internal.corp" ], "related.ip": [ - "10.30.87.51", - "10.156.177.53" + "10.156.177.53", + "10.30.87.51" ], "related.user": [ "psaquaea" @@ -3383,8 +3647,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tatno", "rsa.misc.action": [ - "ptatev", - "Blocked" + "Blocked", + "ptatev" ], "rsa.misc.category": "udexerc", "rsa.misc.filter": "ptatemse", @@ -3407,7 +3671,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "leumiur", "url.original": "https://mail.example.org/consequa/eaqueip.gif?aevitaed=byCic#leumiur", + "url.path": "/consequa/eaqueip.gif", + "url.query": "aevitaed=byCic", + "url.scheme": "https", "user.name": "psaquaea", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -3459,8 +3729,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tatemse", "rsa.misc.action": [ - "Blocked", - "upta" + "upta", + "Blocked" ], "rsa.misc.category": "tlabo", "rsa.misc.filter": "aliqui", @@ -3483,7 +3753,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "uptasnul", "url.original": "https://example.org/tmo/onofdeF.txt?oremip=its#uptasnul", + "url.path": "/tmo/onofdeF.txt", + "url.query": "oremip=its", + "url.scheme": "https", "user.name": "dentsunt", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -3535,8 +3811,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "itesse", "rsa.misc.action": [ - "Allowed", - "uip" + "uip", + "Allowed" ], "rsa.misc.category": "teturad", "rsa.misc.filter": "roquisqu", @@ -3559,7 +3835,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "smodtem", "url.original": "https://mail.example.com/orsitvol/ntor.htm?itqu=minimav#smodtem", + "url.path": "/orsitvol/ntor.htm", + "url.query": "itqu=minimav", + "url.scheme": "https", "user.name": "taliq", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -3596,8 +3878,8 @@ "laboree3880.api.invalid" ], "related.ip": [ - "10.166.195.20", - "10.255.40.12" + "10.255.40.12", + "10.166.195.20" ], "related.user": [ "lamcolab" @@ -3611,8 +3893,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mipsumq", "rsa.misc.action": [ - "citation", - "Allowed" + "Allowed", + "citation" ], "rsa.misc.category": "usant", "rsa.misc.filter": "Nem", @@ -3635,7 +3917,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "gna", "url.original": "https://internal.example.org/rumexe/xerci.gif?olor=quiav#gna", + "url.path": "/rumexe/xerci.gif", + "url.query": "olor=quiav", + "url.scheme": "https", "user.name": "lamcolab", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -3685,8 +3973,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ento", "rsa.misc.action": [ - "Bonoru", - "Blocked" + "Blocked", + "Bonoru" ], "rsa.misc.category": "luptasnu", "rsa.misc.filter": "quamni", @@ -3709,7 +3997,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "equuntu", "url.original": "https://example.org/tvolu/dutper.html?nbyCicer=scipit#equuntu", + "url.path": "/tvolu/dutper.html", + "url.query": "nbyCicer=scipit", + "url.scheme": "https", "user.name": "ute", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -3746,8 +4040,8 @@ "ine3181.www.invalid" ], "related.ip": [ - "10.119.53.68", - "10.121.9.5" + "10.121.9.5", + "10.119.53.68" ], "related.user": [ "ssec" @@ -3761,8 +4055,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "dexea", "rsa.misc.action": [ - "tinvolup", - "Blocked" + "Blocked", + "tinvolup" ], "rsa.misc.category": "ende", "rsa.misc.filter": "onse", @@ -3785,7 +4079,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "nul", "url.original": "https://www.example.com/uiavo/uisaut.htm?paq=uianon#nul", + "url.path": "/uiavo/uisaut.htm", + "url.query": "paq=uianon", + "url.scheme": "https", "user.name": "ssec", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -3822,8 +4122,8 @@ "tsunt3403.www5.test" ], "related.ip": [ - "10.31.153.177", - "10.237.0.173" + "10.237.0.173", + "10.31.153.177" ], "related.user": [ "sci" @@ -3837,8 +4137,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "eritqui", "rsa.misc.action": [ - "Blocked", - "dolor" + "dolor", + "Blocked" ], "rsa.misc.category": "taspe", "rsa.misc.filter": "oremipsu", @@ -3861,7 +4161,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "upt", "url.original": "https://mail.example.com/uasiarch/Malor.jpg?iinea=snos#upt", + "url.path": "/uasiarch/Malor.jpg", + "url.query": "iinea=snos", + "url.scheme": "https", "user.name": "sci", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -3911,8 +4217,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "epor", "rsa.misc.action": [ - "etquasia", - "Allowed" + "Allowed", + "etquasia" ], "rsa.misc.category": "iaturE", "rsa.misc.filter": "rep", @@ -3935,7 +4241,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "com", "url.original": "https://api.example.org/ntiumt/sumquia.jpg?lam=asnu#com", + "url.path": "/ntiumt/sumquia.jpg", + "url.query": "lam=asnu", + "url.scheme": "https", "user.name": "duntut", "user_agent.device.name": "Other", "user_agent.name": "Other", @@ -3983,8 +4295,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "adipisc", "rsa.misc.action": [ - "exer", - "Blocked" + "Blocked", + "exer" ], "rsa.misc.category": "remagna", "rsa.misc.filter": "emvel", @@ -4007,7 +4319,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "todi", "url.original": "https://example.com/itsedqu/paq.jpg?hilmol=oluptate#todi", + "url.path": "/itsedqu/paq.jpg", + "url.query": "hilmol=oluptate", + "url.scheme": "https", "user.name": "picia", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -4044,8 +4362,8 @@ "tamr1693.api.home" ], "related.ip": [ - "10.53.191.49", - "10.133.102.57" + "10.133.102.57", + "10.53.191.49" ], "related.user": [ "onsec" @@ -4083,7 +4401,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "ctionofd", "url.original": "https://api.example.org/remeum/etur.html?Quisa=quiav#ctionofd", + "url.path": "/remeum/etur.html", + "url.query": "Quisa=quiav", + "url.scheme": "https", "user.name": "onsec", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -4159,7 +4483,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "nostrum", "url.original": "https://internal.example.org/ree/itten.gif?rsp=imipsa#nostrum", + "url.path": "/ree/itten.gif", + "url.query": "rsp=imipsa", + "url.scheme": "https", "user.name": "tem", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -4235,7 +4565,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "onemulla", "url.original": "https://www.example.net/ritquiin/reseo.jpg?ari=umtot#onemulla", + "url.path": "/ritquiin/reseo.jpg", + "url.query": "ari=umtot", + "url.scheme": "https", "user.name": "uasiarch", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -4272,8 +4608,8 @@ "pici1525.www5.corp" ], "related.ip": [ - "10.155.252.123", - "10.178.148.188" + "10.178.148.188", + "10.155.252.123" ], "related.user": [ "inrepreh" @@ -4311,7 +4647,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "eavolupt", "url.original": "https://mail.example.com/dexe/nemul.jpg?yCicero=inimave#eavolupt", + "url.path": "/dexe/nemul.jpg", + "url.query": "yCicero=inimave", + "url.scheme": "https", "user.name": "inrepreh", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -4363,8 +4705,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uamquaer", "rsa.misc.action": [ - "Blocked", - "aerat" + "aerat", + "Blocked" ], "rsa.misc.category": "quela", "rsa.misc.filter": "qui", @@ -4387,7 +4729,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "sistena", "url.original": "https://mail.example.org/caecat/uel.html?enim=umq#sistena", + "url.path": "/caecat/uel.html", + "url.query": "enim=umq", + "url.scheme": "https", "user.name": "olup", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -4422,8 +4770,8 @@ "imveni193.www5.host" ], "related.ip": [ - "10.112.190.154", - "10.55.38.153" + "10.55.38.153", + "10.112.190.154" ], "related.user": [ "oremeu" @@ -4437,8 +4785,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tin", "rsa.misc.action": [ - "urau", - "Allowed" + "Allowed", + "urau" ], "rsa.misc.category": "isiut", "rsa.misc.filter": "cons", @@ -4461,7 +4809,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "rna", "url.original": "https://mail.example.com/runtmoll/busBon.txt?ionev=vitaedi#rna", + "url.path": "/runtmoll/busBon.txt", + "url.query": "ionev=vitaedi", + "url.scheme": "https", "user.name": "oremeu", "user_agent.device.name": "XiaoMi Redmi 4X", "user_agent.name": "MiuiBrowser", @@ -4537,7 +4891,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "quam", "url.original": "https://api.example.com/lits/tvolu.jpg?squir=gnaaliq#quam", + "url.path": "/lits/tvolu.jpg", + "url.query": "squir=gnaaliq", + "url.scheme": "https", "user.name": "tsedquia", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -4574,8 +4934,8 @@ "remips1499.www.local" ], "related.ip": [ - "10.60.52.219", - "10.252.164.230" + "10.252.164.230", + "10.60.52.219" ], "related.user": [ "gnamali" @@ -4613,7 +4973,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "nder", "url.original": "https://mail.example.net/loremi/queporro.jpg?ade=nihilmol#nder", + "url.path": "/loremi/queporro.jpg", + "url.query": "ade=nihilmol", + "url.scheme": "https", "user.name": "gnamali", "user_agent.device.name": "Other", "user_agent.name": "Other", @@ -4646,8 +5012,8 @@ "mdoloree96.domain" ], "related.ip": [ - "10.187.16.73", - "10.122.102.156" + "10.122.102.156", + "10.187.16.73" ], "related.user": [ "emoen" @@ -4685,7 +5051,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iad", "url.original": "https://api.example.com/nge/psum.gif?exerci=isnostru#iad", + "url.path": "/nge/psum.gif", + "url.query": "exerci=isnostru", + "url.scheme": "https", "user.name": "emoen", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -4722,8 +5094,8 @@ "iatnulap7662.internal.local" ], "related.ip": [ - "10.120.215.174", - "10.248.108.55" + "10.248.108.55", + "10.120.215.174" ], "related.user": [ "prehend" @@ -4761,7 +5133,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "onorumet", "url.original": "https://internal.example.org/ddoeiusm/apa.txt?uptatemU=rem#onorumet", + "url.path": "/ddoeiusm/apa.txt", + "url.query": "uptatemU=rem", + "url.scheme": "https", "user.name": "prehend", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -4835,7 +5213,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "htm", + "url.fragment": "quam", "url.original": "https://www5.example.net/yCice/uinesci.htm?taevitae=dminimv#quam", + "url.path": "/yCice/uinesci.htm", + "url.query": "taevitae=dminimv", + "url.scheme": "https", "user.name": "abo", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", @@ -4911,7 +5295,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "txt", + "url.fragment": "nse", "url.original": "https://api.example.com/itinvolu/adeserun.txt?tinv=Utenima#nse", + "url.path": "/itinvolu/adeserun.txt", + "url.query": "tinv=Utenima", + "url.scheme": "https", "user.name": "equamn", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -4963,8 +5353,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "orinrep", "rsa.misc.action": [ - "Blocked", - "squirat" + "squirat", + "Blocked" ], "rsa.misc.category": "sequa", "rsa.misc.filter": "orainci", @@ -4987,7 +5377,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "loreeuf", "url.original": "https://www.example.org/sci/isquames.gif?tlabor=itecto#loreeuf", + "url.path": "/sci/isquames.gif", + "url.query": "tlabor=itecto", + "url.scheme": "https", "user.name": "evelite", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -5024,8 +5420,8 @@ "beataevi7552.api.test" ], "related.ip": [ - "10.215.63.248", - "10.138.0.214" + "10.138.0.214", + "10.215.63.248" ], "related.user": [ "eavolupt" @@ -5039,8 +5435,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "odita", "rsa.misc.action": [ - "Blocked", - "dqu" + "dqu", + "Blocked" ], "rsa.misc.category": "ipex", "rsa.misc.filter": "ine", @@ -5063,7 +5459,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "usmod", "url.original": "https://mail.example.org/umdolo/nimv.htm?equunt=tutla#usmod", + "url.path": "/umdolo/nimv.htm", + "url.query": "equunt=tutla", + "url.scheme": "https", "user.name": "eavolupt", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", @@ -5100,8 +5502,8 @@ "rvelill1981.www.invalid" ], "related.ip": [ - "10.26.115.88", - "10.12.130.224" + "10.12.130.224", + "10.26.115.88" ], "related.user": [ "Nequepo" @@ -5139,7 +5541,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "byCice", "url.original": "https://mail.example.net/tvol/ostru.htm?oei=iquipex#byCice", + "url.path": "/tvol/ostru.htm", + "url.query": "oei=iquipex", + "url.scheme": "https", "user.name": "Nequepo", "user_agent.device.name": "STK-L21", "user_agent.name": "Chrome Mobile", @@ -5176,8 +5584,8 @@ "quia7214.example" ], "related.ip": [ - "10.193.152.42", - "10.91.20.27" + "10.91.20.27", + "10.193.152.42" ], "related.user": [ "edict" @@ -5215,7 +5623,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "atemacc", "url.original": "https://mail.example.org/pariatur/cita.html?equuntur=rve#atemacc", + "url.path": "/pariatur/cita.html", + "url.query": "equuntur=rve", + "url.scheme": "https", "user.name": "edict", "user_agent.device.name": "QMobile X700 PRO II", "user_agent.name": "Chrome Mobile", @@ -5252,8 +5666,8 @@ "aturExc7343.invalid" ], "related.ip": [ - "10.146.69.38", - "10.55.192.102" + "10.55.192.102", + "10.146.69.38" ], "related.user": [ "quia" @@ -5291,7 +5705,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "elillumq", "url.original": "https://example.org/aturE/aaliqu.gif?nvol=doloreeu#elillumq", + "url.path": "/aturE/aaliqu.gif", + "url.query": "nvol=doloreeu", + "url.scheme": "https", "user.name": "quia", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -5367,7 +5787,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "dese", "url.original": "https://internal.example.org/olorin/orisnisi.gif?eritquii=atevelit#dese", + "url.path": "/olorin/orisnisi.gif", + "url.query": "eritquii=atevelit", + "url.scheme": "https", "user.name": "isciveli", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -5443,7 +5869,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "deo", "url.original": "https://example.org/vel/preh.html?sequamni=edutpers#deo", + "url.path": "/vel/preh.html", + "url.query": "sequamni=edutpers", + "url.scheme": "https", "user.name": "estla", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -5480,8 +5912,8 @@ "agna5654.www.corp" ], "related.ip": [ - "10.200.74.101", - "10.203.47.23" + "10.203.47.23", + "10.200.74.101" ], "related.user": [ "litesse" @@ -5495,8 +5927,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "nde", "rsa.misc.action": [ - "Allowed", - "iqu" + "iqu", + "Allowed" ], "rsa.misc.category": "ametco", "rsa.misc.filter": "ntincul", @@ -5519,7 +5951,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "aUtenim", "url.original": "https://example.com/nonproi/dolor.jpg?molli=oeiusm#aUtenim", + "url.path": "/nonproi/dolor.jpg", + "url.query": "molli=oeiusm", + "url.scheme": "https", "user.name": "litesse", "user_agent.device.name": "Samsung SM-A305FN", "user_agent.name": "YandexSearch", @@ -5571,8 +6009,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ereprehe", "rsa.misc.action": [ - "Blocked", - "tutl" + "tutl", + "Blocked" ], "rsa.misc.category": "mip", "rsa.misc.filter": "umSecti", @@ -5595,7 +6033,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "uepor", "url.original": "https://example.com/sedqui/iuntNe.gif?epteu=nvent#uepor", + "url.path": "/sedqui/iuntNe.gif", + "url.query": "epteu=nvent", + "url.scheme": "https", "user.name": "ntore", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -5647,8 +6091,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "diconseq", "rsa.misc.action": [ - "umet", - "Allowed" + "Allowed", + "umet" ], "rsa.misc.category": "ciad", "rsa.misc.filter": "oeiusmod", @@ -5671,7 +6115,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "imidest", "url.original": "https://www5.example.net/lits/Nemoen.txt?elillu=seruntmo#imidest", + "url.path": "/lits/Nemoen.txt", + "url.query": "elillu=seruntmo", + "url.scheme": "https", "user.name": "squir", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -5723,8 +6173,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iamquisn", "rsa.misc.action": [ - "lupta", - "Blocked" + "Blocked", + "lupta" ], "rsa.misc.category": "uasiarch", "rsa.misc.filter": "usBonor", @@ -5747,7 +6197,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "ipis", "url.original": "https://example.org/eius/evo.jpg?iarchit=volupt#ipis", + "url.path": "/eius/evo.jpg", + "url.query": "iarchit=volupt", + "url.scheme": "https", "user.name": "mes", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -5784,8 +6240,8 @@ "tiumtot3611.internal.localdomain" ], "related.ip": [ - "10.107.68.114", - "10.84.9.150" + "10.84.9.150", + "10.107.68.114" ], "related.user": [ "sequatDu" @@ -5823,7 +6279,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "uiineavo", "url.original": "https://www5.example.net/equun/veli.gif?tem=iadeseru#uiineavo", + "url.path": "/equun/veli.gif", + "url.query": "tem=iadeseru", + "url.scheme": "https", "user.name": "sequatDu", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -5899,7 +6361,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "rese", "url.original": "https://internal.example.com/ecatcu/tMalo.txt?nse=rauto#rese", + "url.path": "/ecatcu/tMalo.txt", + "url.query": "nse=rauto", + "url.scheme": "https", "user.name": "nre", "user_agent.device.name": "Samsung SM-A305FN", "user_agent.name": "YandexSearch", @@ -5936,8 +6404,8 @@ "psaqu6066.www5.localhost" ], "related.ip": [ - "10.223.11.164", - "10.164.190.2" + "10.164.190.2", + "10.223.11.164" ], "related.user": [ "ten" @@ -5951,8 +6419,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "officiad", "rsa.misc.action": [ - "Allowed", - "antium" + "antium", + "Allowed" ], "rsa.misc.category": "emoeni", "rsa.misc.filter": "itvo", @@ -5975,7 +6443,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "tionevol", "url.original": "https://mail.example.org/ntutlabo/leumiure.htm?eacommo=amqua#tionevol", + "url.path": "/ntutlabo/leumiure.htm", + "url.query": "eacommo=amqua", + "url.scheme": "https", "user.name": "ten", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6012,8 +6486,8 @@ "iavol5202.api.example" ], "related.ip": [ - "10.14.37.8", - "10.121.181.243" + "10.121.181.243", + "10.14.37.8" ], "related.user": [ "umwr" @@ -6027,8 +6501,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "vitaedic", "rsa.misc.action": [ - "Blocked", - "rinc" + "rinc", + "Blocked" ], "rsa.misc.category": "prehende", "rsa.misc.filter": "rume", @@ -6051,7 +6525,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "mquis", "url.original": "https://www.example.org/ugitsed/ritatis.jpg?xplic=stenat#mquis", + "url.path": "/ugitsed/ritatis.jpg", + "url.query": "xplic=stenat", + "url.scheme": "https", "user.name": "umwr", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -6088,8 +6568,8 @@ "uame1361.api.local" ], "related.ip": [ - "10.10.93.133", - "10.90.20.202" + "10.90.20.202", + "10.10.93.133" ], "related.user": [ "evita" @@ -6127,7 +6607,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "cidu", "url.original": "https://mail.example.com/aute/dictasu.gif?ptas=iadolo#cidu", + "url.path": "/aute/dictasu.gif", + "url.query": "ptas=iadolo", + "url.scheme": "https", "user.name": "evita", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -6164,8 +6650,8 @@ "rsitame4049.internal.corp" ], "related.ip": [ - "10.77.102.206", - "10.34.98.144" + "10.34.98.144", + "10.77.102.206" ], "related.user": [ "tectobe" @@ -6179,8 +6665,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Exce", "rsa.misc.action": [ - "ulapa", - "Allowed" + "Allowed", + "ulapa" ], "rsa.misc.category": "reprehen", "rsa.misc.filter": "itsedqui", @@ -6203,7 +6689,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "psamvolu", "url.original": "https://mail.example.net/enbyCic/aturau.gif?orroqui=sci#psamvolu", + "url.path": "/enbyCic/aturau.gif", + "url.query": "orroqui=sci", + "url.scheme": "https", "user.name": "tectobe", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -6240,8 +6732,8 @@ "elit912.www5.test" ], "related.ip": [ - "10.176.233.249", - "10.75.144.118" + "10.75.144.118", + "10.176.233.249" ], "related.user": [ "isnos" @@ -6279,7 +6771,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "aeratvol", "url.original": "https://example.org/olu/mqua.txt?mdolore=ita#aeratvol", + "url.path": "/olu/mqua.txt", + "url.query": "mdolore=ita", + "url.scheme": "https", "user.name": "isnos", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -6331,8 +6829,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uis", "rsa.misc.action": [ - "mvele", - "Allowed" + "Allowed", + "mvele" ], "rsa.misc.category": "vitaedi", "rsa.misc.filter": "ndeomni", @@ -6355,7 +6853,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "ccusa", "url.original": "https://api.example.net/mnisiut/eabil.jpg?psumqui=trude#ccusa", + "url.path": "/mnisiut/eabil.jpg", + "url.query": "psumqui=trude", + "url.scheme": "https", "user.name": "redolo", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6392,8 +6896,8 @@ "uis5050.www.local" ], "related.ip": [ - "10.97.202.149", - "10.13.125.101" + "10.13.125.101", + "10.97.202.149" ], "related.user": [ "colab" @@ -6407,8 +6911,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "atcupi", "rsa.misc.action": [ - "Blocked", - "uaUten" + "uaUten", + "Blocked" ], "rsa.misc.category": "modt", "rsa.misc.filter": "magnidol", @@ -6431,7 +6935,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "ntNeque", "url.original": "https://api.example.net/uamestq/eetdol.html?ctionofd=uianonnu#ntNeque", + "url.path": "/uamestq/eetdol.html", + "url.query": "ctionofd=uianonnu", + "url.scheme": "https", "user.name": "colab", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -6468,8 +6978,8 @@ "ficiad1312.api.host" ], "related.ip": [ - "10.141.66.163", - "10.230.61.102" + "10.230.61.102", + "10.141.66.163" ], "related.user": [ "umdolo" @@ -6507,7 +7017,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "gnido", "url.original": "https://mail.example.net/ius/msequ.jpg?ptat=tionula#gnido", + "url.path": "/ius/msequ.jpg", + "url.query": "ptat=tionula", + "url.scheme": "https", "user.name": "umdolo", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -6544,8 +7060,8 @@ "itaspe921.mail.invalid" ], "related.ip": [ - "10.10.25.145", - "10.224.249.228" + "10.224.249.228", + "10.10.25.145" ], "related.user": [ "mnisiuta" @@ -6559,8 +7075,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "issuscip", "rsa.misc.action": [ - "remap", - "Blocked" + "Blocked", + "remap" ], "rsa.misc.category": "eetdolo", "rsa.misc.filter": "rsitam", @@ -6583,7 +7099,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "umqu", "url.original": "https://www.example.org/iat/acom.html?umdolo=oluptass#umqu", + "url.path": "/iat/acom.html", + "url.query": "umdolo=oluptass", + "url.scheme": "https", "user.name": "mnisiuta", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6659,7 +7181,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "trumexer", "url.original": "https://www.example.com/onorum/umiure.gif?lites=admini#trumexer", + "url.path": "/onorum/umiure.gif", + "url.query": "lites=admini", + "url.scheme": "https", "user.name": "aeabillo", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -6735,7 +7263,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "dquiac", "url.original": "https://mail.example.org/veni/rspi.htm?ntium=imadmi#dquiac", + "url.path": "/veni/rspi.htm", + "url.query": "ntium=imadmi", + "url.scheme": "https", "user.name": "tNequ", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -6772,8 +7306,8 @@ "Bonoru7444.www5.example" ], "related.ip": [ - "10.166.205.159", - "10.154.188.132" + "10.154.188.132", + "10.166.205.159" ], "related.user": [ "uptat" @@ -6787,8 +7321,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "proid", "rsa.misc.action": [ - "onevolu", - "Allowed" + "Allowed", + "onevolu" ], "rsa.misc.category": "iratio", "rsa.misc.filter": "odita", @@ -6811,7 +7345,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "beat", "url.original": "https://www.example.com/tem/litsedq.htm?ium=utfugit#beat", + "url.path": "/tem/litsedq.htm", + "url.query": "ium=utfugit", + "url.scheme": "https", "user.name": "uptat", "user_agent.device.name": "Spider", "user_agent.name": "Other", @@ -6883,7 +7423,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "laborisn", "url.original": "https://www.example.com/amcola/eumiurer.gif?stiaeco=equu#laborisn", + "url.path": "/amcola/eumiurer.gif", + "url.query": "stiaeco=equu", + "url.scheme": "https", "user.name": "sintocca", "user_agent.device.name": "Spider", "user_agent.name": "Other", @@ -6916,8 +7462,8 @@ "oloremeu5047.www5.invalid" ], "related.ip": [ - "10.254.119.31", - "10.172.159.251" + "10.172.159.251", + "10.254.119.31" ], "related.user": [ "usm" @@ -6955,7 +7501,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "veleum", "url.original": "https://api.example.net/sedquian/lamcorpo.html?sequatD=Nequepo#veleum", + "url.path": "/sedquian/lamcorpo.html", + "url.query": "sequatD=Nequepo", + "url.scheme": "https", "user.name": "usm", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -6992,8 +7544,8 @@ "edutpe1255.internal.lan" ], "related.ip": [ - "10.195.62.230", - "10.98.126.206" + "10.98.126.206", + "10.195.62.230" ], "related.user": [ "ptassit" @@ -7031,7 +7583,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "uisnostr", "url.original": "https://www5.example.com/ictasun/iumto.txt?erro=admin#uisnostr", + "url.path": "/ictasun/iumto.txt", + "url.query": "erro=admin", + "url.scheme": "https", "user.name": "ptassit", "user_agent.device.name": "Samsung SM-A715F", "user_agent.name": "Facebook", @@ -7107,7 +7665,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "isi", "url.original": "https://www5.example.org/oriosa/ssusc.htm?atemacc=rsitvolu#isi", + "url.path": "/oriosa/ssusc.htm", + "url.query": "atemacc=rsitvolu", + "url.scheme": "https", "user.name": "eroi", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -7159,8 +7723,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tquovo", "rsa.misc.action": [ - "qua", - "Allowed" + "Allowed", + "qua" ], "rsa.misc.category": "ectet", "rsa.misc.filter": "lites", @@ -7183,7 +7747,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "ame", "url.original": "https://mail.example.net/tseddoei/byCi.gif?assitas=nul#ame", + "url.path": "/tseddoei/byCi.gif", + "url.query": "assitas=nul", + "url.scheme": "https", "user.name": "unt", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -7235,8 +7805,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "rrorsi", "rsa.misc.action": [ - "exe", - "Allowed" + "Allowed", + "exe" ], "rsa.misc.category": "mnihi", "rsa.misc.filter": "consequa", @@ -7259,7 +7829,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "suntin", "url.original": "https://www5.example.org/liquipe/rehe.gif?niamqu=uioffi#suntin", + "url.path": "/liquipe/rehe.gif", + "url.query": "niamqu=uioffi", + "url.scheme": "https", "user.name": "hende", "user_agent.device.name": "Samsung GT-P3100 ", "user_agent.name": "Android", @@ -7296,8 +7872,8 @@ "ueip6097.api.host" ], "related.ip": [ - "10.152.217.174", - "10.128.43.71" + "10.128.43.71", + "10.152.217.174" ], "related.user": [ "mquiado" @@ -7311,8 +7887,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "olupt", "rsa.misc.action": [ - "Blocked", - "temvele" + "temvele", + "Blocked" ], "rsa.misc.category": "natuser", "rsa.misc.filter": "amnihil", @@ -7335,7 +7911,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "siuta", "url.original": "https://www.example.org/erit/asiarch.gif?tdolor=oremagna#siuta", + "url.path": "/erit/asiarch.gif", + "url.query": "tdolor=oremagna", + "url.scheme": "https", "user.name": "mquiado", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -7411,7 +7993,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "ainci", "url.original": "https://mail.example.org/maven/tectob.jpg?litsedd=mnis#ainci", + "url.path": "/maven/tectob.jpg", + "url.query": "litsedd=mnis", + "url.scheme": "https", "user.name": "uisa", "user_agent.device.name": "QMobile X700 PRO II", "user_agent.name": "Chrome Mobile", @@ -7448,8 +8036,8 @@ "onsequ3168.www.corp" ], "related.ip": [ - "10.109.192.53", - "10.172.17.6" + "10.172.17.6", + "10.109.192.53" ], "related.user": [ "eprehen" @@ -7487,7 +8075,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "tNe", "url.original": "https://www.example.com/siarch/oloremi.htm?one=iduntutl#tNe", + "url.path": "/siarch/oloremi.htm", + "url.query": "one=iduntutl", + "url.scheme": "https", "user.name": "eprehen", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -7524,8 +8118,8 @@ "oremquel3120.internal.localhost" ], "related.ip": [ - "10.135.38.213", - "10.119.106.108" + "10.119.106.108", + "10.135.38.213" ], "related.user": [ "ore" @@ -7563,7 +8157,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "umtota", "url.original": "https://mail.example.com/ostr/liqu.txt?niam=mullamc#umtota", + "url.path": "/ostr/liqu.txt", + "url.query": "niam=mullamc", + "url.scheme": "https", "user.name": "ore", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile",