diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b1479241288..f297b6619e0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -61,6 +61,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif - Netflow: replace invalid field value. {pull}31295[31295] - google_workspace: Fix pagination to prevent skipped events when more than one page is present. {pull}31372[31372] - sophos.xg: Update module to handle new log fields. {issue}31038[31038] {pull}31388[31388] +- cisco: Fix umbrella dns logs populating destination.ip instead of source.nat.ip. {pull}31454[31454] *Heartbeat* diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml index 1719fa9e4d2..914e0b8093d 100644 --- a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -22,7 +22,7 @@ processors: - source.user.name - cisco.umbrella.identities - source.address - - destination.address + - source.nat.ip - cisco.umbrella.action - dns.question.type - dns.response_code @@ -192,16 +192,38 @@ processors: if: ctx?.cisco?.umbrella?.action != null # Converting address fields to either ip or domain -- grok: +- convert: field: source.address - patterns: - - "(?:%{IP:source.ip}|%{GREEDYDATA:source.domain})" - ignore_failure: true -- grok: + target_field: source.ip + type: ip + ignore_missing: true + on_failure: + - set: + copy_from: source.address + field: source.domain + override: true + +- convert: field: destination.address - patterns: - - "(?:%{IP:destination.ip}|%{GREEDYDATA:destination.domain})" - ignore_failure: true + target_field: destination.ip + type: ip + ignore_missing: true + on_failure: + - set: + field: destination.domain + copy_from: destination.address + override: true + +# For nat, there's no address or domain subfield. +# If the value is not a valid IP, it must be removed +# or ingestion will fail. Probably just an empty value. +- convert: + field: source.nat.ip + type: ip + ignore_missing: true + on_failure: + - remove: + field: source.nat.ip - community_id: ignore_missing: true diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log index f88c691e6c8..df55e942230 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log @@ -1,3 +1,3 @@ -"2020-07-23 23:49:54","elasticuser","elasticuser2","some other identity","192.168.1.1","175.16.199.1","Allowed","1 (A)","NOERROR","elastic.co.","Software/Technology,Business Services,Application","Test Policy Name","SomeIdentityType","" -"2020-07-23 23:50:25","elasticuser","elasticuser2","some other identity","192.168.1.1","4.4.4.4","Blocked","1 (A)","NOERROR","elastic.co/something.","Chat,Instant Messaging,Block List,Application","Test Policy Name","SomeIdentityType","BlockedCategories" +"2020-07-23 23:49:54","elasticuser","elasticuser2,some other identity","192.168.1.1","175.16.199.1","Allowed","1 (A)","NOERROR","elastic.co.","Software/Technology,Business Services,Application","Test Policy Name","SomeIdentityType","" +"2020-07-23 23:50:25","elasticuser","elasticuser2,some other identity","192.168.1.1","4.4.4.4","Blocked","1 (A)","NOERROR","elastic.co/something.","Chat,Instant Messaging,Block List,Application","Test Policy Name","SomeIdentityType","BlockedCategories" "2021-05-14 19:39:58","elastic_machine","elastic_machine,Elastic User (ElasticUser@elastic.co)","89.160.20.156","2.2.2.2","Allowed","1 (A)","NOERROR","elastic.co.","Infrastructure","Roaming Computers","Roaming Computers,AD Users","" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json index 6833ac5cd8f..aac97c50e16 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json @@ -1,92 +1,93 @@ [ { "@timestamp": "2020-07-23T23:49:54.000Z", - "cisco.umbrella.blocked_categories": "SomeIdentityType", - "cisco.umbrella.categories": "elastic.co.", + "cisco.umbrella.categories": "Software/Technology,Business Services,Application", "cisco.umbrella.identities": [ - "elasticuser2" + "elasticuser2", + "some other identity" ], - "cisco.umbrella.identity_types": "Test Policy Name", - "cisco.umbrella.policy_identity_type": "Software/Technology,Business Services,Application", - "destination.address": "192.168.1.1", - "destination.domain": "NOERROR", - "destination.ip": "192.168.1.1", - "dns.question.type": "Allowed", - "dns.response_code": "1 (A)", + "cisco.umbrella.identity_types": "SomeIdentityType", + "cisco.umbrella.policy_identity_type": "Test Policy Name", + "destination.domain": "elastic.co.", + "dns.question.type": "1 (A)", + "dns.response_code": "NOERROR", "dns.type": "query", - "event.action": "dns-request-175.16.199.1", + "event.action": "dns-request-Allowed", "event.category": "network", "event.dataset": "cisco.umbrella", "event.module": "cisco", - "event.original": "\\\"2020-07-23 23:49:54\\\",\\\"elasticuser\\\",\\\"elasticuser2\\\",\\\"some other identity\\\",\\\"192.168.1.1\\\",\\\"175.16.199.1\\\",\\\"Allowed\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co.\\\",\\\"Software/Technology,Business Services,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"\\\"", + "event.original": "\\\"2020-07-23 23:49:54\\\",\\\"elasticuser\\\",\\\"elasticuser2,some other identity\\\",\\\"192.168.1.1\\\",\\\"175.16.199.1\\\",\\\"Allowed\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co.\\\",\\\"Software/Technology,Business Services,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"\\\"", "event.type": [ + "allowed", "connection" ], "fileset.name": "umbrella", "input.type": "log", "log.offset": 0, - "message": "\"2020-07-23 23:49:54\",\"elasticuser\",\"elasticuser2\",\"some other identity\",\"192.168.1.1\",\"175.16.199.1\",\"Allowed\",\"1 (A)\",\"NOERROR\",\"elastic.co.\",\"Software/Technology,Business Services,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"\"", + "message": "\"2020-07-23 23:49:54\",\"elasticuser\",\"elasticuser2,some other identity\",\"192.168.1.1\",\"175.16.199.1\",\"Allowed\",\"1 (A)\",\"NOERROR\",\"elastic.co.\",\"Software/Technology,Business Services,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"\"", "observer.product": "Umbrella", "observer.type": "dns", "observer.vendor": "Cisco", "related.hosts": [ - "NOERROR", - "some other identity" + "elastic.co." ], "related.ip": [ + "175.16.199.1", "192.168.1.1" ], "related.user": [ "elasticuser" ], "service.type": "cisco", - "source.address": "some other identity", - "source.domain": "some other identity", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.nat.ip": "175.16.199.1", "source.user.name": "elasticuser" }, { "@timestamp": "2020-07-23T23:50:25.000Z", - "cisco.umbrella.blocked_categories": "SomeIdentityType", - "cisco.umbrella.categories": "elastic.co/something.", + "cisco.umbrella.blocked_categories": "BlockedCategories", + "cisco.umbrella.categories": "Chat,Instant Messaging,Block List,Application", "cisco.umbrella.identities": [ - "elasticuser2" + "elasticuser2", + "some other identity" ], - "cisco.umbrella.identity_types": "Test Policy Name", - "cisco.umbrella.policy_identity_type": "Chat,Instant Messaging,Block List,Application", - "destination.address": "192.168.1.1", - "destination.domain": "NOERROR", - "destination.ip": "192.168.1.1", - "dns.question.type": "Blocked", - "dns.response_code": "1 (A)", + "cisco.umbrella.identity_types": "SomeIdentityType", + "cisco.umbrella.policy_identity_type": "Test Policy Name", + "destination.domain": "elastic.co/something.", + "dns.question.type": "1 (A)", + "dns.response_code": "NOERROR", "dns.type": "query", - "event.action": "dns-request-4.4.4.4", + "event.action": "dns-request-Blocked", "event.category": "network", "event.dataset": "cisco.umbrella", "event.module": "cisco", - "event.original": "\\\"2020-07-23 23:50:25\\\",\\\"elasticuser\\\",\\\"elasticuser2\\\",\\\"some other identity\\\",\\\"192.168.1.1\\\",\\\"4.4.4.4\\\",\\\"Blocked\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co/something.\\\",\\\"Chat,Instant Messaging,Block List,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"BlockedCategories\\\"", + "event.original": "\\\"2020-07-23 23:50:25\\\",\\\"elasticuser\\\",\\\"elasticuser2,some other identity\\\",\\\"192.168.1.1\\\",\\\"4.4.4.4\\\",\\\"Blocked\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co/something.\\\",\\\"Chat,Instant Messaging,Block List,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"BlockedCategories\\\"", "event.type": [ - "connection" + "connection", + "denied" ], "fileset.name": "umbrella", "input.type": "log", - "log.offset": 237, - "message": "\"2020-07-23 23:50:25\",\"elasticuser\",\"elasticuser2\",\"some other identity\",\"192.168.1.1\",\"4.4.4.4\",\"Blocked\",\"1 (A)\",\"NOERROR\",\"elastic.co/something.\",\"Chat,Instant Messaging,Block List,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"BlockedCategories\"", + "log.offset": 235, + "message": "\"2020-07-23 23:50:25\",\"elasticuser\",\"elasticuser2,some other identity\",\"192.168.1.1\",\"4.4.4.4\",\"Blocked\",\"1 (A)\",\"NOERROR\",\"elastic.co/something.\",\"Chat,Instant Messaging,Block List,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"BlockedCategories\"", "observer.product": "Umbrella", "observer.type": "dns", "observer.vendor": "Cisco", "related.hosts": [ - "NOERROR", - "some other identity" + "elastic.co/something." ], "related.ip": [ - "192.168.1.1" + "192.168.1.1", + "4.4.4.4" ], "related.user": [ "elasticuser" ], "service.type": "cisco", - "source.address": "some other identity", - "source.domain": "some other identity", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.nat.ip": "4.4.4.4", "source.user.name": "elasticuser" }, { @@ -98,9 +99,7 @@ ], "cisco.umbrella.identity_types": "Roaming Computers,AD Users", "cisco.umbrella.policy_identity_type": "Roaming Computers", - "destination.address": "2.2.2.2", "destination.domain": "elastic.co.", - "destination.ip": "2.2.2.2", "dns.question.type": "1 (A)", "dns.response_code": "NOERROR", "dns.type": "query", @@ -115,7 +114,7 @@ ], "fileset.name": "umbrella", "input.type": "log", - "log.offset": 492, + "log.offset": 488, "message": "\"2021-05-14 19:39:58\",\"elastic_machine\",\"elastic_machine,Elastic User (ElasticUser@elastic.co)\",\"89.160.20.156\",\"2.2.2.2\",\"Allowed\",\"1 (A)\",\"NOERROR\",\"elastic.co.\",\"Infrastructure\",\"Roaming Computers\",\"Roaming Computers,AD Users\",\"\"", "observer.product": "Umbrella", "observer.type": "dns", @@ -134,6 +133,7 @@ "service.type": "cisco", "source.address": "89.160.20.156", "source.ip": "89.160.20.156", + "source.nat.ip": "2.2.2.2", "source.user.name": "elastic_machine" } ] \ No newline at end of file