diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7960cd604db4..1840924c9952 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -125,6 +125,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Allow the `misp` fileset in the Filebeat `threatintel` module to ignore CIDR ranges for an IP field. {issue}29949[29949] {pull}34195[34195] - Remove incorrect reference to CEL ext extensions package. {issue}34610[34610] {pull}34620[34620] - Fix handling of RFC5988 links' relation parameters by `getRFC5988Link` in HTTPJSON. {issue}34603[34603] {pull}34622[34622] +- Drop empty API response events for Microsoft module. {issue}34786[34786] {pull}34893[34893] *Auditbeat* diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml index 937cc6cbf064..9107c2db3a76 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml @@ -29,6 +29,7 @@ request.transforms: response.split: target: body.value + ignore_empty_value: true split: target: body.evidence keep_parent: true diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index 0e7ce8abfefa..81b92a63faa7 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -1,6 +1,8 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- drop: + if: ctx.json?.value != null && ctx.json.value.isEmpty() - set: field: event.ingested value: '{{_ingest.timestamp}}' diff --git a/x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log b/x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log index 44ada18d449e..e6db58e92ed8 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log +++ b/x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log @@ -2,3 +2,4 @@ {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"543bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"123543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"Process","sha1":"b6d237154f2e528f0b503b58b025862d66b02b73","sha256":"a92056d772260b39a876d01552496b2f8b4610a0b1e084952fe1176784e2ce77","fileName":"notepad.exe","filePath":"C:\\Windows\\System32","processId":4104,"processCommandLine":"\"notepad.exe\"","processCreationTime":"2020-06-30T09:45:38.9784654Z","parentProcessId":6012,"parentProcessCreationTime":"2020-06-30T09:04:51.487396Z","ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}} {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"53425a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"43521344-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"User","sha1":null,"sha256":null,"fileName":null,"filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":"administrator1","domainName":"TestServer4","userSid":"S-1-5-21-46152456-1367606905-4031241297-500","aadUserId":null,"userPrincipalName":null}} {"id":"da637291063515066999_-2102938302","incidentId":12,"investigationId":9,"assignedTo":"Automation","severity":"Informational","status":"Resolved","classification":null,"determination":null,"investigationState":"Benign","detectionSource":"WindowsDefenderAv","category":"Malware","threatFamilyName":null,"title":"'Mountsi' malware was detected","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","alertCreationTime":"2020-06-30T09:32:31.4579225Z","firstEventTime":"2020-06-30T09:31:22.5729558Z","lastEventTime":"2020-06-30T09:46:15.0876676Z","lastUpdateTime":"2020-06-30T11:13:12.9Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","machineId":"t4563234bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"1234543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":null,"comments":[],"evidence":{"entityType":"File","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}} +{"value":[],"note":"THIS MESSAGE SHOULD NOT END UP IN THE EXPECTS JSON FILE."} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml index 0e5df8243808..6716568ba141 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml @@ -24,6 +24,7 @@ request.transforms: response.split: target: body.value + ignore_empty_value: true split: target: body.alerts keep_parent: true diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index ce339b3e785d..3215a61d1a38 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -1,6 +1,8 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- drop: + if: ctx.json?.value != null && ctx.json.value.isEmpty() - set: field: event.ingested value: '{{_ingest.timestamp}}' diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log index 8bd804528332..e5c48c2a1f19 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log @@ -1 +1,2 @@ {"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]} +{"value":[],"note":"THIS MESSAGE SHOULD NOT END UP IN THE EXPECTS JSON FILE."} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json index 3e57d460d68f..f08a15e75c4b 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json @@ -31,7 +31,7 @@ "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", "microsoft.m365_defender.alerts.devices": [ { - "deviceDnsName": "TestServer5", + "deviceDnsName": "TestServer4", "firstSeen": "2020-06-30T08:55:08.8320449Z", "healthStatus": "Inactive", "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", @@ -43,7 +43,7 @@ "version": "Other" }, { - "deviceDnsName": "TestServer4", + "deviceDnsName": "TestServer5", "firstSeen": "2020-06-30T08:55:08.8320449Z", "healthStatus": "Inactive", "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",