elastic · efd6 · Mar 28, 2023 · Mar 28, 2023 · Mar 28, 2023
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -125,6 +125,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Allow the `misp` fileset in the Filebeat `threatintel` module to ignore CIDR ranges for an IP field. {issue}29949[29949] {pull}34195[34195]
 - Remove incorrect reference to CEL ext extensions package. {issue}34610[34610] {pull}34620[34620]
 - Fix handling of RFC5988 links' relation parameters by `getRFC5988Link` in HTTPJSON. {issue}34603[34603] {pull}34622[34622]
+- Drop empty API response events for Microsoft module. {issue}34786[34786] {pull}34893[34893]
 
 *Auditbeat*
 

diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml
@@ -29,6 +29,7 @@ request.transforms:
 
 response.split:
   target: body.value
+  ignore_empty_value: true
   split:
     target: body.evidence
     keep_parent: true

diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml
@@ -1,6 +1,8 @@
 ---
 description: Pipeline for parsing microsoft atp logs
 processors:
+- drop:
+    if: ctx.json?.value != null && ctx.json.value.isEmpty()
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'

diff --git a/x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log b/x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log
@@ -2,3 +2,4 @@
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"543bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"123543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"Process","sha1":"b6d237154f2e528f0b503b58b025862d66b02b73","sha256":"a92056d772260b39a876d01552496b2f8b4610a0b1e084952fe1176784e2ce77","fileName":"notepad.exe","filePath":"C:\\Windows\\System32","processId":4104,"processCommandLine":"\"notepad.exe\"","processCreationTime":"2020-06-30T09:45:38.9784654Z","parentProcessId":6012,"parentProcessCreationTime":"2020-06-30T09:04:51.487396Z","ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"53425a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"43521344-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"User","sha1":null,"sha256":null,"fileName":null,"filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":"administrator1","domainName":"TestServer4","userSid":"S-1-5-21-46152456-1367606905-4031241297-500","aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291063515066999_-2102938302","incidentId":12,"investigationId":9,"assignedTo":"Automation","severity":"Informational","status":"Resolved","classification":null,"determination":null,"investigationState":"Benign","detectionSource":"WindowsDefenderAv","category":"Malware","threatFamilyName":null,"title":"'Mountsi' malware was detected","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","alertCreationTime":"2020-06-30T09:32:31.4579225Z","firstEventTime":"2020-06-30T09:31:22.5729558Z","lastEventTime":"2020-06-30T09:46:15.0876676Z","lastUpdateTime":"2020-06-30T11:13:12.9Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","machineId":"t4563234bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"1234543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":null,"comments":[],"evidence":{"entityType":"File","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
+{"value":[],"note":"THIS MESSAGE SHOULD NOT END UP IN THE EXPECTS JSON FILE."}
diff --git a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml
@@ -24,6 +24,7 @@ request.transforms:
 
 response.split:
   target: body.value
+  ignore_empty_value: true
   split:
     target: body.alerts
     keep_parent: true

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml
@@ -1,6 +1,8 @@
 ---
 description: Pipeline for parsing microsoft atp logs
 processors:
+- drop:
+    if: ctx.json?.value != null && ctx.json.value.isEmpty()
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log
@@ -1 +1,2 @@
 {"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}
+{"value":[],"note":"THIS MESSAGE SHOULD NOT END UP IN THE EXPECTS JSON FILE."}
diff --git a/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json
@@ -31,7 +31,7 @@
         "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv",
         "microsoft.m365_defender.alerts.devices": [
             {
-                "deviceDnsName": "TestServer5",
+                "deviceDnsName": "TestServer4",
                 "firstSeen": "2020-06-30T08:55:08.8320449Z",
                 "healthStatus": "Inactive",
                 "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",
@@ -43,7 +43,7 @@
                 "version": "Other"
             },
             {
-                "deviceDnsName": "TestServer4",
+                "deviceDnsName": "TestServer5",
                 "firstSeen": "2020-06-30T08:55:08.8320449Z",
                 "healthStatus": "Inactive",
                 "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		{"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}
		{"value":[],"note":"THIS MESSAGE SHOULD NOT END UP IN THE EXPECTS JSON FILE."}