m365_defender,microsoft_defender_endpoint: drop empty event sets

packages/m365_defender/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.1"
+  changes:
+    - description: Drop empty event sets in log data stream.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5164
 - version: "1.5.0"
   changes:
     - description: Update package to ECS 8.6.0.

packages/m365_defender/data_stream/log/_dev/test/pipeline/test-m365-defender-empty-ndjson.log

@@ -1 +1,2 @@
 {"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}
+{"value":[]}

packages/m365_defender/data_stream/log/_dev/test/pipeline/test-m365-defender-empty-ndjson.log-expected.json

@@ -1,5 +1,6 @@
 {
     "expected": [
+        null,
         null
     ]
 }

packages/m365_defender/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -13,7 +13,7 @@ processors:
       target_field: json
   - drop:
       description: Drops duplicated events without alerts
-      if: ctx.json?.status == 'Redirected'
+      if: ctx.json?.status == 'Redirected' || (ctx.json?.value != null && ctx.json.value.isEmpty())
   - remove:
       field:
         - json.alerts

packages/m365_defender/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: m365_defender
 title: Microsoft M365 Defender
-version: "1.5.0"
+version: "1.5.1"
 description: Collect logs from Microsoft M365 Defender with Elastic Agent.
 categories:
   - "network"

packages/microsoft_defender_endpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.8.1"
+  changes:
+    - description: Drop empty event sets.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5164
 - version: "2.8.0"
   changes:
     - description: Adding support for Oauth2 scopes that is required for some users

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log

@@ -2,3 +2,4 @@
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"543bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"123543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"Process","sha1":"b6d237154f2e528f0b503b58b025862d66b02b73","sha256":"a92056d772260b39a876d01552496b2f8b4610a0b1e084952fe1176784e2ce77","fileName":"notepad.exe","filePath":"C:\\Windows\\System32","processId":4104,"processCommandLine":"\"notepad.exe\"","processCreationTime":"2020-06-30T09:45:38.9784654Z","parentProcessId":6012,"parentProcessCreationTime":"2020-06-30T09:04:51.487396Z","ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"53425a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"43521344-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"User","sha1":null,"sha256":null,"fileName":null,"filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":"administrator1","domainName":"TestServer4","userSid":"S-1-5-21-46152456-1367606905-4031241297-500","aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291063515066999_-2102938302","incidentId":12,"investigationId":9,"assignedTo":"Automation","severity":"Informational","status":"Resolved","classification":null,"determination":null,"investigationState":"Benign","detectionSource":"WindowsDefenderAv","category":"Malware","threatFamilyName":null,"title":"'Mountsi' malware was detected","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","alertCreationTime":"2020-06-30T09:32:31.4579225Z","firstEventTime":"2020-06-30T09:31:22.5729558Z","lastEventTime":"2020-06-30T09:46:15.0876676Z","lastUpdateTime":"2020-06-30T11:13:12.9Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","machineId":"t4563234bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"1234543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":null,"comments":[],"evidence":{"entityType":"File","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
+{"value":[]}

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log-expected.json

@@ -350,6 +350,7 @@
                     "name": "Malware"
                 }
             }
-        }
+        },
+        null
     ]
 }

packages/microsoft_defender_endpoint/data_stream/log/agent/stream/httpjson.yml.hbs

@@ -30,6 +30,7 @@ request.transforms:
       default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05.9999999Z"]]'
 response.split:
   target: body.value
+  ignore_empty_value: true
   split:
     target: body.evidence
     keep_parent: true

packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -11,6 +11,8 @@ processors:
   - json:
       field: event.original
       target_field: json
+  - drop:
+      if: ctx.json?.value != null && ctx.json.value.isEmpty()
   - remove:
       field:
         - json.comments

packages/microsoft_defender_endpoint/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: microsoft_defender_endpoint
 title: Microsoft Defender for Endpoint
-version: 2.8.0
+version: 2.8.1
 description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent.
 categories:
   - "network"