[Filebeat] Update crowdstrike module

CHANGELOG.next.asciidoc

@@ -221,6 +221,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984]
 - Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552]
 - Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962]
+- Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138]
 
 *Heartbeat*
 
@@ -487,6 +488,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add experimental dataset squid/log for Squid Proxy Server logs {pull}19713[19713]
 - Add experimental dataset zscaler/zia for Zscaler Internet Access logs {pull}19713[19713]
 - Add initial support for configurable file identity tracking. {pull}18748[18748]
+- Add event.ingested for CrowdStrike module {pull}20138[20138]
 
 *Heartbeat*
 

filebeat/tests/system/test_modules.py

@@ -239,6 +239,9 @@ def clean_keys(obj):
         "redis.log",
         "system.auth",
         "system.syslog",
+        "microsoft.defender_atp",
+        "crowdstrike.falcon_endpoint",
+        "crowdstrike.falcon_audit",
     }
     # dataset + log file pairs for which @timestamp is kept as an exception from above
     remove_timestamp_exception = {
@@ -265,6 +268,8 @@ def clean_keys(obj):
             delete_key(obj, "@timestamp")
             # Also remove alternate time field from rsa parsers.
             delete_key(obj, "rsa.time.event_time")
+            # Remove event.ingested from testing, as it will never be the same.
+            delete_key(obj, "event.ingested")
         else:
             # excluded events need to have their filename saved to the expected.json
             # so that the exception mechanism can be triggered when the json is
@@ -276,11 +281,6 @@ def clean_keys(obj):
         if "event.end" not in obj:
             delete_key(obj, "@timestamp")
 
-    # Remove event.ingested from testing, as it will never be the same.
-    if obj["event.dataset"] == "microsoft.defender_atp":
-        delete_key(obj, "event.ingested")
-        delete_key(obj, "@timestamp")
-
     if obj["event.module"] == "gsuite":
         delete_key(obj, "event.ingested")
 

x-pack/filebeat/module/crowdstrike/falcon/_meta/fields.yml

@@ -36,7 +36,7 @@
     Event data fields for each event and alert.
   type: group
   default_field: false
-  fields: 
+  fields:
     - name: ProcessStartTime
       type: date
       description: >
@@ -106,7 +106,7 @@
       type: keyword
       description: >
         SHA256 sum of the executable associated with the detection.
- 
+
     - name: MD5String
       type: keyword
       description: >

x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js

@@ -2,11 +2,26 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-var crowdstrikeFalcon = (function() {
+var crowdstrikeFalcon = (function () {
     var processor = require("processor");
 
-    var convertUnderscore = function(text) {
-        return text.split(/(?=[A-Z])/).join('_').toLowerCase(); 
+    var convertUnderscore = function (text) {
+        return text.split(/(?=[A-Z])/).join('_').toLowerCase();
+    };
+
+    var convertToMSEpoch = function (evt, field) {
+        var timestamp = evt.Get(field);
+        if (timestamp) {
+            if (timestamp < 100000000000) { // check if we have a seconds timestamp, this is roughly 1973 in MS
+                evt.Put(field, timestamp * 1000);
+            }
+            (new processor.Timestamp({
+                field: field,
+                target_field: field,
+                timezone: "UTC",
+                layouts: ["UNIX_MS"]
+            })).Run(evt);
+        }
     };
 
     var decodeJson = new processor.DecodeJSONFields({
@@ -16,57 +31,79 @@ var crowdstrikeFalcon = (function() {
         max_depth: 8
     });
 
-    var dropFields = function(evt) {
+    var dropFields = function (evt) {
         evt.Delete("message");
         evt.Delete("host.name");
     };
-  
+
     var setFields = function (evt) {
         evt.Put("agent.name", "falcon");
     };
 
     var convertFields = new processor.Convert({
         fields: [
             // DetectionSummaryEvent
-            { from: "crowdstrike.event.LocalIP", to: "source.ip", type: "ip" },
-            { from: "crowdstrike.event.ProcessId", to: "process.pid" },
+            {
+                from: "crowdstrike.event.LocalIP",
+                to: "source.ip",
+                type: "ip"
+            },
+            {
+                from: "crowdstrike.event.ProcessId",
+                to: "process.pid"
+            },
             // UserActivityAuditEvent and AuthActivityAuditEvent
-            { from: "crowdstrike.event.UserIp", to: "source.ip", type: "ip" },
+            {
+                from: "crowdstrike.event.UserIp",
+                to: "source.ip",
+                type: "ip"
+            },
         ],
         mode: "copy",
         ignore_missing: true,
-        ignore_failure: true
+        fail_on_error: false
     });
 
-    var parseTimestamp = new processor.Timestamp({
-        field: "crowdstrike.metadata.eventCreationTime",
-        target_field: "@timestamp",
-        timezone: "UTC",
-        layouts: ["UNIX_MS"],
+    var addTimestamp = new processor.Convert({
+        fields: [{
+            from: "crowdstrike.metadata.eventCreationTime",
+            to: "@timestamp",
+        }],
+        mode: "copy",
         ignore_missing: false,
+        fail_on_error: true
     });
 
-    var processEvent = function(evt) {
+    var normalizeEpochMS = function (evt) {
+        convertToMSEpoch(evt, "crowdstrike.event.ProcessStartTime")
+        convertToMSEpoch(evt, "crowdstrike.event.ProcessEndTime")
+        convertToMSEpoch(evt, "crowdstrike.event.IncidentStartTime")
+        convertToMSEpoch(evt, "crowdstrike.event.IncidentEndTime")
+        convertToMSEpoch(evt, "crowdstrike.event.StartTimestamp")
+        convertToMSEpoch(evt, "crowdstrike.event.EndTimestamp")
+        convertToMSEpoch(evt, "crowdstrike.event.UTCTimestamp")
+        convertToMSEpoch(evt, "crowdstrike.metadata.eventCreationTime")
+    };
+
+    var processEvent = function (evt) {
         var eventType = evt.Get("crowdstrike.metadata.eventType")
         var outcome = evt.Get("crowdstrike.event.Success")
 
         evt.Put("event.kind", "event")
 
         if (outcome === true) {
             evt.Put("event.outcome", "success")
-        }
-        else if (outcome === false) {
+        } else if (outcome === false) {
             evt.Put("event.outcome", "failure")
-        }
-        else {
+        } else {
             evt.Put("event.outcome", "unknown")
         }
 
         switch (eventType) {
             case "DetectionSummaryEvent":
                 var tactic = evt.Get("crowdstrike.event.Tactic").toLowerCase()
                 var technique = evt.Get("crowdstrike.event.Technique").toLowerCase()
-                evt.Put("threat.technique.name", technique) 
+                evt.Put("threat.technique.name", technique)
                 evt.Put("threat.tactic.name", tactic)
 
                 evt.Put("event.action", evt.Get("crowdstrike.event.PatternDispositionDescription"))
@@ -167,14 +204,16 @@ var crowdstrikeFalcon = (function() {
             default:
                 break;
         }
-    } 
+    }
 
     var pipeline = new processor.Chain()
         .Add(decodeJson)
-        .Add(parseTimestamp)
+        .Add(normalizeEpochMS)
         .Add(dropFields)
+        .Add(addTimestamp)
         .Add(convertFields)
         .Add(processEvent)
+        .Add(setFields)
         .Build();
 
     return {

x-pack/filebeat/module/crowdstrike/falcon/ingest/pipeline.yml

@@ -0,0 +1,31 @@
+description: Ingest pipeline for normalizing CrowdStrike Falcon logs
+processors:
+  - set:
+      field: event.ingested
+      value: '{{_ingest.timestamp}}'
+  - script:
+      lang: painless
+      if: ctx?.crowdstrike?.event != null
+      params:
+        values:
+          - null
+          - ''
+          - '-'
+          - 'N/A'
+      source: |
+        ctx.crowdstrike.event.entrySet().removeIf(entry -> params.values.contains(entry.getValue()));
+  - script:
+      lang: painless
+      if: ctx?.crowdstrike?.metadata != null
+      params:
+        values:
+          - null
+          - ''
+          - '-'
+          - 'N/A'
+      source: |
+        ctx.crowdstrike.metadata.entrySet().removeIf(entry -> params.values.contains(entry.getValue()));
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'

x-pack/filebeat/module/crowdstrike/falcon/manifest.yml

@@ -8,3 +8,4 @@ var:
     default: [forwarded]
 
 input: config/falcon.yml
+ingest_pipeline: ingest/pipeline.yml