crowdstrike/fdr: Prevent ingesting documents without @timestamp

packages/crowdstrike/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.4"
+  changes:
+    - description: Prevent missing `@timestamp` field.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3484
 - version: "1.3.3"
   changes:
     - description: Optimize FDR pipeline script processor.

packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr.log

@@ -122,4 +122,5 @@
 {"AuthenticationId":"703298","ConfigBuild":"1007.3.0011603.1","ConfigStateHash":"2642284486","ContextProcessId":"1161025471861","ContextThreadId":"34929528116709","ContextTimeStamp":"1604851030.593","DiskParentDeviceInstanceId":"USB\\VID_1058\u0026PID_2621\\57583431453939315A4C5255","EffectiveTransmissionClass":"3","Entitlements":"15","FileEcpBitmask":"0","FileIdentifier":"262fbc677256cf4c8d6c6a227285a072c06830873b000000","FileObject":"18446664963104449168","IrpFlags":"1028","IsOnNetwork":"0","IsOnRemovableDisk":"1","MajorFunction":"18","MinorFunction":"0","OperationFlags":"0","Size":"517029","TargetFileName":"\\Device\\HarddiskVolume5\\01.png.tmp$$","TokenType":"1","UserName":"user9","aid":"ffffffff16bf4c7bb5ad755a4722025c","aip":"67.43.156.13","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Win","event_simpleName":"GenericFileWritten","id":"ffffffff-1111-11eb-800a-06cecfd73923","name":"GenericFileWrittenV11","timestamp":"1604851031298"}
 {"ConfigBuild":"1007.3.0011603.1","ConfigStateHash":"666346415","ContextProcessId":"1717987648455","ContextThreadId":"55064470042288","ContextTimeStamp":"1604850899.164","EffectiveTransmissionClass":"3","Entitlements":"15","VolumeName":"\\Device\\HarddiskVolume27","aid":"ffffffff896b43725b83c79aa79959da","aip":"67.43.156.13","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Win","event_simpleName":"FsVolumeUnmounted","id":"ffffffff-1111-11eb-9f70-0634389d9ea9","name":"FsVolumeUnmountedV2","timestamp":"1604850899812"}
 {"ConfigBuild":"1007.4.0009906.1","ConfigStateHash":"3429017943","ContextProcessId":"66426035996442255","ContextTimeStamp":"1604851098.548","Entitlements":"15","aid":"ffffffff899541b94b9adff8922aa70a","aip":"67.43.156.14","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Mac","event_simpleName":"FirewallDisabled","id":"ffffffff-1111-11eb-9d4c-02f402df8c1f","name":"FirewallDisabledMacV1","timestamp":"1604851040625"}
-{"AgentLoadFlags":"0","AgentLocalTime":"1636436839.9529998","AgentTimeOffset":"125.319","AgentVersion":"6.31.14404.0","BiosManufacturer":"Apple Inc.","BiosVersion":"1554.140.20.0.0 (iBridge: 18.16.14759.0.1,0)","ChassisType":"Laptop","City":"San Francisco","ComputerName":"mac1","ConfigBuild":"1007.4.0014404.1","ConfigIDBuild":"14404","Continent":"North America","Country":"United States","FalconGroupingTags":"-","FirstSeen":"1625682391.0","HostHiddenStatus":"Visible","MachineDomain":"none","OU":"none","PointerSize":"none","ProductType":"1","SensorGroupingTags":"-","ServicePackMajor":"none","SiteName":"none","SystemManufacturer":"Apple Inc.","SystemProductName":"MacBookPro16,2","Time":"1636448427.3539999","Timezone":"America/Los_Angeles","Version":"Big Sur (11.0)","aid":"fffffffffffaaaaaaaaabbbbbbbb","aip":"67.43.156.14","cid":"ffffffff30a3407dae27d0503611022ff","event_platform":"Mac"}
+{"AgentLoadFlags":"0","AgentLocalTime":"1636436839.9529998","AgentTimeOffset":"125.319","AgentVersion":"6.31.14404.0","BiosManufacturer":"Apple Inc.","BiosVersion":"1554.140.20.0.0 (iBridge: 18.16.14759.0.1,0)","ChassisType":"Laptop","City":"San Francisco","ComputerName":"mac1","ConfigBuild":"1007.4.0014404.1","ConfigIDBuild":"14404","Continent":"North America","Country":"United States","FalconGroupingTags":"-","FirstSeen":"1625682391.0","HostHiddenStatus":"Visible","MachineDomain":"none","OU":"none","PointerSize":"none","ProductType":"1","SensorGroupingTags":"-","ServicePackMajor":"none","SiteName":"none","SystemManufacturer":"Apple Inc.","SystemProductName":"MacBookPro16,2","Time":"1636448427.3539999","Timezone":"America/Los_Angeles","Version":"Big Sur (11.0)","aid":"fffffffffffaaaaaaaaabbbbbbbb","aip":"67.43.156.14","cid":"ffffffff30a3407dae27d0503611022ff","event_platform":"Mac"}
+{"AuthenticationId":"317005428","AuthenticationPackage":"Negotiate","ConfigBuild":"1007.3.0011603.1","ConfigStateHash":"3950066843","EffectiveTransmissionClass":"2","Entitlements":"15","LogoffTime":"1604855132.756","LogonDomain":"dom1","LogonServer":"srv2","LogonTime":"1604855131.666","LogonType":"7","PasswordLastSet":"1598119332.510","RemoteAccount":"1","UserFlags":"32","UserIsAdmin":"0","UserLogoffType":"3","UserLogonFlags":"0","UserName":"user4","UserPrincipal":"user.name@dom2.com","UserSid":"S-1-5-21-606747145-1364589140-725345543-28636","aid":"ffffffffe0104823bd3de859d5bc8bc7","aip":"67.43.156.13","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Win","event_simpleName":"UserLogoff","id":"ffffffff-1111-11eb-8913-0287fd11c79b","name":"UserLogoffV3","UTCTimestamp":"1604855134461"}

packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr.log-expected.json

@@ -10814,6 +10814,7 @@
             }
         },
         {
+            "@timestamp": "2021-11-09T05:47:19.952Z",
             "crowdstrike": {
                 "AgentLoadFlags": "0",
                 "AgentLocalTime": "2021-11-09T05:47:19.952Z",
@@ -10835,6 +10836,7 @@
                 "version": "8.2.0"
             },
             "event": {
+                "created": "2021-11-09T05:47:19.952Z",
                 "original": "{\"AgentLoadFlags\":\"0\",\"AgentLocalTime\":\"1636436839.9529998\",\"AgentTimeOffset\":\"125.319\",\"AgentVersion\":\"6.31.14404.0\",\"BiosManufacturer\":\"Apple Inc.\",\"BiosVersion\":\"1554.140.20.0.0 (iBridge: 18.16.14759.0.1,0)\",\"ChassisType\":\"Laptop\",\"City\":\"San Francisco\",\"ComputerName\":\"mac1\",\"ConfigBuild\":\"1007.4.0014404.1\",\"ConfigIDBuild\":\"14404\",\"Continent\":\"North America\",\"Country\":\"United States\",\"FalconGroupingTags\":\"-\",\"FirstSeen\":\"1625682391.0\",\"HostHiddenStatus\":\"Visible\",\"MachineDomain\":\"none\",\"OU\":\"none\",\"PointerSize\":\"none\",\"ProductType\":\"1\",\"SensorGroupingTags\":\"-\",\"ServicePackMajor\":\"none\",\"SiteName\":\"none\",\"SystemManufacturer\":\"Apple Inc.\",\"SystemProductName\":\"MacBookPro16,2\",\"Time\":\"1636448427.3539999\",\"Timezone\":\"America/Los_Angeles\",\"Version\":\"Big Sur (11.0)\",\"aid\":\"fffffffffffaaaaaaaaabbbbbbbb\",\"aip\":\"67.43.156.14\",\"cid\":\"ffffffff30a3407dae27d0503611022ff\",\"event_platform\":\"Mac\"}"
             },
             "host": {
@@ -10883,6 +10885,94 @@
             "url": {
                 "scheme": "http"
             }
+        },
+        {
+            "@timestamp": "2020-11-08T17:05:34.461Z",
+            "crowdstrike": {
+                "AuthenticationId": "317005428",
+                "AuthenticationPackage": "Negotiate",
+                "ConfigStateHash": "3950066843",
+                "EffectiveTransmissionClass": "2",
+                "Entitlements": "15",
+                "LogoffTime": "2020-11-08T17:05:32.756Z",
+                "LogonDomain": "dom1",
+                "LogonServer": "srv2",
+                "LogonTime": "2020-11-08T17:05:31.666Z",
+                "LogonType": "7",
+                "PasswordLastSet": "1598119332.510",
+                "RemoteAccount": "1",
+                "UserFlags": "32",
+                "UserLogoffType": "3",
+                "UserLogonFlags": "0",
+                "cid": "ffffffff30a3407dae27d0503611022d",
+                "name": "UserLogoffV3"
+            },
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "action": "UserLogoff",
+                "category": [
+                    "authentication"
+                ],
+                "created": "2020-11-08T17:05:34.461Z",
+                "id": "ffffffff-1111-11eb-8913-0287fd11c79b",
+                "kind": "event",
+                "original": "{\"AuthenticationId\":\"317005428\",\"AuthenticationPackage\":\"Negotiate\",\"ConfigBuild\":\"1007.3.0011603.1\",\"ConfigStateHash\":\"3950066843\",\"EffectiveTransmissionClass\":\"2\",\"Entitlements\":\"15\",\"LogoffTime\":\"1604855132.756\",\"LogonDomain\":\"dom1\",\"LogonServer\":\"srv2\",\"LogonTime\":\"1604855131.666\",\"LogonType\":\"7\",\"PasswordLastSet\":\"1598119332.510\",\"RemoteAccount\":\"1\",\"UserFlags\":\"32\",\"UserIsAdmin\":\"0\",\"UserLogoffType\":\"3\",\"UserLogonFlags\":\"0\",\"UserName\":\"user4\",\"UserPrincipal\":\"user.name@dom2.com\",\"UserSid\":\"S-1-5-21-606747145-1364589140-725345543-28636\",\"aid\":\"ffffffffe0104823bd3de859d5bc8bc7\",\"aip\":\"67.43.156.13\",\"cid\":\"ffffffff30a3407dae27d0503611022d\",\"event_platform\":\"Win\",\"event_simpleName\":\"UserLogoff\",\"id\":\"ffffffff-1111-11eb-8913-0287fd11c79b\",\"name\":\"UserLogoffV3\",\"UTCTimestamp\":\"1604855134461\"}",
+                "outcome": "success",
+                "type": [
+                    "end"
+                ]
+            },
+            "observer": {
+                "address": "67.43.156.13",
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.13",
+                "serial_number": "ffffffffe0104823bd3de859d5bc8bc7",
+                "type": "agent",
+                "vendor": "crowdstrike",
+                "version": "1007.3.0011603.1"
+            },
+            "os": {
+                "type": "windows"
+            },
+            "related": {
+                "hash": [
+                    "3950066843"
+                ],
+                "hosts": [
+                    "67.43.156.13",
+                    "srv2"
+                ],
+                "ip": [
+                    "67.43.156.13"
+                ],
+                "user": [
+                    "user4",
+                    "user.name"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "scheme": "http"
+            },
+            "user": {
+                "domain": "dom2.com",
+                "email": "user.name@dom2.com",
+                "full_name": "user.name",
+                "id": "S-1-5-21-606747145-1364589140-725345543-28636",
+                "name": "user4"
+            }
         }
     ]
 }

packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml

@@ -11,28 +11,56 @@ processors:
       description: Decodes original JSON into `crowdstrike` field.
       field: event.original
       target_field: crowdstrike
+  - date:
+      tag: date-timestamp-utc
+      description: Parse timestamp from event.
+      field: crowdstrike.UTCTimestamp
+      target_field: event.created
+      formats:
+        - UNIX_MS
+        - ISO8601
+      ignore_failure: true
+      if: ctx.event?.created == null
   - date:
       tag: date-timestamp
       description: Parse timestamp from event.
       field: crowdstrike.timestamp
       target_field: event.created
       formats:
         - UNIX_MS
+        - ISO8601
       ignore_failure: true
+      if: ctx.event?.created == null
   - date:
       tag: date-event-created
       description: Parse timestamp from event.
       field: crowdstrike.CreationTimeStamp
       target_field: event.created
       formats:
         - UNIX
+        - ISO8601
+      ignore_failure: true
+      if: ctx.event?.created == null
+  - date:
+      tag: date-agent-local-time
+      description: Parse timestamp from event.
+      field: crowdstrike.AgentLocalTime
+      target_field: event.created
+      formats:
+        - ISO8601
+        - UNIX
       ignore_failure: true
       if: ctx.event?.created == null
   - set:
       tag: set-timestamp
       field: "@timestamp"
       copy_from: event.created
       if: ctx.event?.created != null && (ctx.crowdstrike?.ContextTimeStamp == null || ctx.crowdstrike?.ContextTimeStamp == "")
+  - set:
+      tag: set-timestamp-ingest
+      field: "@timestamp"
+      copy_from: _ingest.timestamp
+      if: ctx["@timestamp"] == null
   - date:
       tag: date-context-timestamp
       if: ctx.crowdstrike?.ContextTimeStamp != null
@@ -2218,6 +2246,7 @@ processors:
         - crowdstrike.DomainName
         - crowdstrike.ConnectionDirection
         - crowdstrike.UserIsAdmin
+        - crowdstrike.UTCTimestamp
         - crowdstrike.TargetDirectoryName
       ignore_missing: true
       ignore_failure: true

packages/crowdstrike/manifest.yml

@@ -1,6 +1,6 @@
 name: crowdstrike
 title: CrowdStrike Logs
-version: "1.3.3"
+version: "1.3.4"
 description: Collect and parse falcon logs from Crowdstrike products with Elastic Agent.
 type: integration
 format_version: 1.0.0