[mimecast] Add use cases, docs, and update sample events

packages/mimecast/changelog.yml

@@ -1,3 +1,8 @@
+- version: "0.0.6"
+  changes:
+    - description: Add use cases for audit events and update sample events and docs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2690
 - version: "0.0.5"
   changes:
     - description: Fix typo

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log

@@ -22,4 +22,6 @@
 {"id":"eNqrVipOTS4tSs1MUbJSigzJC_ZNzg-vcjYKcwz3icotC0nVdgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYG5kqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAD-SK84","auditType":"Archive Mailbox Restore","user":"johndoejr@example.com","eventTime":"2021-10-12T18:19:33+0000","eventInfo":"Archive mailbox restore created. Restored data from johndoe@example.com to johndoejr@example.com by johndoejr@example.com, Date: 2021-10-12, Time: 18:19:33+0000, IP: 67.43.156.15, Application: Administration Console","category":"archive_service_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJScjMvyjIxr6yoLDY2qQopLq3yDnM1dwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYGZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE5dK-0","auditType":"Archive Mailbox Export Download","user":"johndoe@example.com","eventTime":"2021-10-12T17:55:14+0000","eventInfo":"Mailbox export downloaded. Download filename (HTML Report recovery id): eNqrVipOTS4tSs1MUbJSyo3RDw81rTCpynMpdiuICMopyihxynZztcisDMoN9zWLSCrPzAjz9PALNzFwySrLMNQ2yUs38g9zS860cHKNMExR0lFKLi0uyc9NLUrOT0kFGulsYuZobgoUL0pNzi9LLarULUksztYFWWdpaKqjBBQqzszPU7IyrAUAsSEteA by johndoe@example.com, Date: 2021-10-12, Time: 17:55:14+0000, IP: 67.43.156.15, Application: Administration Console","category":"archive_service_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJSitH39gl1cS509PT1MSnw90l0CinPCQgLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsYmBsYqqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAxASul","auditType":"Review Set Action","user":"johndoe@example.com","eventTime":"2021-10-12T17:07:00+0000","eventInfo":"Viewed Review Set Details - Case: Class Action, Review Set: Contracts, Date: 2021-10-12, Time: 17:07:00+0000, IP: 67.43.156.15, Application: mimecast-case-review","category":"case_review_logs"}
-{"id":"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38","auditType":"Remediation Incident Adjustment","user":"johndoe@example.com","eventTime":"2021-10-12T15:38:05+0000","eventInfo":"Restore Remediation Incident Created -  TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console","category":"account_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38","auditType":"Remediation Incident Adjustment","user":"johndoe@example.com","eventTime":"2021-10-12T15:38:05+0000","eventInfo":"Restore Remediation Incident Created -  TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console","category":"account_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked","category":"authentication_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password","category":"authentication_logs"}

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json

@@ -629,7 +629,7 @@
                 "ip": "67.43.156.15"
             },
             "event": {
-                "reason": "Reason: Wrong password",
+                "reason": "Wrong password",
                 "action": "logon-authentication-failed",
                 "ingested": "2021-12-14T14:48:19.342448528Z",
                 "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-12, Time: 09:47:55 BST, IP: 67.43.156.15, Application: mimecast-moa, Method: Office 365, Reason: Wrong password\",\"category\":\"authentication_logs\"}",
@@ -1233,6 +1233,106 @@
                 "category": "account_logs",
                 "eventInfo": "Restore Remediation Incident Created -  TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console"
             }
+        },
+        {
+            "@timestamp": "2021-10-12T08:47:55.000Z",
+            "client": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "event": {
+                "action": "logon-authentication-failed",
+                "created": "2022-01-11T22:54:04.000Z",
+                "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg",
+                "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked\",\"category\":\"authentication_logs\"}",
+                "reason": "Account Locked"
+            },
+            "mimecast": {
+                "application": "POP-POP2",
+                "category": "authentication_logs",
+                "eventInfo": "Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked"
+            },
+            "related": {
+                "ip": [
+                    "67.43.156.15"
+                ],
+                "user": [
+                    "johndoe",
+                    "johndoe@example.com"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "domain": "example.com",
+                "email": "johndoe@example.com",
+                "name": "johndoe"
+            }
+        },
+        {
+            "@timestamp": "2021-10-12T08:47:55.000Z",
+            "client": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "event": {
+                "action": "logon-authentication-failed",
+                "created": "2022-01-11T21:48:01.000Z",
+                "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg",
+                "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password\",\"category\":\"authentication_logs\"}",
+                "reason": "Wrong Password"
+            },
+            "mimecast": {
+                "application": "POP-POP2",
+                "category": "authentication_logs",
+                "eventInfo": "Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password"
+            },
+            "related": {
+                "ip": [
+                    "67.43.156.15"
+                ],
+                "user": [
+                    "johndoe",
+                    "johndoe@example.com"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "domain": "example.com",
+                "email": "johndoe@example.com",
+                "name": "johndoe"
+            }
         }
     ]
 }

packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml

@@ -22,7 +22,15 @@ processors:
       field: mimecast.eventTime
       timezone: UTC
       formats:
-        - yyyy-MM-dd'T'HH:mm:ssZ
+        - "yyyy-MM-dd'T'HH:mm:ssz"
+        - "yyyy-MM-dd'T'HH:mm:ssZ"
+        - "yyyy-MM-dd'T'HH:mm:ss.Sz"
+        - "yyyy-MM-dd'T'HH:mm:ss.SZ"
+        - "yyyy-MM-dd'T'HH:mm:ss.SSz"
+        - "yyyy-MM-dd'T'HH:mm:ss.SSZ"
+        - "yyyy-MM-dd'T'HH:mm:ss.SSSz"
+        - "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
+        - "yyyy-MM-dd'T'HH:mm:ss z"
 
   ###
 
@@ -86,16 +94,47 @@ processors:
       if: 'ctx?.event?.action == "message-action"'
   - dissect:
       field: mimecast.eventInfo
-      pattern: "%{mimecast.info}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}, %{mimecast.application_method}, %{event.reason}"
+      pattern: "%{mimecast.info}, %{?key}:%{mimecast.email.address}[%{mimecast.email.metadata}] %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
       if: 'ctx?.event?.action=="logon-authentication-failed"'
       ignore_missing: true
       ignore_failure: true
   - dissect:
       field: mimecast.eventInfo
-      pattern: "%{mimecast.info}, %{?key}:%{mimecast.email.address}[%{mimecast.email.metadata}] %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
+      pattern: "%{mimecast.info}, %{mimecast.rest_of_event_info}"
       if: 'ctx?.event?.action=="logon-authentication-failed"'
       ignore_missing: true
       ignore_failure: true
+  - kv:
+      field: mimecast.rest_of_event_info
+      field_split: ", "
+      value_split: ": "
+      target_field: mimecast.event_info_parts
+      ignore_missing: true
+      if: 'ctx?.event?.action=="logon-authentication-failed"'
+  - rename:
+      field: mimecast.event_info_parts.Date
+      target_field: mimecast.date
+      ignore_missing: true
+  - rename:
+      field: mimecast.event_info_parts.Time
+      target_field: mimecast.time
+      ignore_missing: true
+  - rename:
+      field: mimecast.event_info_parts.IP
+      target_field: client.ip
+      ignore_missing: true
+  - rename:
+      field: mimecast.event_info_parts.Application
+      target_field: mimecast.application
+      ignore_missing: true
+  - rename:
+      field: mimecast.event_info_parts.Method
+      target_field: mimecast.application_method
+      ignore_missing: true
+  - rename:
+      field: mimecast.event_info_parts.Reason
+      target_field: event.reason
+      ignore_missing: true
   - dissect:
       field: mimecast.eventInfo
       pattern: "%{?drop->} - %{mimecast.info}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
@@ -152,6 +191,15 @@ processors:
         - yyyy-MM-dd HH:mm:ssZ
         - yyyy-MM-dd HH:mm:ss z
         - yyyy-MM-dd HH:mm:ss
+        - yyyy-MM-dd'T'HH:mm:ssz
+        - yyyy-MM-dd'T'HH:mm:ssZ
+        - yyyy-MM-dd'T'HH:mm:ss.Sz
+        - yyyy-MM-dd'T'HH:mm:ss.SZ
+        - yyyy-MM-dd'T'HH:mm:ss.SSz
+        - yyyy-MM-dd'T'HH:mm:ss.SSZ
+        - yyyy-MM-dd'T'HH:mm:ss.SSSz
+        - yyyy-MM-dd'T'HH:mm:ss.SSSZ
+        - yyyy-MM-dd'T'HH:mm:ss z
       if: 'ctx?.event?.created != null'
   - geoip:
       field: client.ip
@@ -221,6 +269,8 @@ processors:
         - mimecast.columns_exported
         - mimecast.as.asn
         - mimecast.organization_name
+        - mimecast.event_info_parts
+        - mimecast.rest_of_event_info
       ignore_missing: true
   - remove:
       description: Remove 'event.original' if 'preserve_original_event' is not set.

packages/mimecast/data_stream/audit_events/sample_event.json

@@ -1,12 +1,25 @@
 {
-    "@timestamp": "2021-11-16T12:01:37.000Z",
-    "agent": {
-        "ephemeral_id": "57841034-22ed-4fcd-bcfd-0a9518249e2d",
-        "hostname": "docker-fleet-agent",
-        "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab",
-        "name": "docker-fleet-agent",
-        "type": "filebeat",
-        "version": "7.16.0"
+    "@timestamp": "2022-02-09T02:45:01.000Z",
+    "file": {
+        "extension": "zip",
+        "name": "Threat intel multiple feeds download  - malware_customer_csv_20220209024500934.zip"
+    },
+    "ecs": {
+        "version": "1.12.0"
+    },
+    "related": {
+        "ip": [
+            "8.8.8.8"
+        ],
+        "user": [
+            "johndoe",
+            "johndoe@example.com"
+        ]
+    },
+    "data_stream": {
+        "namespace": "default",
+        "type": "logs",
+        "dataset": "mimecast.audit_events"
     },
     "client": {
         "as": {
@@ -26,53 +39,22 @@
         },
         "ip": "8.8.8.8"
     },
-    "data_stream": {
-        "dataset": "mimecast.audit_events",
-        "namespace": "ep",
-        "type": "logs"
-    },
-    "ecs": {
-        "version": "1.12.0"
-    },
-    "elastic_agent": {
-        "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab",
-        "snapshot": true,
-        "version": "7.16.0"
-    },
     "event": {
-        "action": "case-action",
         "agent_id_status": "verified",
-        "created": "2021-11-16T12:01:37.000Z",
-        "dataset": "mimecast.audit_events",
-        "id": "eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI",
-        "ingested": "2021-11-24T15:39:11Z",
-        "original": "{\"auditType\":\"Case Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI\",\"user\":\"johndoe@example.com\"}"
+        "ingested": "2022-02-09T09:45:25Z",
+        "created": "2022-02-09T02:45:01.000Z",
+        "action": "threat-intel-feed-download",
+        "id": "eNqrVipOTS4tSs1MUbJSyvMxyknzzcqN0S9Nzs_PqCoNCTE2j3ILS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsZGhobmJkYKKjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCY1Sx4",
+        "dataset": "mimecast.audit_events"
     },
-    "input": {
-        "type": "httpjson"
-    },
-    "mimecast": {
-        "application": "mimecast-case-review",
-        "category": "case_review_logs",
-        "eventInfo": "Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review"
-    },
-    "related": {
-        "ip": [
-            "8.8.8.8"
-        ],
-        "user": [
-            "johndoe",
-            "johndoe@example.com"
-        ]
-    },
-    "tags": [
-        "preserve_original_event",
-        "forwarded",
-        "mimecast-audit-events"
-    ],
     "user": {
         "domain": "example.com",
-        "email": "johndoe@example.com",
-        "name": "johndoe"
+        "name": "johdoe",
+        "email": "johndoe@example.com"
+    },
+    "mimecast": {
+        "eventInfo": "Threat intel multiple feeds download  - malware_customer_csv_20220209024500934.zip, Date: 2022-02-09, Time: 02:45:01+0000, IP: 8.8.8.8, Application: Integrations",
+        "application": "Integrations",
+        "category": "reporting_logs"
     }
 }

packages/mimecast/data_stream/dlp_logs/sample_event.json

@@ -1,13 +1,5 @@
 {
     "@timestamp": "2021-11-18T21:41:18.000Z",
-    "agent": {
-        "ephemeral_id": "1aef981f-3448-4d12-bd5a-723ac1cdcc81",
-        "hostname": "docker-fleet-agent",
-        "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab",
-        "name": "docker-fleet-agent",
-        "type": "filebeat",
-        "version": "7.16.0"
-    },
     "data_stream": {
         "dataset": "mimecast.dlp_logs",
         "namespace": "ep",