[mimecast] Add more use cases for parsing audit events

packages/mimecast/changelog.yml

@@ -1,3 +1,8 @@
+- version: "0.0.12"
+  changes:
+    - description: Add more use cases for parsing audit events.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3231
 - version: "0.0.11"
   changes:
     - description: Update integration description for consistency with other integrations.

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log

@@ -26,4 +26,6 @@
 {"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked","category":"authentication_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password","category":"authentication_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password","category":"authentication_logs"}
-{ "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI", "auditType": "User Logged On", "user": "johndoe@example.com", "eventTime": "2021-10-11T16:03:38+0000", "eventInfo": "Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15", "category": "authentication_logs"} 
+{ "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI", "auditType": "User Logged On", "user": "johndoe@example.com", "eventTime": "2021-10-11T16:03:38+0000", "eventInfo": "Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15", "category": "authentication_logs"} 
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo": "User johndoe@example.com attempted to access the mimecast-matfe but does not have the required permissions to do so, Date : 2022-03-29, Time : 13:31:03+0000, IP : 67.43.156.15, Application : API, Remote IP is 67.43.156.15","category":"authentication_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo": "Failed authentication for johndoe@example.com <John Doe>, Date: 2022-03-29, Time: 19:33:05 BST, IP: : 67.43.156.15,, Application: SMTP-MTA2, Reason: Account locked","category":"authentication_logs"}

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json

@@ -1373,16 +1373,137 @@
             "ecs": {
                 "version": "8.2.0"
             },
+            "client": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
             "event": {
                 "action": "user-logged-on",
                 "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI",
                 "original": "{ \"id\": \"eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI\", \"auditType\": \"User Logged On\", \"user\": \"johndoe@example.com\", \"eventTime\": \"2021-10-11T16:03:38+0000\", \"eventInfo\": \"Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15\", \"category\": \"authentication_logs\"} "
             },
             "mimecast": {
                 "category": "authentication_logs",
-                "eventInfo": "Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15"
+                "eventInfo": "Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15",
+                "remote": "Remote IP is 67.43.156.15",
+                "remote_ip": "67.43.156.15"
+            },
+            "related": {
+                "user": [
+                    "johndoe",
+                    "johndoe@example.com"
+                ],
+                "ip": [
+                    "67.43.156.15"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "domain": "example.com",
+                "email": "johndoe@example.com",
+                "name": "johndoe"
+            }
+        },
+        {
+            "@timestamp": "2021-10-12T08:47:55.000Z",
+            "client": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "action": "logon-authentication-failed",
+                "created": "2022-03-29T13:31:03.000Z",
+                "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg",
+                "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\": \"User johndoe@example.com attempted to access the mimecast-matfe but does not have the required permissions to do so, Date : 2022-03-29, Time : 13:31:03+0000, IP : 67.43.156.15, Application : API, Remote IP is 67.43.156.15\",\"category\":\"authentication_logs\"}"
+            },
+            "mimecast": {
+                "application": "API",
+                "category": "authentication_logs",
+                "eventInfo": "User johndoe@example.com attempted to access the mimecast-matfe but does not have the required permissions to do so, Date : 2022-03-29, Time : 13:31:03+0000, IP : 67.43.156.15, Application : API, Remote IP is 67.43.156.15",
+                "remote": "Remote IP is 67.43.156.15",
+                "remote_ip": "67.43.156.15"
+            },
+            "related": {
+                "ip": [
+                    "67.43.156.15"
+                ],
+                "user": [
+                    "johndoe",
+                    "johndoe@example.com"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "domain": "example.com",
+                "email": "johndoe@example.com",
+                "name": "johndoe"
+            }
+        },
+        {
+            "@timestamp": "2021-10-12T08:47:55.000Z",
+            "client": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "action": "logon-authentication-failed",
+                "created": "2022-03-29T19:33:05.000Z",
+                "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg",
+                "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\": \"Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-03-29, Time: 19:33:05 BST, IP: : 67.43.156.15,, Application: SMTP-MTA2, Reason: Account locked\",\"category\":\"authentication_logs\"}",
+                "reason": "Account locked"
+            },
+            "mimecast": {
+                "application": "SMTP-MTA2",
+                "category": "authentication_logs",
+                "eventInfo": "Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-03-29, Time: 19:33:05 BST, IP: : 67.43.156.15,, Application: SMTP-MTA2, Reason: Account locked"
             },
             "related": {
+                "ip": [
+                    "67.43.156.15"
+                ],
                 "user": [
                     "johndoe",
                     "johndoe@example.com"

packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml

@@ -64,9 +64,21 @@ processors:
   - dissect:
       field: mimecast.eventInfo
       pattern: "%{mimecast.info}, %{?key}:%{mimecast.email.address}[%{mimecast.email.metadata}] %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
-      if: 'ctx?.event?.action=="logon-authentication-failed"'
+      if: 'ctx.event?.action=="logon-authentication-failed"'
       ignore_missing: true
       ignore_failure: true
+  - dissect:
+       field: mimecast.eventInfo
+       pattern: "%{mimecast.info}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time} %{mimecast.timezone}, %{?key}: : %{client.ip},, %{?key}: %{mimecast.application}, %{?key}: %{event.reason}"
+       if: 'ctx.event?.action=="logon-authentication-failed"'
+       ignore_missing: true
+       ignore_failure: true
+  - dissect:
+       field: mimecast.eventInfo
+       pattern: "%{mimecast.info}, %{?key} : %{mimecast.date}, %{?key} : %{mimecast.time}, %{?key} : %{client.ip}, %{?key} : %{mimecast.application}, %{mimecast.remote}"
+       if: 'ctx.event?.action=="logon-authentication-failed"'
+       ignore_missing: true
+       ignore_failure: true
   - dissect:
       field: mimecast.eventInfo
       pattern: "%{mimecast.info}, %{mimecast.rest_of_event_info}"
@@ -76,7 +88,7 @@ processors:
   - dissect:
        field: mimecast.eventInfo
        pattern: "%{?drop->} - %{mimecast.info}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
-       if: 'ctx?.event?.action=="folder-log-entry" || ctx?.event?.action=="custom-report-definition-created" || ctx?.event?.action=="mimecast-support-login"'
+       if: 'ctx.event?.action=="folder-log-entry" || ctx.event?.action=="custom-report-definition-created" || ctx.event?.action=="mimecast-support-login"'
        ignore_missing: true
        ignore_failure: true
   - kv:
@@ -86,6 +98,15 @@ processors:
       target_field: mimecast.event_info_parts
       ignore_failure: true
       ignore_missing: true
+  - set:
+      field: mimecast.remote
+      value: "{{{mimecast.rest_of_event_info}}}"
+      if: 'ctx.event?.action=="user-logged-on" && ctx?.mimecast?.event_info_parts?.IP == null'
+  - grok:
+      field: mimecast.remote
+      patterns:
+        - "%{IP:mimecast.remote_ip}"
+      ignore_missing: true
   - rename:
       field: mimecast.event_info_parts.Date
       target_field: mimecast.date
@@ -114,7 +135,7 @@ processors:
       field: mimecast.info
       target_field: mimecast.filename
       ignore_missing: true
-      if: 'ctx?.event?.action == "threat-intel-feed-download"'
+      if: 'ctx.event?.action == "threat-intel-feed-download"'
   - rename: 
       field: mimecast.event_info_parts.Processed
       target_field: email.origination_timestamp
@@ -130,28 +151,28 @@ processors:
   - dissect:
        field:  mimecast.event_info_parts.From
        pattern: "<%{?drop}> %{email.from.address}"
-       if: 'ctx?.event?.action=="message-action"'
+       if: 'ctx.event?.action=="message-action"'
        ignore_missing: true
        ignore_failure: true
   - dissect:
        field: mimecast.event_info_parts.To
        pattern: "<%{?drop}> %{email.to.address}"
-       if: 'ctx?.event?.action=="message-action"'
+       if: 'ctx.event?.action=="message-action"'
        ignore_missing: true
        ignore_failure: true     
   - dissect:
       field: mimecast.eventInfo
       pattern: "[%{?key} : %{mimecast.export_type},%{?key} :%{mimecast.export_name},%{?key} :%{user.email},%{?key} :%{mimecast.weekday} %{mimecast.month} %{mimecast.monthday} %{mimecast.time} %{mimecast.timezone} %{mimecast.year},%{?key} :%{client.ip},%{?key} :%{mimecast.columns_exported},%{?key} : %{file.name},%{?key}: %{file.size},%{?key} : %{file.extension}], %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}"
-      if: 'ctx?.event?.action=="page-data-exports"'
+      if: 'ctx.event?.action=="page-data-exports"'
       ignore_missing: true
       ignore_failure: true
   - grok:
-      field: mimecast.eventInfo
+      field: mimecast.rest_of_event_info
       patterns:
-        - "%{IP:mimecast.event_info_parts.IP}"
+        - "%{IP:client.ip}"
       ignore_missing: true
       ignore_failure: true
-      if: 'ctx?.event?.action=="user-logged-on"'
+      if: 'ctx?.event?.action=="user-logged-on" && ctx?.mimecast?.event_info_parts?.IP == null'
   - set:
       field: email.from.address
       value: ["{{{email.from.address}}}"]

packages/mimecast/data_stream/audit_events/fields/field.yml

@@ -22,3 +22,9 @@
     - name: 2FA
       type: keyword
       description: Info about two-factor authentication.
+    - name: remote
+      type: keyword
+      description: Info about remote IP trying to access the API.
+    - name: remote_ip
+      type: keyword
+      description: Remote IP.

packages/mimecast/data_stream/audit_events/sample_event.json

@@ -1,12 +1,11 @@
 {
     "@timestamp": "2021-11-16T12:01:37.000Z",
     "agent": {
-        "ephemeral_id": "3126099e-107b-4959-b9e0-62ad3c5740ca",
-        "hostname": "docker-fleet-agent",
-        "id": "01800603-1f81-46c1-b412-764819259d1b",
+        "ephemeral_id": "7eb67cfe-6ef1-4f2e-890d-7aab6d531683",
+        "id": "fd4e8fc3-415d-4412-be0d-db9a9d3b6919",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.16.0"
+        "version": "8.1.0"
     },
     "data_stream": {
         "dataset": "mimecast.audit_events",
@@ -17,17 +16,17 @@
         "version": "8.2.0"
     },
     "elastic_agent": {
-        "id": "01800603-1f81-46c1-b412-764819259d1b",
-        "snapshot": true,
-        "version": "7.16.0"
+        "id": "fd4e8fc3-415d-4412-be0d-db9a9d3b6919",
+        "snapshot": false,
+        "version": "8.1.0"
     },
     "event": {
         "action": "search-action",
         "agent_id_status": "verified",
-        "created": "2022-04-21T08:23:36.847Z",
+        "created": "2022-05-05T09:45:04.710Z",
         "dataset": "mimecast.audit_events",
         "id": "eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o",
-        "ingested": "2022-04-21T08:23:37Z",
+        "ingested": "2022-05-05T09:45:05Z",
         "original": "{\"auditType\":\"Search Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o\",\"user\":\"johndoe@example.com\"}"
     },
     "input": {

packages/mimecast/data_stream/dlp_logs/sample_event.json

@@ -1,12 +1,11 @@
 {
     "@timestamp": "2021-11-18T21:41:18.000Z",
     "agent": {
-        "ephemeral_id": "f05546e4-1114-4375-9f2a-6a0b35c3c0f1",
-        "hostname": "docker-fleet-agent",
-        "id": "01800603-1f81-46c1-b412-764819259d1b",
+        "ephemeral_id": "8d23cacf-9a26-4c53-99fa-68de8661eb44",
+        "id": "fd4e8fc3-415d-4412-be0d-db9a9d3b6919",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.16.0"
+        "version": "8.1.0"
     },
     "data_stream": {
         "dataset": "mimecast.dlp_logs",
@@ -17,9 +16,9 @@
         "version": "8.2.0"
     },
     "elastic_agent": {
-        "id": "01800603-1f81-46c1-b412-764819259d1b",
-        "snapshot": true,
-        "version": "7.16.0"
+        "id": "fd4e8fc3-415d-4412-be0d-db9a9d3b6919",
+        "snapshot": false,
+        "version": "8.1.0"
     },
     "email": {
         "direction": "inbound",
@@ -41,7 +40,7 @@
         "agent_id_status": "verified",
         "created": "2021-11-18T21:41:18+0000",
         "dataset": "mimecast.dlp_logs",
-        "ingested": "2022-04-21T08:24:23Z",
+        "ingested": "2022-05-05T09:45:50Z",
         "original": "{\"action\":\"notification\",\"eventTime\":\"2021-11-18T21:41:18+0000\",\"messageId\":\"\\u003c20211118214115.B346F10021D@mail.emailsec.ninja\\u003e\",\"policy\":\"Content Inspection - Watermark\",\"recipientAddress\":\"johndoe@example.com\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Undelivered Mail Returned to Sender\"}"
     },
     "input": {