[mimecast] audit events - add additional email fields

packages/mimecast/changelog.yml

@@ -1,3 +1,8 @@
+- version: "0.0.10"
+  changes:
+    - description: Add more use cases to audit-events pipeline, implent geo.ip for siem logs, remove user part for ttp-url logs and add email.to.address for recipient
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2917
 - version: "0.0.9"
   changes:
     - description: Update ecs to version 8.2.0 and implement better practice for email ECS fields.
@@ -45,4 +50,4 @@
   changes:
     - description: Initial draft of the package
       type: enhancement
-      link: https://github.com/elastic/integrations/pull/2157
+      link: https://github.com/elastic/integrations/pull/2157

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log

@@ -24,4 +24,5 @@
 {"id":"eNqrVipOTS4tSs1MUbJSitH39gl1cS509PT1MSnw90l0CinPCQgLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsYmBsYqqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAxASul","auditType":"Review Set Action","user":"johndoe@example.com","eventTime":"2021-10-12T17:07:00+0000","eventInfo":"Viewed Review Set Details - Case: Class Action, Review Set: Contracts, Date: 2021-10-12, Time: 17:07:00+0000, IP: 67.43.156.15, Application: mimecast-case-review","category":"case_review_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38","auditType":"Remediation Incident Adjustment","user":"johndoe@example.com","eventTime":"2021-10-12T15:38:05+0000","eventInfo":"Restore Remediation Incident Created -  TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console","category":"account_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked","category":"authentication_logs"}
-{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password","category":"authentication_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password","category":"authentication_logs"}
+{ "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI", "auditType": "User Logged On", "user": "johndoe@example.com", "eventTime": "2021-10-11T16:03:38+0000", "eventInfo": "Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15", "category": "authentication_logs"}

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json

@@ -1316,6 +1316,53 @@
                 "email": "johndoe@example.com",
                 "name": "johndoe"
             }
+        },
+        {
+            "@timestamp": "2021-10-11T16:03:38.000Z",
+            "client": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "action": "user-logged-on",
+                "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI",
+                "original": "{ \"id\": \"eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI\", \"auditType\": \"User Logged On\", \"user\": \"johndoe@example.com\", \"eventTime\": \"2021-10-11T16:03:38+0000\", \"eventInfo\": \"Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15\", \"category\": \"authentication_logs\"}"
+            },
+            "mimecast": {
+                "category": "authentication_logs",
+                "eventInfo": "Succesfully enrolled user for user device enrollment, Remote IP is 67.43.156.15"
+            },
+            "related": {
+                "ip": [
+                    "67.43.156.15"
+                ],
+                "user": [
+                    "johndoe",
+                    "johndoe@example.com"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "domain": "example.com",
+                "email": "johndoe@example.com",
+                "name": "johndoe"
+            }
         }
     ]
 }

packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml

@@ -86,6 +86,13 @@ processors:
       target_field: mimecast.event_info_parts
       ignore_failure: true
       ignore_missing: true
+  - grok:
+      field: mimecast.eventInfo
+      patterns:
+        - "%{IP:mimecast.event_info_parts.IP}"
+      ignore_missing: true
+      ignore_failure: true
+      if: 'ctx?.event?.action=="user-logged-on"'
   - rename:
       field: mimecast.event_info_parts.Date
       target_field: mimecast.date
@@ -252,6 +259,9 @@ processors:
       value: "{{user.email}}"
       allow_duplicates: false
       if: ctx?.user?.email != null
+  - lowercase:
+      field: email.direction
+      ignore_missing: true
   # Cleanup
   - remove:
       description: Cleanup of repeated/unwanted/temporary fields.

packages/mimecast/data_stream/audit_events/sample_event.json

@@ -1,9 +1,9 @@
 {
     "@timestamp": "2021-11-16T12:01:37.000Z",
     "agent": {
-        "ephemeral_id": "b2833ed3-e047-442e-945f-291f7d6ace9d",
+        "ephemeral_id": "2be73539-59e1-4458-a099-3d97c4d3e261",
         "hostname": "docker-fleet-agent",
-        "id": "755dd553-bc87-4d8b-9736-61e8bbd15a3d",
+        "id": "4a72bf66-d0fb-4afd-8824-5fbe877771d8",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "7.16.0"
@@ -17,17 +17,17 @@
         "version": "8.2.0"
     },
     "elastic_agent": {
-        "id": "755dd553-bc87-4d8b-9736-61e8bbd15a3d",
+        "id": "4a72bf66-d0fb-4afd-8824-5fbe877771d8",
         "snapshot": true,
         "version": "7.16.0"
     },
     "event": {
         "action": "search-action",
         "agent_id_status": "verified",
-        "created": "2022-04-01T12:35:03.501Z",
+        "created": "2022-04-10T21:21:57.926Z",
         "dataset": "mimecast.audit_events",
         "id": "eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o",
-        "ingested": "2022-04-01T12:35:04Z",
+        "ingested": "2022-04-10T21:21:58Z",
         "original": "{\"auditType\":\"Search Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o\",\"user\":\"johndoe@example.com\"}"
     },
     "input": {

packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml

@@ -69,7 +69,9 @@ processors:
       field: event.created
       value: "{{mimecast.eventTime}}"
       if: 'ctx?.mimecast?.eventTime != null'
-
+  - lowercase:
+      field: email.direction
+      ignore_missing: true
   # Cleanup
   - remove:
       description: Cleanup of repeated/unwanted/temporary fields.

packages/mimecast/data_stream/dlp_logs/sample_event.json

@@ -1,9 +1,9 @@
 {
     "@timestamp": "2021-11-18T21:41:18.000Z",
     "agent": {
-        "ephemeral_id": "4a1c8c13-aee6-49b9-afc3-a2e62a310761",
+        "ephemeral_id": "8998934e-54d0-4749-82f4-be92de17c892",
         "hostname": "docker-fleet-agent",
-        "id": "755dd553-bc87-4d8b-9736-61e8bbd15a3d",
+        "id": "4a72bf66-d0fb-4afd-8824-5fbe877771d8",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "7.16.0"
@@ -17,7 +17,7 @@
         "version": "8.2.0"
     },
     "elastic_agent": {
-        "id": "755dd553-bc87-4d8b-9736-61e8bbd15a3d",
+        "id": "4a72bf66-d0fb-4afd-8824-5fbe877771d8",
         "snapshot": true,
         "version": "7.16.0"
     },
@@ -41,7 +41,7 @@
         "agent_id_status": "verified",
         "created": "2021-11-18T21:41:18+0000",
         "dataset": "mimecast.dlp_logs",
-        "ingested": "2022-04-01T12:35:41Z",
+        "ingested": "2022-04-10T21:22:42Z",
         "original": "{\"action\":\"notification\",\"eventTime\":\"2021-11-18T21:41:18+0000\",\"messageId\":\"\\u003c20211118214115.B346F10021D@mail.emailsec.ninja\\u003e\",\"policy\":\"Content Inspection - Watermark\",\"recipientAddress\":\"johndoe@example.com\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Undelivered Mail Returned to Sender\"}"
     },
     "input": {

packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log

@@ -3,5 +3,5 @@
 {"acc":"ABC123","Sender":"postmaster@twotoeight.com","datetime":"2021-10-19T07:04:55+0100","AttSize":0,"Content-Disposition":"attachment; filename=\"process_20211018093329655.json\"","Act":"Acc","aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","AttCnt":0,"AttNames":null,"MsgSize":49025,"MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":"You have new held messages"}
 {"acc":"ABC123","Delivered":true,"IP":"67.43.156.15","AttCnt":0,"Dir":"Internal","ReceiptAck":"250 SmtpInternalThread-19194240-1634623495703@uk-mta-151.uk.mimecast.lan Received OK [61dfe7da-4c6d-34e1-9667-69b04f0d564f.uk151]","MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":null,"Latency":1090,"Sender":"johndoe@example.com","datetime":"2021-10-19T07:04:55+0100","Rcpt":"johndoejr@example.com","AttSize":0,"Attempt":1,"Snt":51666,"aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","UseTls":"No", "Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""},{"acc":"ABC123","Delivered":false,"IP":"67.43.156.15","RejType":"Recipient email address is possibly incorrect","RejCode":"550","AttCnt":0,"Dir":"Internal","ReceiptAck":null,"MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":"You have new held messages","Latency":1534,"Sender":"johndoe@example.com","datetime":"2021-10-19T07:04:56+0100","Rcpt":"johndoejr@example.com","AttSize":0,"Attempt":1,"RejInfo":"5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]","TlsVer":"TLSv1.2","Cphr":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","Snt":147,"aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","UseTls":"Yes","Route":"Office365","Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""}
 {"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:09:18+0000","Rcpt":"o365_service_account@example.com","RcptActType":"Jnl","aCode":"CYSuuaBUMjOpk3k1Xhvy_Q","Dir":"Internal","RcptHdrType":"Unknown", "Content-Disposition":"attachment; filename=\"jrnl_20211018093329655.json\""}
-{"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:10:19+0000","Rcpt":"johndoejr@example.com","Act":"Acc","IP":"81.2.69.193","aCode":"3dbe9918-f91f-3043-b61f-d3164badfe50","Dir":"Internal","Subject":"You have new held messages","MsgId":"<140943948-1636373419265@uk-mta-286.uk.mimecast.lan>","headerFrom":"johndoe@example.com", "Content-Disposition":"attachment; filename=\"receipt_20211018093329655.json\""}
-{"acc":"C46A75","reason":"malicious","subject":"DocuSign- Contract #45576744333","msgid":null,"url":"http:\/\/docusign.swrodgods.x10.mx\/Docun\/Docu\/index2.php","datetime":"2021-11-29T15:13:58+0000","route":"inbound","sourceIp":"81.2.69.193","sender":"docusign-services@zenz.us","recipient":"aorchard@twotoeight.com","action":"Block","urlCategory":"Phishing & Fraud","credentialTheft":null,"senderDomain":"zenz.us", "Content-Disposition":"attachment; filename=\"ttp_url_20211129153015541.json\""}
+{"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:10:19+0000","Rcpt":"johndoejr@example.com","Act":"Acc","IP":"67.43.156.15","aCode":"3dbe9918-f91f-3043-b61f-d3164badfe50","Dir":"Internal","Subject":"You have new held messages","MsgId":"<140943948-1636373419265@uk-mta-286.uk.mimecast.lan>","headerFrom":"johndoe@example.com", "Content-Disposition":"attachment; filename=\"receipt_20211018093329655.json\""}
+{"acc":"C46A75","reason":"malicious","subject":"DocuSign- Contract #45576744333","msgid":null,"url":"http:\/\/docusign.swrodgods.x10.mx\/Docun\/Docu\/index2.php","datetime":"2021-11-29T15:13:58+0000","route":"inbound","sourceIp":"67.43.156.15","sender":"docusign-services@zenz.us","recipient":"aorchard@twotoeight.com","action":"Block","urlCategory":"Phishing & Fraud","credentialTheft":null,"senderDomain":"zenz.us", "Content-Disposition":"attachment; filename=\"ttp_url_20211129153015541.json\""}