elastic · marc-gr · Dec 9, 2021 · Nov 30, 2021 · Dec 2, 2021 · Dec 2, 2021
diff --git a/packages/mimecast/_dev/build/docs/README.md b/packages/mimecast/_dev/build/docs/README.md
@@ -2,67 +2,74 @@
 
 The Mimecast integration collects events from the Mimecast API.
 
+## Configuration
+
+Authorization parameters for the Mimecast API (`Application Key`, `Application ID`, `Access Key`, and `Secret Key`), should be provided by a Mimecast representative for this integration.
+Under `Advanced options` you can set the time interval between two API requests as well as the API URL. A Mimecast representative should also be able to give you with this information in case you need to change the defaults. 
+
+Note that rate limit quotas may require you to set up different credentials for the different available log types.
+
 ## Logs
 
-### AUDIT EVENTS
+### Audit Events
 
-This is the `mimecast.audit_events` dataset.
+This is the `mimecast.audit_events` dataset. These logs contain Mimecast audit events with the following details: audit type, event category and detailed information about the event. More information about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-audit-events/).
 
 {{event "audit_events"}}
 
 {{fields "audit_events"}}
 
-### DLP LOGS
+### DLP Logs
 
-This is the `mimecast.dlp_logs` dataset.
+This is the `mimecast.dlp_logs` dataset. These logs contain information about messages that triggered a DLP or Content Examination policy. More information about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-dlp-logs/). 
 
 {{event "dlp_logs"}}
 
 {{fields "dlp_logs"}}
 
-### SIEM LOGS
+### SIEM Logs
 
-This is the `mimecast.siem_logs` dataset.
+This is the `mimecast.siem_logs` dataset. These logs contain information about messages that contains MTA logs (MTA = message transfer agent) – all Inbound, outbound and internal messages. More about these logs [here](https://integrations.mimecast.com/documentation/tutorials/understanding-siem-logs/).
 
 {{event "siem_logs"}}
 
 {{fields "siem_logs"}}
 
-### TTP IMPERSONATION LOGS
+### TTP Impersonation Logs
 
-This is the `mimecast.ttp_ip_logs` dataset.
+This is the `mimecast.ttp_ip_logs` dataset. These logs contain information about messages containing information flagged by an Impersonation Protection configuration. Learn more about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-impersonation-protect-logs/). 
 
 {{event "ttp_ip_logs"}}
 
 {{fields "ttp_ip_logs"}}
 
-### TTP ATTACHMENT LOGS
+### TTP Attachment Logs
 
-This is the `mimecast.ttp_ap_logs` dataset.
+This is the `mimecast.ttp_ap_logs` dataset. These logs contain Mimecast TTP attachment protection logs with the following details: result of attachment analysis (if it is malicious or not etc.), date when file is released, sender and recipient address, filename and type, action triggered for the attachment, the route of the original email containing the attachment and details. Learn more about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-attachment-protection-logs/).
 
 {{event "ttp_ap_logs"}}
 
 {{fields "ttp_ap_logs"}}
 
-### TTP URL LOGS
+### TTP URL Logs
 
-This is the `mimecast.ttp_url_logs` dataset.
+This is the `mimecast.ttp_url_logs` dataset. These logs contain Mimecast TTP attachment protection logs with the following details: the category of the URL clicked, the email address of the user who clicked the link, the url clicked, the action taken by the user if user awareness was applied, the route of the email that contained the link, the action defined by the administrator for the URL, the date that the URL was clicked, url scan result, the action that was taken for the click, the description of the definition that triggered the URL to be rewritten by Mimecast, the action requested by the user, an array of components of the message where the URL was found. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-url-logs/). 
 
 {{event "ttp_url_logs"}}
 
 {{fields "ttp_url_logs"}}
 
-### THREAT INTEL FEED MALWARE CUSTOMER
+### Threat Intel Feed Malware: Customer
 
-This is the `mimecast.threat_intel_malware_customer` dataset.
+This is the `mimecast.threat_intel_malware_customer` dataset. These logs contain information about messages that return identified malware threats at a customer level. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). 
 
 {{event "threat_intel_malware_customer"}}
 
 {{fields "threat_intel_malware_customer"}}
 
-### THREAT INTEL FEED MALWARE GRID
+### Threat Intel Feed Malware: Grid
 
-This is the `mimecast.threat_intel_malware_grid` dataset.
+This is the `mimecast.threat_intel_malware_grid` dataset. These logs contain information about messages that return identified malware threats at a regional grid level. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). 
 
 {{event "threat_intel_malware_grid"}}
 

diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,5 +1,10 @@
 # newer versions go on top
 
+- version: "0.0.2"
+  changes:
+    - description: Tweaking the dashboards
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2289
 - version: "0.0.1"
   changes:
     - description: Initial draft of the package

diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log
@@ -3,4 +3,5 @@
 {"acc":"ABC123","Sender":"postmaster@twotoeight.com","datetime":"2021-10-19T07:04:55+0100","AttSize":0,"Content-Disposition":"attachment; filename=\"process_20211018093329655.json\"","Act":"Acc","aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","AttCnt":0,"AttNames":null,"MsgSize":49025,"MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":"You have new held messages"}
 {"acc":"ABC123","Delivered":true,"IP":"8.8.8.8","AttCnt":0,"Dir":"Internal","ReceiptAck":"250 SmtpInternalThread-19194240-1634623495703@uk-mta-151.uk.mimecast.lan Received OK [61dfe7da-4c6d-34e1-9667-69b04f0d564f.uk151]","MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":null,"Latency":1090,"Sender":"johndoe@example.com","datetime":"2021-10-19T07:04:55+0100","Rcpt":"johndoejr@example.com","AttSize":0,"Attempt":1,"Snt":51666,"aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","UseTls":"No", "Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""},{"acc":"ABC123","Delivered":false,"IP":"8.8.8.8","RejType":"Recipient email address is possibly incorrect","RejCode":"550","AttCnt":0,"Dir":"Internal","ReceiptAck":null,"MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":"You have new held messages","Latency":1534,"Sender":"johndoe@example.com","datetime":"2021-10-19T07:04:56+0100","Rcpt":"johndoejr@example.com","AttSize":0,"Attempt":1,"RejInfo":"5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]","TlsVer":"TLSv1.2","Cphr":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","Snt":147,"aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","UseTls":"Yes","Route":"Office365","Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""}
 {"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:09:18+0000","Rcpt":"o365_service_account@example.com","RcptActType":"Jnl","aCode":"CYSuuaBUMjOpk3k1Xhvy_Q","Dir":"Internal","RcptHdrType":"Unknown", "Content-Disposition":"attachment; filename=\"jrnl_20211018093329655.json\""}
-{"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:10:19+0000","Rcpt":"johndoejr@example.com","Act":"Acc","IP":"8.8.8.8","aCode":"3dbe9918-f91f-3043-b61f-d3164badfe50","Dir":"Internal","Subject":"You have new held messages","MsgId":"<140943948-1636373419265@uk-mta-286.uk.mimecast.lan>","headerFrom":"johndoe@example.com", "Content-Disposition":"attachment; filename=\"receipt_20211018093329655.json\""}
+{"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:10:19+0000","Rcpt":"johndoejr@example.com","Act":"Acc","IP":"8.8.8.8","aCode":"3dbe9918-f91f-3043-b61f-d3164badfe50","Dir":"Internal","Subject":"You have new held messages","MsgId":"<140943948-1636373419265@uk-mta-286.uk.mimecast.lan>","headerFrom":"johndoe@example.com", "Content-Disposition":"attachment; filename=\"receipt_20211018093329655.json\""}
+{"acc":"C46A75","reason":"malicious","subject":"DocuSign- Contract #45576744333","msgid":null,"url":"http:\/\/docusign.swrodgods.x10.mx\/Docun\/Docu\/index2.php","datetime":"2021-11-29T15:13:58+0000","route":"inbound","sourceIp":"52.100.141.34","sender":"docusign-services@zenz.us","recipient":"aorchard@twotoeight.com","action":"Block","urlCategory":"Phishing & Fraud","credentialTheft":null,"senderDomain":"zenz.us", "Content-Disposition":"attachment; filename=\"ttp_url_20211129153015541.json\""}
diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json
@@ -8,7 +8,7 @@
             "event": {
                 "reason": "Spm",
                 "action": "Hld",
-                "ingested": "2021-11-25T11:34:11.459620200Z",
+                "ingested": "2021-12-06T21:18:44.898227900Z",
                 "original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":null,\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Hld\":\"Spm\",\"MsgId\":\"\\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\\u003e\",\"MsgSize\":157436,\"Sender\":\"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu\",\"Subject\":\"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!\",\"aCode\":\"HhuwRf_AOcuJZINE2ZgcKw\",\"acc\":\"ABC123\",\"datetime\":\"2021-10-18T09:02:43+0100\"}",
                 "created": "2021-10-18T09:02:43+0100",
                 "outcome": "unknown"
@@ -61,7 +61,7 @@
             },
             "event": {
                 "reason": "5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]",
-                "ingested": "2021-11-25T11:34:11.459623100Z",
+                "ingested": "2021-12-06T21:18:44.898231300Z",
                 "original": "{\"acc\":\"ABC123\",\"Delivered\":false,\"IP\":\"8.8.8.8\",\"RejType\":\"Recipient email address is possibly incorrect\",\"RejCode\":\"550\",\"AttCnt\":0,\"Dir\":\"Inbound\",\"ReceiptAck\":null,\"MsgId\":null,\"Subject\":null,\"Latency\":505,\"Sender\":\"\u003c\u003e\",\"datetime\":\"2021-10-19T07:06:40+0100\",\"Rcpt\":\"johndoe@example.com\",\"AttSize\":0,\"Attempt\":1,\"RejInfo\":\"5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]\",\"TlsVer\":\"TLSv1.2\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"Snt\":125,\"aCode\":\"29be076e-44cd-354d-a7c2-083d4a312371\",\"UseTls\":\"Yes\",\"Route\":\"Office365\",\"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"}",
                 "created": "2021-10-19T07:06:40+0100",
                 "outcome": "failure"
@@ -97,7 +97,7 @@
             },
             "event": {
                 "action": "Acc",
-                "ingested": "2021-11-25T11:34:11.459624200Z",
+                "ingested": "2021-12-06T21:18:44.898232900Z",
                 "original": "{\"acc\":\"ABC123\",\"Sender\":\"postmaster@twotoeight.com\",\"datetime\":\"2021-10-19T07:04:55+0100\",\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Act\":\"Acc\",\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"AttCnt\":0,\"AttNames\":null,\"MsgSize\":49025,\"MsgId\":\"\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e\",\"Subject\":\"You have new held messages\"}",
                 "created": "2021-10-19T07:04:55+0100",
                 "outcome": "unknown"
@@ -137,7 +137,7 @@
                 "ip": "8.8.8.8"
             },
             "event": {
-                "ingested": "2021-11-25T11:34:11.459625200Z",
+                "ingested": "2021-12-06T21:18:44.898247800Z",
                 "original": "{\"acc\":\"ABC123\",\"Delivered\":true,\"IP\":\"8.8.8.8\",\"AttCnt\":0,\"Dir\":\"Internal\",\"ReceiptAck\":\"250 SmtpInternalThread-19194240-1634623495703@uk-mta-151.uk.mimecast.lan Received OK [61dfe7da-4c6d-34e1-9667-69b04f0d564f.uk151]\",\"MsgId\":\"\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e\",\"Subject\":null,\"Latency\":1090,\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-10-19T07:04:55+0100\",\"Rcpt\":\"johndoejr@example.com\",\"AttSize\":0,\"Attempt\":1,\"Snt\":51666,\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"UseTls\":\"No\", \"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"},{\"acc\":\"ABC123\",\"Delivered\":false,\"IP\":\"8.8.8.8\",\"RejType\":\"Recipient email address is possibly incorrect\",\"RejCode\":\"550\",\"AttCnt\":0,\"Dir\":\"Internal\",\"ReceiptAck\":null,\"MsgId\":\"\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e\",\"Subject\":\"You have new held messages\",\"Latency\":1534,\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-10-19T07:04:56+0100\",\"Rcpt\":\"johndoejr@example.com\",\"AttSize\":0,\"Attempt\":1,\"RejInfo\":\"5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]\",\"TlsVer\":\"TLSv1.2\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"Snt\":147,\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"UseTls\":\"Yes\",\"Route\":\"Office365\",\"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"}",
                 "created": "2021-10-19T07:04:55+0100",
                 "outcome": "success"
@@ -177,7 +177,7 @@
                 "version": "1.12.0"
             },
             "event": {
-                "ingested": "2021-11-25T11:34:11.459630600Z",
+                "ingested": "2021-12-06T21:18:44.898248900Z",
                 "original": "{\"acc\":\"C46A75\",\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-11-08T12:09:18+0000\",\"Rcpt\":\"o365_service_account@example.com\",\"RcptActType\":\"Jnl\",\"aCode\":\"CYSuuaBUMjOpk3k1Xhvy_Q\",\"Dir\":\"Internal\",\"RcptHdrType\":\"Unknown\", \"Content-Disposition\":\"attachment; filename=\\\"jrnl_20211018093329655.json\\\"\"}",
                 "created": "2021-11-08T12:09:18+0000",
                 "outcome": "unknown"
@@ -212,7 +212,7 @@
             },
             "event": {
                 "action": "Acc",
-                "ingested": "2021-11-25T11:34:11.459631700Z",
+                "ingested": "2021-12-06T21:18:44.898250100Z",
                 "original": "{\"acc\":\"C46A75\",\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-11-08T12:10:19+0000\",\"Rcpt\":\"johndoejr@example.com\",\"Act\":\"Acc\",\"IP\":\"8.8.8.8\",\"aCode\":\"3dbe9918-f91f-3043-b61f-d3164badfe50\",\"Dir\":\"Internal\",\"Subject\":\"You have new held messages\",\"MsgId\":\"\u003c140943948-1636373419265@uk-mta-286.uk.mimecast.lan\u003e\",\"headerFrom\":\"johndoe@example.com\", \"Content-Disposition\":\"attachment; filename=\\\"receipt_20211018093329655.json\\\"\"}",
                 "created": "2021-11-08T12:10:19+0000",
                 "outcome": "unknown"
@@ -237,6 +237,45 @@
                 "acc": "C46A75",
                 "log_type": "receipt"
             }
+        },
+        {
+            "@timestamp": "2021-11-29T15:13:58.000Z",
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "source": {
+                "domain": "zenz.us",
+                "ip": "52.100.141.34"
+            },
+            "event": {
+                "reason": "malicious",
+                "action": "Block",
+                "ingested": "2021-12-06T21:18:44.898251200Z",
+                "original": "{\"acc\":\"C46A75\",\"reason\":\"malicious\",\"subject\":\"DocuSign- Contract #45576744333\",\"msgid\":null,\"url\":\"http:\\/\\/docusign.swrodgods.x10.mx\\/Docun\\/Docu\\/index2.php\",\"datetime\":\"2021-11-29T15:13:58+0000\",\"route\":\"inbound\",\"sourceIp\":\"52.100.141.34\",\"sender\":\"docusign-services@zenz.us\",\"recipient\":\"aorchard@twotoeight.com\",\"action\":\"Block\",\"urlCategory\":\"Phishing \u0026 Fraud\",\"credentialTheft\":null,\"senderDomain\":\"zenz.us\", \"Content-Disposition\":\"attachment; filename=\\\"ttp_url_20211129153015541.json\\\"\"}",
+                "created": "2021-11-29T15:13:58+0000",
+                "outcome": "unknown"
+            },
+            "email": {
+                "from": {
+                    "address": "docusign-services@zenz.us"
+                },
+                "to": {
+                    "address": "aorchard@twotoeight.com"
+                },
+                "subject": "DocuSign- Contract #45576744333",
+                "direction": "inbound"
+            },
+            "url": {
+                "full": "http://docusign.swrodgods.x10.mx/Docun/Docu/index2.php"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "mimecast": {
+                "acc": "C46A75",
+                "log_type": "ttp_url",
+                "urlCategory": "Phishing \u0026 Fraud"
+            }
         }
     ]
 }
diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml
@@ -259,8 +259,31 @@ processors:
       if: 'ctx?.mimecast?.action !=null'
   - dissect:
       field: mimecast.Content-Disposition
-      pattern: "%{?drop->}=\"%{mimecast.log_type}_%{?drop->}"
+      pattern: "%{?drop->}=\"%{mimecast.log_type}.%{?drop->}"
       ignore_missing: true
+  - split:
+      field: mimecast.log_type
+      separator: "_"
+      target_field: mimecast.log_type_parts
+      if: 'ctx?.mimecast?.log_type != null'
+  - set:
+      field: mimecast.log_type_part1
+      copy_from: mimecast.log_type_parts.0
+      if: 'ctx?.mimecast?.log_type_parts !=null'
+  - set:
+      field: mimecast.log_type_part2
+      copy_from: mimecast.log_type_parts.1
+      if: 'ctx?.mimecast?.log_type_parts !=null'
+  - set:
+      field: mimecast.log_type
+      value: "{{mimecast.log_type_part1}}"
+      if: 'ctx?.mimecast?.log_type_part1 != "ttp"'
+      ignore_failure: true
+  - set:
+      field: mimecast.log_type
+      value: "{{mimecast.log_type_part1}}_{{mimecast.log_type_part2}}"
+      if: 'ctx?.mimecast?.log_type_part1 =="ttp"'
+      ignore_failure: true 
   - set:
       field: event.created
       value: "{{mimecast.datetime}}"
@@ -297,6 +320,9 @@ processors:
         - mimecast.eventTime
         - mimecast.Content-Disposition
         - mimecast.datetime
+        - mimecast.log_type_part1
+        - mimecast.log_type_part2
+        - mimecast.log_type_parts
       ignore_missing: true
   - remove:
       description: Remove 'mimecast.RecieptApk' if null
@@ -323,8 +349,16 @@ processors:
       field: event.original
       if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
       ignore_failure: true
-
-
+  - remove:
+      description: Remove 'mimecast.credentialTheft' if null
+      field: mimecast.credentialTheft
+      if: 'ctx?.mimecast?.credentialTheft == null'
+      ignore_missing: true    
+  - remove:
+      description: Remove 'mimecast.msgid' if null
+      field: mimecast.msgid
+      if: 'ctx?.mimecast?.msgid == null'
+      ignore_missing: true     
   ###
 
 # Error handling

diff --git a/packages/mimecast/data_stream/siem_logs/fields/field.yml b/packages/mimecast/data_stream/siem_logs/fields/field.yml
@@ -134,3 +134,6 @@
     - name: log_type
       type: keyword
       description: String to get type of SIEM log.
+    - name: msgid
+      type: keyword
+      description: The internet message id of the email.