[mimecast] Update ECS version to 8.2 with email fields

packages/mimecast/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@8.0
+    reference: git@8.2

packages/mimecast/changelog.yml

@@ -1,3 +1,8 @@
+- version: "0.0.9"
+  changes:
+    - description: Update ecs to version 8.2.0 and implement better practice for email ECS fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2841
 - version: "0.0.8"
   changes:
     - description: Add documentation for multi-fields

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json

@@ -18,7 +18,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "threat-intel-feed-download",
@@ -71,7 +71,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "threat-intel-feed-download",
@@ -124,7 +124,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "user-logged-on",
@@ -175,7 +175,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "logon-requires-challenge",
@@ -226,7 +226,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "user-logged-on",
@@ -276,7 +276,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "mimecast-support-login",
@@ -325,7 +325,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "mimecast-support-login",
@@ -374,16 +374,20 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "email": {
                 "from": {
-                    "address": "johndoe@example.com"
+                    "address": [
+                        "johndoe@example.com"
+                    ]
                 },
                 "origination_timestamp": "2021-09-28 07:59:23+0000",
                 "subject": "Test on Tues 28th Sept",
                 "to": {
-                    "address": "johndoe@example.com"
+                    "address": [
+                        "johndoe@example.com"
+                    ]
                 }
             },
             "event": {
@@ -433,7 +437,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "search-action",
@@ -482,7 +486,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "logon-authentication-failed",
@@ -519,7 +523,7 @@
         {
             "@timestamp": "2021-10-11T13:21:06.000Z",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "completed-directory-sync",
@@ -560,7 +564,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "case-action",
@@ -609,7 +613,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "logon-authentication-failed",
@@ -660,7 +664,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "existing-archive-task-changed",
@@ -709,7 +713,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "connectors-management",
@@ -758,7 +762,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "page-data-exports",
@@ -812,7 +816,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "custom-report-definition-created",
@@ -861,7 +865,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "folder-log-entry",
@@ -892,7 +896,7 @@
         {
             "@timestamp": "2021-10-12T19:56:55.000Z",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "user-password-changed",
@@ -936,7 +940,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "remediation-incident-adjustment",
@@ -985,7 +989,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "archive-mailbox-restore",
@@ -1034,7 +1038,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "archive-mailbox-restore",
@@ -1083,7 +1087,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "archive-mailbox-export-download",
@@ -1132,7 +1136,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "review-set-action",
@@ -1181,7 +1185,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "remediation-incident-adjustment",
@@ -1230,7 +1234,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "logon-authentication-failed",
@@ -1280,7 +1284,7 @@
                 "ip": "67.43.156.15"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "logon-authentication-failed",

packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml

@@ -4,7 +4,7 @@ processors:
   #  # Generic event/ecs fields we always want to populate
   - set:
       field: ecs.version
-      value: "8.0.0"
+      value: "8.2.0"
   - rename:
       field: message
       target_field: event.original
@@ -143,6 +143,26 @@ processors:
       field: mimecast.eventInfo
       pattern: "[%{?key} : %{mimecast.export_type},%{?key} :%{mimecast.export_name},%{?key} :%{user.email},%{?key} :%{mimecast.weekday} %{mimecast.month} %{mimecast.monthday} %{mimecast.time} %{mimecast.timezone} %{mimecast.year},%{?key} :%{client.ip},%{?key} :%{mimecast.columns_exported},%{?key} : %{file.name},%{?key}: %{file.size},%{?key} : %{file.extension}], %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}"
       if: 'ctx?.event?.action=="page-data-exports"'
+      ignore_missing: true
+      ignore_failure: true    
+  - set:
+      field: email.from.address
+      value: ["{{{email.from.address}}}"]
+      if: "ctx?.email?.from?.address != null"
+  - set:
+      field: email.to.address
+      value: ["{{{email.to.address}}}"]
+      if: "ctx?.email?.to?.address != null"
+  - dissect:
+       field: email.from.address
+       pattern: "<%{email.from.address}>"
+       ignore_missing: true
+       ignore_failure: true
+  - dissect:
+       field: email.to.address
+       pattern: "<%{email.to.address}>"
+       ignore_missing: true
+       ignore_failure: true     
   - convert:
       field: file.size
       type: long

packages/mimecast/data_stream/audit_events/sample_event.json

@@ -1,33 +1,33 @@
 {
     "@timestamp": "2021-11-16T12:01:37.000Z",
     "agent": {
-        "ephemeral_id": "fa35babb-45a8-4537-b7e9-037256a9d3e5",
+        "ephemeral_id": "b2833ed3-e047-442e-945f-291f7d6ace9d",
         "hostname": "docker-fleet-agent",
-        "id": "1e76e2b6-7664-4905-9a0b-11e1d4dc6fa9",
+        "id": "755dd553-bc87-4d8b-9736-61e8bbd15a3d",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "7.16.0"
     },
     "data_stream": {
         "dataset": "mimecast.audit_events",
         "namespace": "ep",
         "type": "logs"
     },
     "ecs": {
-        "version": "8.0.0"
+        "version": "8.2.0"
     },
     "elastic_agent": {
-        "id": "1e76e2b6-7664-4905-9a0b-11e1d4dc6fa9",
-        "snapshot": false,
-        "version": "7.17.0"
+        "id": "755dd553-bc87-4d8b-9736-61e8bbd15a3d",
+        "snapshot": true,
+        "version": "7.16.0"
     },
     "event": {
         "action": "search-action",
         "agent_id_status": "verified",
-        "created": "2022-02-22T15:33:36.764Z",
+        "created": "2022-04-01T12:35:03.501Z",
         "dataset": "mimecast.audit_events",
         "id": "eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o",
-        "ingested": "2022-02-22T15:33:37Z",
+        "ingested": "2022-04-01T12:35:04Z",
         "original": "{\"auditType\":\"Search Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o\",\"user\":\"johndoe@example.com\"}"
     },
     "input": {