From 8d3b9ad19f651ac4df645962d016d9ce34ead90a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 21:43:14 -0400 Subject: [PATCH 01/18] Regenerate expected files for formatting purposes --- .../test-audit-events.log-expected.json | 2 +- .../pipeline/test-siem-logs.log-expected.json | 18 +++++++++--------- .../test-ttp-url-logs.log-expected.json | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json index def4324e9bc..c9b22d04289 100644 --- a/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json +++ b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json @@ -1338,8 +1338,8 @@ "version": "8.2.0" }, "event": { - "created": "2022-01-11T21:48:01.000Z", "action": "logon-authentication-failed", + "created": "2022-01-11T21:48:01.000Z", "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg", "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password\",\"category\":\"authentication_logs\"}", "reason": "Wrong Password" diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json index 171a8c2430b..1ba296870df 100644 --- a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json @@ -81,7 +81,6 @@ "name": "Office365" }, "source": { - "ip": "67.43.156.15", "as": { "asn": 35908 }, @@ -93,7 +92,8 @@ "lat": 27.5, "lon": 90.5 } - } + }, + "ip": "67.43.156.15" }, "tags": [ "preserve_original_event" @@ -178,7 +178,6 @@ "log_type": "delivery" }, "source": { - "ip": "67.43.156.15", "as": { "asn": 35908 }, @@ -190,7 +189,8 @@ "lat": 27.5, "lon": 90.5 } - } + }, + "ip": "67.43.156.15" }, "tags": [ "preserve_original_event" @@ -262,7 +262,6 @@ "log_type": "receipt" }, "source": { - "ip": "67.43.156.15", "as": { "asn": 35908 }, @@ -274,7 +273,8 @@ "lat": 27.5, "lon": 90.5 } - } + }, + "ip": "67.43.156.15" }, "tags": [ "preserve_original_event" @@ -310,11 +310,10 @@ "urlCategory": "Phishing \u0026 Fraud" }, "source": { - "domain": "zenz.us", - "ip": "67.43.156.15", "as": { "asn": 35908 }, + "domain": "zenz.us", "geo": { "continent_name": "Asia", "country_iso_code": "BT", @@ -323,7 +322,8 @@ "lat": 27.5, "lon": 90.5 } - } + }, + "ip": "67.43.156.15" }, "tags": [ "preserve_original_event" diff --git a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json index a815f62d9be..69841dab1e9 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json +++ b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json @@ -12,13 +12,13 @@ "bestbuyinfo@emailinfo.bestbuy.com" ] }, + "message_id": "\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e", + "subject": "Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.", "to": { "address": [ "johndoe@example.com" ] - }, - "message_id": "\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e", - "subject": "Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more." + } }, "event": { "action": "Continue", @@ -75,13 +75,13 @@ "noreply@r.livingsocial.com" ] }, + "message_id": "\u003c803962655.28921622.1634393221485.JavaMail.rocketman@push-dispatcher65.sac1\u003e", + "subject": "Jump Pass + Mega Sale", "to": { "address": [ "johndoe@example.com" ] - }, - "message_id": "\u003c803962655.28921622.1634393221485.JavaMail.rocketman@push-dispatcher65.sac1\u003e", - "subject": "Jump Pass + Mega Sale" + } }, "event": { "action": "Continue", @@ -138,13 +138,13 @@ "nflshop.com@eml.nflshop.com" ] }, + "message_id": "\u003c28ad4be3-2d3a-491d-9aa7-a5a907123da1@ind1s01mta1115.xt.local\u003e", + "subject": "25% Off Tees to Give During Early Gifting Sale", "to": { "address": [ "johndoe@example.com" ] - }, - "message_id": "\u003c28ad4be3-2d3a-491d-9aa7-a5a907123da1@ind1s01mta1115.xt.local\u003e", - "subject": "25% Off Tees to Give During Early Gifting Sale" + } }, "event": { "action": "Continue", From 6e6b0eed748b9059994237447b853cb886743683 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 21:46:20 -0400 Subject: [PATCH 02/18] Sort ecs.yml files [git-generate] cd packages/mimecast for i in $(find . -name ecs.yml); do yq -i '. | sort_keys(..) | sort_by(.name)' $i; done --- .../data_stream/audit_events/fields/ecs.yml | 80 ++++----- .../data_stream/dlp_logs/fields/ecs.yml | 52 +++--- .../data_stream/siem_logs/fields/ecs.yml | 162 +++++++++--------- .../fields/ecs.yml | 20 +-- .../threat_intel_malware_grid/fields/ecs.yml | 20 +-- .../data_stream/ttp_ap_logs/fields/ecs.yml | 78 ++++----- .../data_stream/ttp_ip_logs/fields/ecs.yml | 58 +++---- .../data_stream/ttp_url_logs/fields/ecs.yml | 62 +++---- 8 files changed, 266 insertions(+), 266 deletions(-) diff --git a/packages/mimecast/data_stream/audit_events/fields/ecs.yml b/packages/mimecast/data_stream/audit_events/fields/ecs.yml index 1553bd395d4..341e09e8ada 100644 --- a/packages/mimecast/data_stream/audit_events/fields/ecs.yml +++ b/packages/mimecast/data_stream/audit_events/fields/ecs.yml @@ -1,25 +1,13 @@ +- description: Client ASN number. + name: client.as.asn + type: long - external: ecs - name: event.original -- external: ecs - name: event.action -- external: ecs - name: user.email -- external: ecs - name: event.id -- external: ecs - name: tags -- external: ecs - name: ecs.version -- external: ecs - name: client.ip -- external: ecs - name: file.name -- external: ecs - name: user.name -- external: ecs - name: user.domain + name: client.as.number - external: ecs - name: file.extension + name: client.as.organization.name +- description: Client Organization name. + name: client.as.organization_name + type: keyword - external: ecs name: client.geo.city_name - external: ecs @@ -36,37 +24,49 @@ name: client.geo.region_iso_code - external: ecs name: client.geo.region_name -- description: Client ASN number. - name: client.as.asn - type: long -- description: Client Organization name. - name: client.as.organization_name - type: keyword - external: ecs - name: client.as.number + name: client.ip - external: ecs - name: client.as.organization.name -- description: The email address(es) of the message recipient(s) - type: keyword - name: email.to.address + name: ecs.version - description: Stores the from email address from the RFC5322 From - header field. - type: keyword name: email.from.address -- description: A brief summary of the topic of the message type: keyword - name: email.subject +- description: The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. + name: email.origination_timestamp + type: date +- description: A brief summary of the topic of the message ignore_above: 1024 multi_fields: - - name: text - type: text + - default_field: false + name: text norms: false - default_field: false -- description: The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. - type: date - name: email.origination_timestamp + type: text + name: email.subject + type: keyword +- description: The email address(es) of the message recipient(s) + name: email.to.address + type: keyword +- external: ecs + name: event.action +- external: ecs + name: event.id +- external: ecs + name: event.original +- external: ecs + name: file.extension +- external: ecs + name: file.name - external: ecs name: file.size - external: ecs name: related.ip - external: ecs name: related.user +- external: ecs + name: tags +- external: ecs + name: user.domain +- external: ecs + name: user.email +- external: ecs + name: user.name diff --git a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml index b540179f326..546dbb73afa 100644 --- a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml @@ -1,36 +1,36 @@ - external: ecs - name: event.original -- external: ecs - name: event.action -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - type: wildcard - name: email.message_id - multi_fields: - - name: text - type: text - norms: false - default_field: false + name: ecs.version - description: Direction of the message based on the sending and receiving domains - type: keyword name: email.direction -- external: ecs - name: rule.name -- external: ecs - name: tags -- external: ecs - name: ecs.version -- description: The email address(es) of the message recipient(s) type: keyword - name: email.to.address - description: Stores the from email address from the RFC5322 From - header field. - type: keyword name: email.from.address -- description: A brief summary of the topic of the message type: keyword - name: email.subject - ignore_above: 1024 +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. multi_fields: - - name: text + - default_field: false + name: text + norms: false type: text + name: email.message_id + type: wildcard +- description: A brief summary of the topic of the message + ignore_above: 1024 + multi_fields: + - default_field: false + name: text norms: false - default_field: false + type: text + name: email.subject + type: keyword +- description: The email address(es) of the message recipient(s) + name: email.to.address + type: keyword +- external: ecs + name: event.action +- external: ecs + name: event.original +- external: ecs + name: rule.name +- external: ecs + name: tags diff --git a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml index d59d753c5cf..cd55b339d76 100644 --- a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml @@ -1,100 +1,96 @@ -- external: ecs - name: event.original -- external: ecs - name: event.action -- external: ecs - name: user.email -- external: ecs - name: event.id -- external: ecs - name: tags - external: ecs name: ecs.version -- description: Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the X-MS-Exchange-Organization-Network-Message-Id id). +- description: Attachment file extension, excluding the leading dot. + name: email.attachments.file.extension type: keyword - name: email.local_id -- external: ecs - name: event.action -- external: ecs - name: tls.cipher -- description: Direction of the message based on the sending and receiving domains. +- description: MIME type of the attachment file. + name: email.attachments.file.mime_type + type: keyword +- description: Name of the attachment file including the extension. + name: email.attachments.file.name + type: keyword +- description: Name of the attachment file including the extension. + name: email.attachments.file.name + type: keyword +- description: Attachment file size in bytes. + name: email.attachments.file.size + type: long +- description: MD5 hash of the file attachment. + name: email.attachments.hash.md5 + type: keyword +- description: SHA-1 hash of the file attachment. + name: email.attachments.hash.sha1 + type: keyword +- description: SHA-256 hash of the file attachment. + name: email.attachments.hash.sha256 type: keyword +- description: Direction of the message based on the sending and receiving domains. name: email.direction -- external: ecs - name: error.message -- external: ecs - name: source.ip + type: keyword +- description: Stores the from email address from the RFC5322 From - header field. + name: email.from.address + type: keyword +- description: The sender address found in the from header of the email. + name: email.header_from + type: keyword +- description: Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the X-MS-Exchange-Organization-Network-Message-Id id). + name: email.local_id + type: keyword - description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - type: wildcard - name: email.message_id multi_fields: - - name: text + - default_field: false + name: text + norms: false type: text + name: email.message_id + type: wildcard +- description: The total size of the email.The total size of the email. + name: email.message_size + type: long +- description: A brief summary of the topic of the message + ignore_above: 1024 + multi_fields: + - default_field: false + name: text norms: false - default_field: false -- description: The email address(es) of the message recipient(s). + type: text + name: email.subject type: keyword +- description: The email address(es) of the message recipient(s). name: email.to.address -- description: The sender address found in the from header of the email. type: keyword - name: email.header_from - external: ecs name: error.code - external: ecs - name: event.reason + name: error.message - external: ecs name: error.type -- description: Stores the from email address from the RFC5322 From - header field. - type: keyword - name: email.from.address -- description: A brief summary of the topic of the message - type: keyword - name: email.subject - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - external: ecs - name: tls.version -- description: Attachment file size in bytes. - type: long - name: email.attachments.file.size -- description: Name of the attachment file including the extension. - type: keyword - name: email.attachments.file.name -- description: The total size of the email.The total size of the email. - type: long - name: email.message_size + name: event.action - external: ecs - name: tls.established + name: event.action +- external: ecs + name: event.id +- external: ecs + name: event.original +- external: ecs + name: event.outcome +- external: ecs + name: event.reason - external: ecs name: rule.name -- description: Attachment file extension, excluding the leading dot. - type: keyword - name: email.attachments.file.extension -- description: MIME type of the attachment file. - type: keyword - name: email.attachments.file.mime_type +- description: Client ASN number. + name: source.as.asn + type: long - external: ecs - name: source.domain -- description: SHA-1 hash of the file attachment. - type: keyword - name: email.attachments.hash.sha1 -- description: SHA-256 hash of the file attachment. - type: keyword - name: email.attachments.hash.sha256 -- description: MD5 hash of the file attachment. - type: keyword - name: email.attachments.hash.md5 -- description: Name of the attachment file including the extension. - type: keyword - name: email.attachments.file.name + name: source.as.number - external: ecs - name: url.full + name: source.as.organization.name +- description: Client Organization name. + name: source.as.organization_name + type: keyword - external: ecs - name: event.outcome + name: source.domain - external: ecs name: source.geo.city_name - external: ecs @@ -111,13 +107,17 @@ name: source.geo.region_iso_code - external: ecs name: source.geo.region_name -- description: Client ASN number. - name: source.as.asn - type: long -- description: Client Organization name. - name: source.as.organization_name - type: keyword - external: ecs - name: source.as.number + name: source.ip - external: ecs - name: source.as.organization.name + name: tags +- external: ecs + name: tls.cipher +- external: ecs + name: tls.established +- external: ecs + name: tls.version +- external: ecs + name: url.full +- external: ecs + name: user.email diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml index fcb3df2855e..29e6f290f42 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml @@ -1,22 +1,22 @@ - external: ecs - name: message -- external: ecs - name: tags + name: ecs.version - external: ecs name: event.original - external: ecs - name: ecs.version + name: message - external: ecs - name: threat.indicator.type + name: related.hash - external: ecs - name: threat.indicator.first_seen + name: tags - external: ecs - name: threat.indicator.modified_at + name: threat.indicator.file.hash.md5 +- external: ecs + name: threat.indicator.file.hash.sha1 - external: ecs name: threat.indicator.file.hash.sha256 - external: ecs - name: threat.indicator.file.hash.sha1 + name: threat.indicator.first_seen - external: ecs - name: threat.indicator.file.hash.md5 + name: threat.indicator.modified_at - external: ecs - name: related.hash + name: threat.indicator.type diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml index fcb3df2855e..29e6f290f42 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml @@ -1,22 +1,22 @@ - external: ecs - name: message -- external: ecs - name: tags + name: ecs.version - external: ecs name: event.original - external: ecs - name: ecs.version + name: message - external: ecs - name: threat.indicator.type + name: related.hash - external: ecs - name: threat.indicator.first_seen + name: tags - external: ecs - name: threat.indicator.modified_at + name: threat.indicator.file.hash.md5 +- external: ecs + name: threat.indicator.file.hash.sha1 - external: ecs name: threat.indicator.file.hash.sha256 - external: ecs - name: threat.indicator.file.hash.sha1 + name: threat.indicator.first_seen - external: ecs - name: threat.indicator.file.hash.md5 + name: threat.indicator.modified_at - external: ecs - name: related.hash + name: threat.indicator.type diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml index d5cf859eb65..b9a208648b0 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml @@ -1,55 +1,55 @@ -- external: ecs - name: event.original -- external: ecs - name: event.action -- external: ecs - name: tags - external: ecs name: ecs.version -- external: ecs - name: event.action -- description: Direction of the message based on the sending and receiving domains +- description: Attachment file extension, excluding the leading dot. + name: email.attachments.file.extension + type: keyword +- description: MIME type of the attachment file. + name: email.attachments.file.mime_type type: keyword +- description: MIME type of the attachment file. + name: email.attachments.file.mime_type + type: keyword +- description: Name of the attachment file including the extension. + name: email.attachments.file.name + type: keyword +- description: File hash. + name: email.attachments.hash + type: keyword +- description: Direction of the message based on the sending and receiving domains name: email.direction -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - type: wildcard - name: email.message_id - multi_fields: - - name: text - type: text - norms: false - default_field: false -- description: The email address(es) of the message recipient(s) type: keyword - name: email.to.address - description: Stores the from email address from the RFC5322 From - header field. - type: keyword name: email.from.address -- description: A brief summary of the topic of the message type: keyword - name: email.subject - ignore_above: 1024 +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. multi_fields: - - name: text + - default_field: false + name: text + norms: false type: text + name: email.message_id + type: wildcard +- description: A brief summary of the topic of the message + ignore_above: 1024 + multi_fields: + - default_field: false + name: text norms: false - default_field: false -- description: Name of the attachment file including the extension. + type: text + name: email.subject type: keyword - name: email.attachments.file.name -- description: MIME type of the attachment file. +- description: The email address(es) of the message recipient(s) + name: email.to.address type: keyword - name: email.attachments.file.mime_type - external: ecs - name: rule.name -- description: File hash. - type: keyword - name: email.attachments.hash -- description: Attachment file extension, excluding the leading dot. - type: keyword - name: email.attachments.file.extension + name: event.action +- external: ecs + name: event.action +- external: ecs + name: event.original - external: ecs name: related.hash -- description: MIME type of the attachment file. - type: keyword - name: email.attachments.file.mime_type +- external: ecs + name: rule.name +- external: ecs + name: tags diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml index bee1ef94972..7ea68e09c51 100644 --- a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml @@ -1,43 +1,43 @@ -- external: ecs - name: event.original -- external: ecs - name: event.action -- external: ecs - name: tags - external: ecs name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: source.ip - description: Stores the from email address from the RFC5322 From - header field. - type: keyword name: email.from.address -- description: A brief summary of the topic of the message type: keyword - name: email.subject - ignore_above: 1024 +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. multi_fields: - - name: text + - default_field: false + name: text + norms: false type: text + name: email.message_id + type: wildcard +- description: A brief summary of the topic of the message + ignore_above: 1024 + multi_fields: + - default_field: false + name: text norms: false - default_field: false + type: text + name: email.subject + type: keyword +- description: The email address(es) of the message recipient(s) + name: email.to.address + type: keyword - external: ecs - name: rule.name + name: event.action +- external: ecs + name: event.action - external: ecs name: event.id -- description: The email address(es) of the message recipient(s) - type: keyword - name: email.to.address -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - type: wildcard - name: email.message_id - multi_fields: - - name: text - type: text - norms: false - default_field: false - external: ecs - name: source.domain + name: event.original - external: ecs name: related.ip +- external: ecs + name: rule.name +- external: ecs + name: source.domain +- external: ecs + name: source.ip +- external: ecs + name: tags diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml index 0680cea44d6..df289887b8f 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml @@ -1,48 +1,48 @@ -- external: ecs - name: event.original -- external: ecs - name: event.action -- external: ecs - name: tags - external: ecs name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: source.ip -- description: Stores the from email address from the RFC5322 From - header field. +- description: Direction of the message based on the sending and receiving domains + name: email.direction type: keyword +- description: Stores the from email address from the RFC5322 From - header field. name: email.from.address -- description: Stores the from email address to the RFC5322 From - header field. - type: keyword - name: email.to.address -- description: A brief summary of the topic of the message type: keyword - name: email.subject - ignore_above: 1024 +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. multi_fields: - - name: text - type: text + - default_field: false + name: text norms: false - default_field: false -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - type: wildcard + type: text name: email.message_id + type: wildcard +- description: A brief summary of the topic of the message + ignore_above: 1024 multi_fields: - - name: text - type: text + - default_field: false + name: text norms: false - default_field: false -- description: Direction of the message based on the sending and receiving domains + type: text + name: email.subject + type: keyword +- description: Stores the from email address to the RFC5322 From - header field. + name: email.to.address type: keyword - name: email.direction - external: ecs - name: rule.name + name: event.action - external: ecs - name: url.original + name: event.action - external: ecs - name: related.ip + name: event.original - external: ecs - name: user.email + name: related.ip - external: ecs name: related.user +- external: ecs + name: rule.name +- external: ecs + name: source.ip +- external: ecs + name: tags +- external: ecs + name: url.original +- external: ecs + name: user.email From 90821a7ee89c1983535b46905bac47f4d16e2698 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 21:58:04 -0400 Subject: [PATCH 03/18] audit_logs cleanup Use ECS fields for audit_logs. Remove unused source.as.{asn,organization_name}. --- .../elasticsearch/ingest_pipeline/default.yml | 4 +-- .../data_stream/audit_events/fields/ecs.yml | 32 ++++++------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml index 4f6a947c556..a993d47e247 100644 --- a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,7 @@ --- -description: Pipeline for processing sample logs +description: Pipeline for processing Mimecast audit_events. processors: - # # Generic event/ecs fields we always want to populate + # Generic event/ecs fields we always want to populate. - set: field: ecs.version value: "8.2.0" diff --git a/packages/mimecast/data_stream/audit_events/fields/ecs.yml b/packages/mimecast/data_stream/audit_events/fields/ecs.yml index 341e09e8ada..9105b62ec94 100644 --- a/packages/mimecast/data_stream/audit_events/fields/ecs.yml +++ b/packages/mimecast/data_stream/audit_events/fields/ecs.yml @@ -1,13 +1,7 @@ -- description: Client ASN number. - name: client.as.asn - type: long - external: ecs name: client.as.number - external: ecs name: client.as.organization.name -- description: Client Organization name. - name: client.as.organization_name - type: keyword - external: ecs name: client.geo.city_name - external: ecs @@ -16,10 +10,8 @@ name: client.geo.country_iso_code - external: ecs name: client.geo.country_name -- description: Longitude and latitude. - level: core +- external: ecs name: client.geo.location - type: geo_point - external: ecs name: client.geo.region_iso_code - external: ecs @@ -28,30 +20,24 @@ name: client.ip - external: ecs name: ecs.version -- description: Stores the from email address from the RFC5322 From - header field. +- external: ecs name: email.from.address - type: keyword -- description: The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. +- external: ecs name: email.origination_timestamp - type: date -- description: A brief summary of the topic of the message - ignore_above: 1024 - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.subject - type: keyword -- description: The email address(es) of the message recipient(s) +- external: ecs name: email.to.address - type: keyword - external: ecs name: event.action +- external: ecs + name: event.created - external: ecs name: event.id - external: ecs name: event.original +- external: ecs + name: event.reason - external: ecs name: file.extension - external: ecs From 396dec8d4351b1e9cc06e74c4e41aa824ffd677e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 21:59:00 -0400 Subject: [PATCH 04/18] dlp_logs cleanup Use ECS fields for dlp_logs. None of the mimecast.* fields are used so remove the mappings. --- .../elasticsearch/ingest_pipeline/default.yml | 4 +-- .../data_stream/dlp_logs/fields/ecs.yml | 28 +++++-------------- .../data_stream/dlp_logs/fields/field.yml | 24 ---------------- 3 files changed, 9 insertions(+), 47 deletions(-) delete mode 100644 packages/mimecast/data_stream/dlp_logs/fields/field.yml diff --git a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml index 4115c5f27fe..df5a832b743 100644 --- a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,7 @@ --- -description: Pipeline for processing sample logs +description: Pipeline for processing Mimecast dlp_logs. processors: - # Generic event/ecs fields we always want to populated + # Generic event/ecs fields we always want to populate. - set: field: ecs.version value: "8.2.0" diff --git a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml index 546dbb73afa..ef925714f24 100644 --- a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml @@ -1,33 +1,19 @@ - external: ecs name: ecs.version -- description: Direction of the message based on the sending and receiving domains +- external: ecs name: email.direction - type: keyword -- description: Stores the from email address from the RFC5322 From - header field. +- external: ecs name: email.from.address - type: keyword -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.message_id - type: wildcard -- description: A brief summary of the topic of the message - ignore_above: 1024 - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.subject - type: keyword -- description: The email address(es) of the message recipient(s) +- external: ecs name: email.to.address - type: keyword - external: ecs name: event.action +- external: ecs + name: event.created - external: ecs name: event.original - external: ecs diff --git a/packages/mimecast/data_stream/dlp_logs/fields/field.yml b/packages/mimecast/data_stream/dlp_logs/fields/field.yml deleted file mode 100644 index 36a1bbebc9c..00000000000 --- a/packages/mimecast/data_stream/dlp_logs/fields/field.yml +++ /dev/null @@ -1,24 +0,0 @@ -- name: mimecast - type: group - fields: - - name: senderAddress - type: keyword - description: Email address of the sender. - - name: action - type: keyword - description: The action taken against the message. - - name: messageId - type: keyword - description: The message-id value of the message. - - name: subject - type: keyword - description: The message subject. - - name: route - type: keyword - description: The message direction. Possible values are inbound, outbound or internal. - - name: policy - type: keyword - description: The name of a DLP or Content Examination configuration that triggered the message. - - name: recipientAddress - type: keyword - description: Email address of the recipient. From 8273b3886019987e177b4f985b0ab0ced23a835e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:00:22 -0400 Subject: [PATCH 05/18] siem_logs cleanup Use ECS fields in siem_logs. Remove email.message_size and email.header_from which are not part of ECS. Use mimecast.MsgSize and email.from.address instead. Remove source.as.asn and source.as.organization_name and use the correct ECS fields. --- .../pipeline/test-siem-logs.log-expected.json | 14 ++-- .../elasticsearch/ingest_pipeline/default.yml | 35 +++++---- .../data_stream/siem_logs/fields/ecs.yml | 77 +++++-------------- 3 files changed, 46 insertions(+), 80 deletions(-) diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json index 1ba296870df..23e54d40c8a 100644 --- a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json @@ -18,7 +18,6 @@ }, "local_id": "HhuwRf_AOcuJZINE2ZgcKw", "message_id": "\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\u003e", - "message_size": 157436, "subject": "Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!" }, "event": { @@ -30,6 +29,7 @@ }, "mimecast": { "AttCnt": 0, + "MsgSize": 157436, "acc": "ABC123", "log_type": "process" }, @@ -82,7 +82,7 @@ }, "source": { "as": { - "asn": 35908 + "number": 35908 }, "geo": { "continent_name": "Asia", @@ -122,7 +122,6 @@ }, "local_id": "61dfe7da-4c6d-34e1-9667-69b04f0d564f", "message_id": "\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e", - "message_size": 49025, "subject": "You have new held messages" }, "event": { @@ -133,6 +132,7 @@ }, "mimecast": { "AttCnt": 0, + "MsgSize": 49025, "acc": "ABC123", "log_type": "process" }, @@ -179,7 +179,7 @@ }, "source": { "as": { - "asn": 35908 + "number": 35908 }, "geo": { "continent_name": "Asia", @@ -240,10 +240,10 @@ "direction": "internal", "from": { "address": [ + "johndoe@example.com", "johndoe@example.com" ] }, - "header_from": "johndoe@example.com", "local_id": "3dbe9918-f91f-3043-b61f-d3164badfe50", "message_id": "\u003c140943948-1636373419265@uk-mta-286.uk.mimecast.lan\u003e", "subject": "You have new held messages", @@ -263,7 +263,7 @@ }, "source": { "as": { - "asn": 35908 + "number": 35908 }, "geo": { "continent_name": "Asia", @@ -311,7 +311,7 @@ }, "source": { "as": { - "asn": 35908 + "number": 35908 }, "domain": "zenz.us", "geo": { diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml index 174e2260c20..12584402652 100644 --- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,7 @@ --- -description: Pipeline for processing sample logs +description: Pipeline for processing Mimecast siem_logs. processors: - # Generic event/ecs fields we always want to populated + # Generic event/ecs fields we always want to populate. - set: field: ecs.version value: "8.2.0" @@ -66,11 +66,10 @@ processors: field: email.to.address value: "{{{mimecast.Rcpt}}}" if: "ctx?.mimecast?.Rcpt != null" - - rename: - field: mimecast.headerFrom - target_field: email.header_from - ignore_missing: true - if: 'ctx?.mimecast?.headerFrom !=null' + - append: + field: email.from.address + value: '{{{mimecast.headerFrom}}}' + if: ctx.mimecast?.headerFrom != null - rename: field: mimecast.RejCode target_field: error.code @@ -116,12 +115,7 @@ processors: target_field: event.reason ignore_missing: true if: 'ctx?.mimecast?.Hld !=null' - - rename: - field: mimecast.MsgSize - target_field: email.message_size - ignore_missing: true - if: 'ctx?.mimecast?.MsgSize !=null' - ### DELIVERY LOGS + ### DELIVERY LOGS - rename: field: mimecast.Err target_field: error.message @@ -150,7 +144,7 @@ processors: if: 'ctx?.mimecast?.fileMime !=null' - rename: field: mimecast.md5 - target_field: email.attachments.hash.md5 + target_field: email.attachments.file.hash.md5 ignore_missing: true if: 'ctx?.mimecast?.md5 !=null' - rename: @@ -169,12 +163,12 @@ processors: if: 'ctx?.mimecast?.SenderDomain !=null' - rename: field: mimecast.sha1 - target_field: email.attachments.hash.sha1 + target_field: email.attachments.file.hash.sha1 ignore_missing: true if: 'ctx?.mimecast?.sha1 !=null' - rename: field: mimecast.sha256 - target_field: email.attachments.hash.sha256 + target_field: email.attachments.file.hash.sha256 ignore_missing: true if: 'ctx?.mimecast?.sha256 !=null' - rename: @@ -335,6 +329,14 @@ processors: - asn - organization_name ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true - dissect: field: email.from.address pattern: "<%{email.from.address}>" @@ -353,6 +355,7 @@ processors: - mimecast.eventTime - mimecast.Content-Disposition - mimecast.datetime + - mimecast.headerFrom - mimecast.log_type_part1 - mimecast.log_type_part2 - mimecast.log_type_parts diff --git a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml index cd55b339d76..31577dc1b52 100644 --- a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml @@ -1,64 +1,33 @@ - external: ecs name: ecs.version -- description: Attachment file extension, excluding the leading dot. +- external: ecs name: email.attachments.file.extension - type: keyword -- description: MIME type of the attachment file. +- external: ecs + name: email.attachments.file.hash.md5 +- external: ecs + name: email.attachments.file.hash.sha1 +- external: ecs + name: email.attachments.file.hash.sha256 +- external: ecs name: email.attachments.file.mime_type - type: keyword -- description: Name of the attachment file including the extension. +- external: ecs name: email.attachments.file.name - type: keyword -- description: Name of the attachment file including the extension. +- external: ecs name: email.attachments.file.name - type: keyword -- description: Attachment file size in bytes. +- external: ecs name: email.attachments.file.size - type: long -- description: MD5 hash of the file attachment. - name: email.attachments.hash.md5 - type: keyword -- description: SHA-1 hash of the file attachment. - name: email.attachments.hash.sha1 - type: keyword -- description: SHA-256 hash of the file attachment. - name: email.attachments.hash.sha256 - type: keyword -- description: Direction of the message based on the sending and receiving domains. +- external: ecs name: email.direction - type: keyword -- description: Stores the from email address from the RFC5322 From - header field. +- external: ecs name: email.from.address - type: keyword -- description: The sender address found in the from header of the email. - name: email.header_from - type: keyword -- description: Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the X-MS-Exchange-Organization-Network-Message-Id id). +- external: ecs name: email.local_id - type: keyword -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.message_id - type: wildcard -- description: The total size of the email.The total size of the email. - name: email.message_size - type: long -- description: A brief summary of the topic of the message - ignore_above: 1024 - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.subject - type: keyword -- description: The email address(es) of the message recipient(s). +- external: ecs name: email.to.address - type: keyword - external: ecs name: error.code - external: ecs @@ -69,6 +38,8 @@ name: event.action - external: ecs name: event.action +- external: ecs + name: event.created - external: ecs name: event.id - external: ecs @@ -79,16 +50,10 @@ name: event.reason - external: ecs name: rule.name -- description: Client ASN number. - name: source.as.asn - type: long - external: ecs name: source.as.number - external: ecs name: source.as.organization.name -- description: Client Organization name. - name: source.as.organization_name - type: keyword - external: ecs name: source.domain - external: ecs @@ -99,10 +64,8 @@ name: source.geo.country_iso_code - external: ecs name: source.geo.country_name -- description: Longitude and latitude. - level: core +- external: ecs name: source.geo.location - type: geo_point - external: ecs name: source.geo.region_iso_code - external: ecs From 0283a43686e0151e2921d2cb0a7b15778c3bf737 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:01:28 -0400 Subject: [PATCH 06/18] threat_intel_malware_customer cleanup Add missing ECS event fields. --- .../threat_intel_malware_customer/fields/ecs.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml index 29e6f290f42..3c764373326 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml @@ -1,7 +1,15 @@ - external: ecs name: ecs.version +- external: ecs + name: event.category +- external: ecs + name: event.created +- external: ecs + name: event.kind - external: ecs name: event.original +- external: ecs + name: event.type - external: ecs name: message - external: ecs From 4420140816c4d7907ab41b92583e09ea5b54453b Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:02:23 -0400 Subject: [PATCH 07/18] threat_intel_malware_grid cleanup Add missing ECS event field mappings. --- .../data_stream/threat_intel_malware_grid/fields/ecs.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml index 29e6f290f42..3c764373326 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml @@ -1,7 +1,15 @@ - external: ecs name: ecs.version +- external: ecs + name: event.category +- external: ecs + name: event.created +- external: ecs + name: event.kind - external: ecs name: event.original +- external: ecs + name: event.type - external: ecs name: message - external: ecs From 75aac87eb1b6343694808f54b28678da8015cc3b Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:03:39 -0400 Subject: [PATCH 08/18] ttp_ap_logs cleanup Use ECS field definitions for ttp_ap_logs. It was using email.attachments.hash which is not a valid ECS field so I changed it to use email.attachments.file.hash.sha256. --- .../elasticsearch/ingest_pipeline/default.yml | 12 ++--- .../data_stream/ttp_ap_logs/fields/ecs.yml | 45 ++++++------------- .../data_stream/ttp_ap_logs/sample_event.json | 6 ++- 3 files changed, 23 insertions(+), 40 deletions(-) diff --git a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml index 331ef2acc52..0e3a8902ab3 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,7 @@ --- -description: Pipeline for processing sample logs +description: Pipeline for processing Mimecast ttp_ap_logs. processors: - # Generic event/ecs fields we always want to populated + # Generic event/ecs fields we always want to populate. - set: field: ecs.version value: "8.2.0" @@ -82,9 +82,9 @@ processors: if: 'ctx?.mimecast?.definition !=null' - rename: field: mimecast.fileHash - target_field: email.attachments.hash + target_field: email.attachments.file.hash.sha256 ignore_missing: true - if: 'ctx?.mimecast?.fileHash !=null' + if: 'ctx.mimecast?.fileHash != null && ctx.mimecast.fileHash.length() == 64' - rename: field: mimecast.fileType target_field: email.attachments.file.mime_type @@ -106,9 +106,9 @@ processors: if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' - append: field: related.hash - value: "{{email.attachments.hash}}" + value: "{{{email.attachments.file.hash.sha256}}}" allow_duplicates: false - if: 'ctx?.email?.attachments?.hash !=null' + if: ctx.email?.attachments?.file?.hash?.sha256 != null - lowercase: field: email.direction ignore_missing: true diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml index b9a208648b0..8c473b28e22 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml @@ -1,50 +1,31 @@ - external: ecs name: ecs.version -- description: Attachment file extension, excluding the leading dot. +- external: ecs name: email.attachments.file.extension - type: keyword -- description: MIME type of the attachment file. +- external: ecs + name: email.attachments.file.hash.sha256 +- external: ecs name: email.attachments.file.mime_type - type: keyword -- description: MIME type of the attachment file. +- external: ecs name: email.attachments.file.mime_type - type: keyword -- description: Name of the attachment file including the extension. +- external: ecs name: email.attachments.file.name - type: keyword -- description: File hash. - name: email.attachments.hash - type: keyword -- description: Direction of the message based on the sending and receiving domains +- external: ecs name: email.direction - type: keyword -- description: Stores the from email address from the RFC5322 From - header field. +- external: ecs name: email.from.address - type: keyword -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.message_id - type: wildcard -- description: A brief summary of the topic of the message - ignore_above: 1024 - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.subject - type: keyword -- description: The email address(es) of the message recipient(s) +- external: ecs name: email.to.address - type: keyword - external: ecs name: event.action - external: ecs name: event.action +- external: ecs + name: event.created - external: ecs name: event.original - external: ecs diff --git a/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json b/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json index 80a9cab0195..03c2296eb48 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json +++ b/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json @@ -25,10 +25,12 @@ "attachments": { "file": { "extension": "pdf", + "hash": { + "sha256": "cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254" + }, "mime_type": "application/pdf", "name": "Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf" - }, - "hash": "cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254" + } }, "direction": "inbound", "from": { From f43d900c81f30d190edddbf92f3d6eb9d01c156a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:04:34 -0400 Subject: [PATCH 09/18] Generate ttp_ap_logs [git-generate] cd packages/mimecast elastic-package test pipeline -g -d=ttp_ap_logs --- .../test-ttp-ap-logs.log-expected.json | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json index 710daa5a44e..7826ba1046d 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json +++ b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json @@ -9,10 +9,12 @@ "attachments": { "file": { "extension": "pdf", + "hash": { + "sha256": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3" + }, "mime_type": "application/pdf", "name": "numbers.pdf" - }, - "hash": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3" + } }, "direction": "inbound", "from": { @@ -58,10 +60,12 @@ "attachments": { "file": { "extension": "docx", + "hash": { + "sha256": "2fb26be55ac710e4d9f80677ba24ae212dbb36bd934a0569fe521839e9f5d16e" + }, "mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "name": "Titus-Test Doc - Classification - InternalUseOnly.docx" - }, - "hash": "2fb26be55ac710e4d9f80677ba24ae212dbb36bd934a0569fe521839e9f5d16e" + } }, "direction": "inbound", "from": { @@ -107,10 +111,12 @@ "attachments": { "file": { "extension": "pptx", + "hash": { + "sha256": "111b86e1244ce6389efb60cddc001d594d334c540e85f9976be467a4ce472973" + }, "mime_type": "application/vnd.openxmlformats-officedocument.presentationml", "name": "Titus classification v0.3.pptx" - }, - "hash": "111b86e1244ce6389efb60cddc001d594d334c540e85f9976be467a4ce472973" + } }, "direction": "inbound", "from": { From 7a8bde0ffd457f27a1dc1c0cd84f9e037e2a5a8b Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:05:26 -0400 Subject: [PATCH 10/18] ttp_ip_logs cleanup Use ECS field definitions for ttp_ip_logs. Add missing event.created mapping. --- .../elasticsearch/ingest_pipeline/default.yml | 4 +-- .../data_stream/ttp_ip_logs/fields/ecs.yml | 25 +++++-------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml index 436f2bb768d..63eb1c2720e 100644 --- a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,7 @@ --- -description: Pipeline for processing sample logs +description: Pipeline for processing Mimecast ttp_ip_logs. processors: - # Generic event/ecs fields we always want to populated + # Generic event/ecs fields we always want to populate. - set: field: ecs.version value: "8.2.0" diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml index 7ea68e09c51..9a1770633fc 100644 --- a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml @@ -1,32 +1,19 @@ - external: ecs name: ecs.version -- description: Stores the from email address from the RFC5322 From - header field. +- external: ecs name: email.from.address - type: keyword -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.message_id - type: wildcard -- description: A brief summary of the topic of the message - ignore_above: 1024 - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.subject - type: keyword -- description: The email address(es) of the message recipient(s) +- external: ecs name: email.to.address - type: keyword - external: ecs name: event.action - external: ecs name: event.action +- external: ecs + name: event.created - external: ecs name: event.id - external: ecs From 9e240ff6d8d098fa0efe6c9836ce44f9cc16f7ea Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:06:17 -0400 Subject: [PATCH 11/18] ttp_url_logs cleanup Use ECS fields for ttp_url_logs. Add missing event.created mapping. --- .../elasticsearch/ingest_pipeline/default.yml | 4 +-- .../data_stream/ttp_url_logs/fields/ecs.yml | 28 +++++-------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml index eb2f6753b58..b0e2a979de4 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,7 @@ --- -description: Pipeline for processing sample logs +description: Pipeline for processing Mimecast ttp_url_logs. processors: - # Generic event/ecs fields we always want to populated + # Generic event/ecs fields we always want to populate. - set: field: ecs.version value: "8.2.0" diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml index df289887b8f..622f81b6fc7 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml @@ -1,35 +1,21 @@ - external: ecs name: ecs.version -- description: Direction of the message based on the sending and receiving domains +- external: ecs name: email.direction - type: keyword -- description: Stores the from email address from the RFC5322 From - header field. +- external: ecs name: email.from.address - type: keyword -- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.message_id - type: wildcard -- description: A brief summary of the topic of the message - ignore_above: 1024 - multi_fields: - - default_field: false - name: text - norms: false - type: text +- external: ecs name: email.subject - type: keyword -- description: Stores the from email address to the RFC5322 From - header field. +- external: ecs name: email.to.address - type: keyword - external: ecs name: event.action - external: ecs name: event.action +- external: ecs + name: event.created - external: ecs name: event.original - external: ecs From d7d42e40d6d3401aa134fc617fdd7eb3ce4aa2a1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:07:05 -0400 Subject: [PATCH 12/18] Make integration description consistent --- packages/mimecast/manifest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml index 6b27772d972..d7b0ac3a01e 100644 --- a/packages/mimecast/manifest.yml +++ b/packages/mimecast/manifest.yml @@ -3,7 +3,7 @@ name: mimecast title: "Mimecast" version: 0.0.10 license: basic -description: "Fetching logs from Mimecast API and ingest into Elasticsearch" +description: "Collect logs from the Mimecast API with Elastic Agent." type: integration categories: - security @@ -23,7 +23,7 @@ icons: policy_templates: - name: mimecast title: Mimecast - description: Mimecast Integration + description: Collect logs from the Mimecast API with Elastic Agent. inputs: - type: httpjson title: Mimecast API From c30859eedac90d0ad852e470b8f03483c3362790 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:07:48 -0400 Subject: [PATCH 13/18] Update dashboard field name And fix a typo in mime_type. --- .../search/mimecast-9749a210-3e4a-11ec-80fa-4dfb04910642.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mimecast/kibana/search/mimecast-9749a210-3e4a-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/search/mimecast-9749a210-3e4a-11ec-80fa-4dfb04910642.json index 2f0fc939366..1aa4d2e5079 100644 --- a/packages/mimecast/kibana/search/mimecast-9749a210-3e4a-11ec-80fa-4dfb04910642.json +++ b/packages/mimecast/kibana/search/mimecast-9749a210-3e4a-11ec-80fa-4dfb04910642.json @@ -3,8 +3,8 @@ "columns": [ "@timestamp", "email.attachments.file.extension", - "email.attachments.file.myme_type", - "email.attachments.hash", + "email.attachments.file.mime_type", + "email.attachments.file.hash.sha256", "email.attachments.file.name" ], "description": "", From 142d6b0ebe2347aa425c541c9296880d03426945 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:26:25 -0400 Subject: [PATCH 14/18] Format and sort readme sections --- packages/mimecast/_dev/build/docs/README.md | 90 +++++++++++++++------ 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/packages/mimecast/_dev/build/docs/README.md b/packages/mimecast/_dev/build/docs/README.md index 34cea0a03dd..bb75b76e96d 100644 --- a/packages/mimecast/_dev/build/docs/README.md +++ b/packages/mimecast/_dev/build/docs/README.md @@ -4,16 +4,24 @@ The Mimecast integration collects events from the Mimecast API. ## Configuration -Authorization parameters for the Mimecast API (`Application Key`, `Application ID`, `Access Key`, and `Secret Key`), should be provided by a Mimecast representative for this integration. -Under `Advanced options` you can set the time interval between two API requests as well as the API URL. A Mimecast representative should also be able to give you with this information in case you need to change the defaults. +Authorization parameters for the Mimecast API (`Application Key`, `Application +ID`, `Access Key`, and `Secret Key`) should be provided by a Mimecast +representative for this integration. Under `Advanced options` you can set the +time interval between two API requests as well as the API URL. A Mimecast +representative should also be able to give you with this information in case you +need to change the defaults. -Note that rate limit quotas may require you to set up different credentials for the different available log types. +Note that rate limit quotas may require you to set up different credentials for +the different available log types. ## Logs ### Audit Events -This is the `mimecast.audit_events` dataset. These logs contain Mimecast audit events with the following details: audit type, event category and detailed information about the event. More information about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-audit-events/). +This is the `mimecast.audit_events` dataset. These logs contain Mimecast audit +events with the following details: audit type, event category, and detailed +information about the event. More information about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-audit-events/). {{event "audit_events"}} @@ -21,7 +29,10 @@ This is the `mimecast.audit_events` dataset. These logs contain Mimecast audit e ### DLP Logs -This is the `mimecast.dlp_logs` dataset. These logs contain information about messages that triggered a DLP or Content Examination policy. More information about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-dlp-logs/). +This is the `mimecast.dlp_logs` dataset. These logs contain information about +messages that triggered a DLP or Content Examination policy. More information +about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-dlp-logs/). {{event "dlp_logs"}} @@ -29,48 +40,75 @@ This is the `mimecast.dlp_logs` dataset. These logs contain information about me ### SIEM Logs -This is the `mimecast.siem_logs` dataset. These logs contain information about messages that contains MTA logs (MTA = message transfer agent) – all Inbound, outbound and internal messages. More about these logs [here](https://integrations.mimecast.com/documentation/tutorials/understanding-siem-logs/). +This is the `mimecast.siem_logs` dataset. These logs contain information about +messages that contains MTA (message transfer agent) log – all inbound, +outbound, and internal messages. More about these logs +[here](https://integrations.mimecast.com/documentation/tutorials/understanding-siem-logs/). {{event "siem_logs"}} {{fields "siem_logs"}} -### TTP Impersonation Logs +### Threat Intel Feed Malware: Customer -This is the `mimecast.ttp_ip_logs` dataset. These logs contain information about messages containing information flagged by an Impersonation Protection configuration. Learn more about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-impersonation-protect-logs/). +This is the `mimecast.threat_intel_malware_customer` dataset. These logs contain +information about messages that return identified malware threats at a customer +level. More about these logs +[here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). -{{event "ttp_ip_logs"}} +{{event "threat_intel_malware_customer"}} -{{fields "ttp_ip_logs"}} +{{fields "threat_intel_malware_customer"}} -### TTP Attachment Logs +### Threat Intel Feed Malware: Grid -This is the `mimecast.ttp_ap_logs` dataset. These logs contain Mimecast TTP attachment protection logs with the following details: result of attachment analysis (if it is malicious or not etc.), date when file is released, sender and recipient address, filename and type, action triggered for the attachment, the route of the original email containing the attachment and details. Learn more about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-attachment-protection-logs/). +This is the `mimecast.threat_intel_malware_grid` dataset. These logs contain +information about messages that return identified malware threats at a regional +grid level. More about these logs +[here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). -{{event "ttp_ap_logs"}} +{{event "threat_intel_malware_grid"}} -{{fields "ttp_ap_logs"}} +{{fields "threat_intel_malware_grid"}} -### TTP URL Logs +### TTP Attachment Logs -This is the `mimecast.ttp_url_logs` dataset. These logs contain Mimecast TTP attachment protection logs with the following details: the category of the URL clicked, the email address of the user who clicked the link, the url clicked, the action taken by the user if user awareness was applied, the route of the email that contained the link, the action defined by the administrator for the URL, the date that the URL was clicked, url scan result, the action that was taken for the click, the description of the definition that triggered the URL to be rewritten by Mimecast, the action requested by the user, an array of components of the message where the URL was found. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-url-logs/). +This is the `mimecast.ttp_ap_logs` dataset. These logs contain Mimecast TTP +attachment protection logs with the following details: result of attachment +analysis (if it is malicious or not etc.), date when file is released, sender +and recipient address, filename and type, action triggered for the attachment, +the route of the original email containing the attachment and details. Learn +more about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-attachment-protection-logs/). -{{event "ttp_url_logs"}} +{{event "ttp_ap_logs"}} -{{fields "ttp_url_logs"}} +{{fields "ttp_ap_logs"}} -### Threat Intel Feed Malware: Customer +### TTP Impersonation Logs -This is the `mimecast.threat_intel_malware_customer` dataset. These logs contain information about messages that return identified malware threats at a customer level. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). +This is the `mimecast.ttp_ip_logs` dataset. These logs contain information about +messages containing information flagged by an Impersonation Protection +configuration. Learn more about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-impersonation-protect-logs/). -{{event "threat_intel_malware_customer"}} +{{event "ttp_ip_logs"}} -{{fields "threat_intel_malware_customer"}} +{{fields "ttp_ip_logs"}} -### Threat Intel Feed Malware: Grid +### TTP URL Logs -This is the `mimecast.threat_intel_malware_grid` dataset. These logs contain information about messages that return identified malware threats at a regional grid level. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). +This is the `mimecast.ttp_url_logs` dataset. These logs contain Mimecast TTP +attachment protection logs with the following details: the category of the URL +clicked, the email address of the user who clicked the link, the url clicked, +the action taken by the user if user awareness was applied, the route of the +email that contained the link, the action defined by the administrator for the +URL, the date that the URL was clicked, url scan result, the action that was +taken for the click, the description of the definition that triggered the URL to +be rewritten by Mimecast, the action requested by the user, an array of +components of the message where the URL was found. More about these logs +[here](https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-url-logs/). -{{event "threat_intel_malware_grid"}} +{{event "ttp_url_logs"}} -{{fields "threat_intel_malware_grid"}} \ No newline at end of file +{{fields "ttp_url_logs"}} From 7fad2e24c9aa032bfdd6d77b2fd71d81280b5933 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:08:34 -0400 Subject: [PATCH 15/18] Update readme [git-generate] cd packages/mimecast elastic-package build --- packages/mimecast/docs/README.md | 768 ++++++++++++++++--------------- 1 file changed, 403 insertions(+), 365 deletions(-) diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md index d2ac818a2e3..76ee0cba89f 100644 --- a/packages/mimecast/docs/README.md +++ b/packages/mimecast/docs/README.md @@ -4,16 +4,24 @@ The Mimecast integration collects events from the Mimecast API. ## Configuration -Authorization parameters for the Mimecast API (`Application Key`, `Application ID`, `Access Key`, and `Secret Key`), should be provided by a Mimecast representative for this integration. -Under `Advanced options` you can set the time interval between two API requests as well as the API URL. A Mimecast representative should also be able to give you with this information in case you need to change the defaults. +Authorization parameters for the Mimecast API (`Application Key`, `Application +ID`, `Access Key`, and `Secret Key`) should be provided by a Mimecast +representative for this integration. Under `Advanced options` you can set the +time interval between two API requests as well as the API URL. A Mimecast +representative should also be able to give you with this information in case you +need to change the defaults. -Note that rate limit quotas may require you to set up different credentials for the different available log types. +Note that rate limit quotas may require you to set up different credentials for +the different available log types. ## Logs ### Audit Events -This is the `mimecast.audit_events` dataset. These logs contain Mimecast audit events with the following details: audit type, event category and detailed information about the event. More information about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-audit-events/). +This is the `mimecast.audit_events` dataset. These logs contain Mimecast audit +events with the following details: audit type, event category, and detailed +information about the event. More information about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-audit-events/). An example event for `audit_events` looks as following: @@ -81,11 +89,9 @@ An example event for `audit_events` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.as.asn | Client ASN number. | long | | client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | client.as.organization.name | Organization name. | keyword | | client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.as.organization_name | Client Organization name. | keyword | | client.geo.city_name | City name. | keyword | | client.geo.continent_name | Name of the continent. | keyword | | client.geo.country_iso_code | Country ISO code. | keyword | @@ -111,16 +117,18 @@ An example event for `audit_events` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | -| email.origination_timestamp | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | date | -| email.subject | A brief summary of the topic of the message | keyword | -| email.subject.text | Multi-field of `email.subject`. | text | -| email.to.address | The email address(es) of the message recipient(s) | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.origination_timestamp | The date and time the email message was composed. Many email clients will fill in this value automatically when the message is sent by a user. | date | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | +| email.to.address | The email address of recipient | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | | file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | | file.name | Name of the file including the extension, without the directory. | keyword | | file.size | File size in bytes. Only relevant when `file.type` is "file". | long | @@ -161,7 +169,10 @@ An example event for `audit_events` looks as following: ### DLP Logs -This is the `mimecast.dlp_logs` dataset. These logs contain information about messages that triggered a DLP or Content Examination policy. More information about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-dlp-logs/). +This is the `mimecast.dlp_logs` dataset. These logs contain information about +messages that triggered a DLP or Content Examination policy. More information +about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-dlp-logs/). An example event for `dlp` looks as following: @@ -248,14 +259,14 @@ An example event for `dlp` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.direction | Direction of the message based on the sending and receiving domains | keyword | -| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | -| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | -| email.message_id.text | Multi-field of `email.message_id`. | text | -| email.subject | A brief summary of the topic of the message | keyword | -| email.subject.text | Multi-field of `email.subject`. | text | -| email.to.address | The email address(es) of the message recipient(s) | keyword | +| email.direction | The direction of the message based on the sending and receiving domains. | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | +| email.to.address | The email address of recipient | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | @@ -278,20 +289,16 @@ An example event for `dlp` looks as following: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| mimecast.action | The action taken against the message. | keyword | -| mimecast.messageId | The message-id value of the message. | keyword | -| mimecast.policy | The name of a DLP or Content Examination configuration that triggered the message. | keyword | -| mimecast.recipientAddress | Email address of the recipient. | keyword | -| mimecast.route | The message direction. Possible values are inbound, outbound or internal. | keyword | -| mimecast.senderAddress | Email address of the sender. | keyword | -| mimecast.subject | The message subject. | keyword | | rule.name | The name of the rule or signature generating the event. | keyword | | tags | List of keywords used to tag each event. | keyword | ### SIEM Logs -This is the `mimecast.siem_logs` dataset. These logs contain information about messages that contains MTA logs (MTA = message transfer agent) – all Inbound, outbound and internal messages. More about these logs [here](https://integrations.mimecast.com/documentation/tutorials/understanding-siem-logs/). +This is the `mimecast.siem_logs` dataset. These logs contain information about +messages that contains MTA (message transfer agent) log – all inbound, +outbound, and internal messages. More about these logs +[here](https://integrations.mimecast.com/documentation/tutorials/understanding-siem-logs/). An example event for `siem` looks as following: @@ -379,26 +386,24 @@ An example event for `siem` looks as following: | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | -| email.attachments.file.mime_type | MIME type of the attachment file. | keyword | -| email.attachments.file.name | Name of the attachment file including the extension. | keyword | +| email.attachments.file.hash.md5 | MD5 hash. | keyword | +| email.attachments.file.hash.sha1 | SHA1 hash. | keyword | +| email.attachments.file.hash.sha256 | SHA256 hash. | keyword | +| email.attachments.file.mime_type | The MIME media type of the attachment. This value will typically be extracted from the `Content-Type` MIME header field. | keyword | +| email.attachments.file.name | Name of the attachment file including the file extension. | keyword | | email.attachments.file.size | Attachment file size in bytes. | long | -| email.attachments.hash.md5 | MD5 hash of the file attachment. | keyword | -| email.attachments.hash.sha1 | SHA-1 hash of the file attachment. | keyword | -| email.attachments.hash.sha256 | SHA-256 hash of the file attachment. | keyword | -| email.direction | Direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | -| email.header_from | The sender address found in the from header of the email. | keyword | -| email.local_id | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the X-MS-Exchange-Organization-Network-Message-Id id). | keyword | -| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | -| email.message_id.text | Multi-field of `email.message_id`. | text | -| email.message_size | The total size of the email.The total size of the email. | long | -| email.subject | A brief summary of the topic of the message | keyword | -| email.subject.text | Multi-field of `email.subject`. | text | -| email.to.address | The email address(es) of the message recipient(s). | keyword | +| email.direction | The direction of the message based on the sending and receiving domains. | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.local_id | Unique identifier given to the email by the source that created the event. Identifier is not persistent across hops. | keyword | +| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | +| email.to.address | The email address of recipient | keyword | | error.code | Error code describing the error. | keyword | | error.message | Error message. | match_only_text | | error.type | The type of the error, for example the class name of the exception. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | @@ -467,11 +472,9 @@ An example event for `siem` looks as following: | mimecast.msgid | The internet message id of the email. | keyword | | mimecast.urlCategory | The category of the URL that was clicked. | keyword | | rule.name | The name of the rule or signature generating the event. | keyword | -| source.as.asn | Client ASN number. | long | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.as.organization_name | Client Organization name. | keyword | | source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | | source.geo.city_name | City name. | keyword | | source.geo.continent_name | Name of the continent. | keyword | @@ -490,17 +493,20 @@ An example event for `siem` looks as following: | user.email | User email address. | keyword | -### TTP Impersonation Logs +### Threat Intel Feed Malware: Customer -This is the `mimecast.ttp_ip_logs` dataset. These logs contain information about messages containing information flagged by an Impersonation Protection configuration. Learn more about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-impersonation-protect-logs/). +This is the `mimecast.threat_intel_malware_customer` dataset. These logs contain +information about messages that return identified malware threats at a customer +level. More about these logs +[here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). -An example event for `ttp_ip` looks as following: +An example event for `threat_intel_malware_customer` looks as following: ```json { - "@timestamp": "2021-11-12T15:27:04.000Z", + "@timestamp": "2021-11-19T01:28:37.099Z", "agent": { - "ephemeral_id": "923dd3fb-0685-4b8f-9f21-90fc828587b1", + "ephemeral_id": "350131de-71cb-4dba-9001-75ff27fc2e0f", "hostname": "docker-fleet-agent", "id": "01800603-1f81-46c1-b412-764819259d1b", "name": "docker-fleet-agent", @@ -508,7 +514,7 @@ An example event for `ttp_ip` looks as following: "version": "7.16.0" }, "data_stream": { - "dataset": "mimecast.ttp_ip_logs", + "dataset": "mimecast.threat_intel_malware_customer", "namespace": "ep", "type": "logs" }, @@ -520,64 +526,50 @@ An example event for `ttp_ip` looks as following: "snapshot": true, "version": "7.16.0" }, - "email": { - "from": { - "address": [ - "johndoe@example.com" - ] - }, - "message_id": "\u003cMN2PR16MB2719879CA4DB60C265F7FD8FB0959@MN2PR16MB2719.namprd16.prod.outlook.com\u003e", - "subject": "Don't read, just fill out!", - "to": { - "address": [ - "johndoe@example.com" - ] - } - }, "event": { - "action": "none", "agent_id_status": "verified", - "created": "2021-11-12T15:27:04+0000", - "dataset": "mimecast.ttp_ip_logs", - "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGMiEg8", - "ingested": "2022-04-21T08:28:03Z", - "original": "{\"action\":\"none\",\"definition\":\"IP - 1 hit (Tag email)\",\"eventTime\":\"2021-11-12T15:27:04+0000\",\"hits\":1,\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGMiEg8\",\"identifiers\":[\"internal_user_name\"],\"impersonationResults\":[{\"checkerResult\":\"hit\",\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe \\u003cjohndoe_cdw@example.com\\u003e\",\"stringSimilarToDomain\":\"John Doe\"}],\"messageId\":\"\\u003cMN2PR16MB2719879CA4DB60C265F7FD8FB0959@MN2PR16MB2719.namprd16.prod.outlook.com\\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"senderAddress\":\"johndoe@example.com\",\"senderIpAddress\":\"8.8.8.8\",\"subject\":\"Don't read, just fill out!\",\"taggedExternal\":false,\"taggedMalicious\":true}" + "category": "threat", + "created": "2022-04-21T08:25:44.963Z", + "dataset": "mimecast.threat_intel_malware_customer", + "ingested": "2022-04-21T08:25:45Z", + "kind": "enrichment", + "original": "{\"created\":\"2021-11-19T01:28:37.099Z\",\"id\":\"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd\",\"labels\":[\"malicious-activity\"],\"modified\":\"2021-11-19T01:28:37.099Z\",\"pattern\":\"[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']\",\"type\":\"indicator\",\"valid_from\":\"2021-11-19T01:28:37.099Z\"}", + "type": "indicator" }, "input": { "type": "httpjson" }, "mimecast": { - "hits": 1, - "identifiers": [ - "internal_user_name" - ], - "impersonationResults": [ - { - "checkerResult": "hit", - "impersonationDomainSource": "internal_user_name", - "similarDomain": "John Doe \u003cjohndoe_cdw@example.com\u003e", - "stringSimilarToDomain": "John Doe" - } + "id": "indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd", + "labels": [ + "malicious-activity" ], - "taggedExternal": false, - "taggedMalicious": true + "pattern": "[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']", + "type": "indicator" }, "related": { - "ip": [ - "8.8.8.8" + "hash": [ + "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" ] }, - "rule": { - "name": "IP - 1 hit (Tag email)" - }, - "source": { - "ip": "8.8.8.8" - }, "tags": [ "preserve_original_event", "forwarded", - "mimecast-ttp-ip" - ] + "mimecast-threat-intel-feed-malware-customer", + "malicious-activity" + ], + "threat": { + "indicator": { + "file": { + "hash": { + "sha256": "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" + } + }, + "first_seen": "2021-11-19T01:28:37.099Z", + "modified_at": "2021-11-19T01:28:37.099Z", + "type": "file" + } + } } ``` @@ -603,17 +595,13 @@ An example event for `ttp_ip` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | -| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | -| email.message_id.text | Multi-field of `email.message_id`. | text | -| email.subject | A brief summary of the topic of the message | keyword | -| email.subject.text | Multi-field of `email.subject`. | text | -| email.to.address | The email address(es) of the message recipient(s) | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -633,40 +621,45 @@ An example event for `ttp_ip` looks as following: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| mimecast.action | The action triggered by the email. | keyword | -| mimecast.definition | The name of the policy definition that triggered the log. | keyword | -| mimecast.hits | The number of identifiers that the message triggered. | long | -| mimecast.id | A token that can be used to retrieve this log again. | keyword | -| mimecast.identifiers | The properties of the message that triggered the action - similar_internal_domain, newly_observed_domain, internal_user_name, reply_address_mismatch, and/or targeted_threat_dictionary. | keyword | -| mimecast.impersonationResults.checkerResult | Result checker. | keyword | -| mimecast.impersonationResults.impersonationDomainSource | Impersonation domain source. | keyword | -| mimecast.impersonationResults.similarDomain | Similar domain. | keyword | -| mimecast.impersonationResults.stringSimilarToDomain | The string that is suspiciously similar to a known value within the Mimecast configuration. Multiple triggers will be comma-separated. | keyword | -| mimecast.messageId | The message-id of the identified message. | keyword | -| mimecast.recipientAddress | The email address of the recipient of the email. | keyword | -| mimecast.senderAddress | The email address of the sender of the message. | keyword | -| mimecast.senderIpAddress | The source IP address of the message. | keyword | -| mimecast.subject | The subject of the email. | keyword | -| mimecast.taggedExternal | Whether the message was tagged as coming from an external address. | boolean | -| mimecast.taggedMalicious | Whether the message was tagged as malicious. | boolean | -| related.ip | All of the IPs seen on your event. | ip | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| mimecast.created | When the indicator was last created. | date | +| mimecast.hashtype | The hash type. | keyword | +| mimecast.id | The ID of the indicator. | keyword | +| mimecast.labels | The labels related to the indicator. | keyword | +| mimecast.log_type | String to get type of Threat intel feed. | keyword | +| mimecast.modified | When the indicator was last modified. | date | +| mimecast.name | Name of the file. | keyword | +| mimecast.pattern | The pattern. | keyword | +| mimecast.relationship_type | Type of the relationship. | keyword | +| mimecast.source_ref | Source of the reference. | keyword | +| mimecast.target_ref | Reference target. | keyword | +| mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | +| mimecast.valid_from | The valid from date. | date | +| mimecast.value | The value of the indicator. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | | tags | List of keywords used to tag each event. | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | -### TTP Attachment Logs +### Threat Intel Feed Malware: Grid -This is the `mimecast.ttp_ap_logs` dataset. These logs contain Mimecast TTP attachment protection logs with the following details: result of attachment analysis (if it is malicious or not etc.), date when file is released, sender and recipient address, filename and type, action triggered for the attachment, the route of the original email containing the attachment and details. Learn more about these logs [here] (https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-attachment-protection-logs/). +This is the `mimecast.threat_intel_malware_grid` dataset. These logs contain +information about messages that return identified malware threats at a regional +grid level. More about these logs +[here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). -An example event for `ttp_ap` looks as following: +An example event for `threat_intel_malware_grid` looks as following: ```json { - "@timestamp": "2021-11-24T11:54:27.000Z", + "@timestamp": "2021-11-19T01:28:37.099Z", "agent": { - "ephemeral_id": "51899d24-0340-41eb-b0aa-1e2e9def2460", + "ephemeral_id": "2c512f3d-fe8b-4751-a5a0-df442fcba073", "hostname": "docker-fleet-agent", "id": "01800603-1f81-46c1-b412-764819259d1b", "name": "docker-fleet-agent", @@ -674,7 +667,7 @@ An example event for `ttp_ap` looks as following: "version": "7.16.0" }, "data_stream": { - "dataset": "mimecast.ttp_ap_logs", + "dataset": "mimecast.threat_intel_malware_grid", "namespace": "ep", "type": "logs" }, @@ -686,57 +679,50 @@ An example event for `ttp_ap` looks as following: "snapshot": true, "version": "7.16.0" }, - "email": { - "attachments": { - "file": { - "extension": "pdf", - "mime_type": "application/pdf", - "name": "Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf" - }, - "hash": "cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254" - }, - "direction": "inbound", - "from": { - "address": [ - "\u003c\u003e" - ] - }, - "message_id": "\u003cCAKUQxhimsCd1bvWQVs14Amuh1+Hnw_bmSuA7ot8hy4eDa9_ziQ@mail.gmail.com\u003e", - "subject": "Test Files", - "to": { - "address": [ - "johndoe@emample.com" - ] - } - }, "event": { - "action": "user_release_none", "agent_id_status": "verified", - "created": "2021-11-24T11:54:27+0000", - "dataset": "mimecast.ttp_ap_logs", - "ingested": "2022-04-21T08:27:16Z", - "original": "{\"actionTriggered\":\"user release, none\",\"date\":\"2021-11-24T11:54:27+0000\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 7 sec\",\"fileHash\":\"cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254\",\"fileName\":\"Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf\",\"fileType\":\"application/pdf\",\"messageId\":\"\\u003cCAKUQxhimsCd1bvWQVs14Amuh1+Hnw_bmSuA7ot8hy4eDa9_ziQ@mail.gmail.com\\u003e\",\"recipientAddress\":\"johndoe@emample.com\",\"result\":\"safe\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Test Files\"}" + "category": "threat", + "created": "2022-04-21T08:26:32.512Z", + "dataset": "mimecast.threat_intel_malware_grid", + "ingested": "2022-04-21T08:26:33Z", + "kind": "enrichment", + "original": "{\"created\":\"2021-11-19T01:28:37.099Z\",\"id\":\"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd\",\"labels\":[\"malicious-activity\"],\"modified\":\"2021-11-19T01:28:37.099Z\",\"pattern\":\"[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']\",\"type\":\"indicator\",\"valid_from\":\"2021-11-19T01:28:37.099Z\"}", + "type": "indicator" }, "input": { "type": "httpjson" }, "mimecast": { - "details": "Safe \r\nTime taken: 0 hrs, 0 min, 7 sec", - "result": "safe" + "id": "indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd", + "labels": [ + "malicious-activity" + ], + "pattern": "[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']", + "type": "indicator" }, "related": { "hash": [ - "cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254" + "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" ] }, - "rule": { - "name": "Inbound - Safe file with On-Demand Sandbox" - }, "tags": [ "preserve_original_event", "forwarded", - "mimecast-ttp-ap" - ] + "mimecast-threat-intel-feed-malware-grid", + "malicious-activity" + ], + "threat": { + "indicator": { + "file": { + "hash": { + "sha256": "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" + } + }, + "first_seen": "2021-11-19T01:28:37.099Z", + "modified_at": "2021-11-19T01:28:37.099Z", + "type": "file" + } + } } ``` @@ -762,21 +748,13 @@ An example event for `ttp_ap` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | -| email.attachments.file.mime_type | MIME type of the attachment file. | keyword | -| email.attachments.file.name | Name of the attachment file including the extension. | keyword | -| email.attachments.hash | File hash. | keyword | -| email.direction | Direction of the message based on the sending and receiving domains | keyword | -| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | -| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | -| email.message_id.text | Multi-field of `email.message_id`. | text | -| email.subject | A brief summary of the topic of the message | keyword | -| email.subject.text | Multi-field of `email.subject`. | text | -| email.to.address | The email address(es) of the message recipient(s) | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -796,34 +774,48 @@ An example event for `ttp_ap` looks as following: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| mimecast.actionTriggered | The action triggered for the attachment. | keyword | -| mimecast.definition | The definition. | keyword | -| mimecast.details | Detailed output of the attachment sandbox processing. | keyword | -| mimecast.fileHash | The hash of the attachment. | keyword | -| mimecast.fileName | The file name of the original attachment. | keyword | -| mimecast.fileType | The file type of the attachment. | keyword | -| mimecast.messageId | The internet message id of the email. | keyword | -| mimecast.recipientAddress | The address of the user that received the attachment. | keyword | -| mimecast.result | The result of the attachment analysis - clean, malicious, unknown, or timeout. | keyword | -| mimecast.route | The route of the original email containing the attachment, either - inbound, outbound, internal, or external. | keyword | -| mimecast.senderAddress | The sender of the attachment. | keyword | -| mimecast.subject | The subject of the email. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| mimecast.created | When the indicator was last created. | date | +| mimecast.hashtype | The hash type. | keyword | +| mimecast.id | The ID of the indicator. | keyword | +| mimecast.labels | The labels related to the indicator. | keyword | +| mimecast.log_type | String to get type of Threat intel feed. | keyword | +| mimecast.modified | When the indicator was last modified. | date | +| mimecast.name | Name of the file. | keyword | +| mimecast.pattern | The pattern. | keyword | +| mimecast.relationship_type | Type of the relationship. | keyword | +| mimecast.source_ref | Source of the reference. | keyword | +| mimecast.target_ref | Reference target. | keyword | +| mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | +| mimecast.valid_from | The valid from date. | date | +| mimecast.value | The value of the indicator. | keyword | | related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | | tags | List of keywords used to tag each event. | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | -### TTP URL Logs +### TTP Attachment Logs -This is the `mimecast.ttp_url_logs` dataset. These logs contain Mimecast TTP attachment protection logs with the following details: the category of the URL clicked, the email address of the user who clicked the link, the url clicked, the action taken by the user if user awareness was applied, the route of the email that contained the link, the action defined by the administrator for the URL, the date that the URL was clicked, url scan result, the action that was taken for the click, the description of the definition that triggered the URL to be rewritten by Mimecast, the action requested by the user, an array of components of the message where the URL was found. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-url-logs/). +This is the `mimecast.ttp_ap_logs` dataset. These logs contain Mimecast TTP +attachment protection logs with the following details: result of attachment +analysis (if it is malicious or not etc.), date when file is released, sender +and recipient address, filename and type, action triggered for the attachment, +the route of the original email containing the attachment and details. Learn +more about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-attachment-protection-logs/). -An example event for `ttp_url` looks as following: +An example event for `ttp_ap` looks as following: ```json { - "@timestamp": "2021-11-10T03:49:53.000Z", + "@timestamp": "2021-11-24T11:54:27.000Z", "agent": { - "ephemeral_id": "e183b143-0352-44a0-a59f-5a9288714e8b", + "ephemeral_id": "51899d24-0340-41eb-b0aa-1e2e9def2460", "hostname": "docker-fleet-agent", "id": "01800603-1f81-46c1-b412-764819259d1b", "name": "docker-fleet-agent", @@ -831,7 +823,7 @@ An example event for `ttp_url` looks as following: "version": "7.16.0" }, "data_stream": { - "dataset": "mimecast.ttp_url_logs", + "dataset": "mimecast.ttp_ap_logs", "namespace": "ep", "type": "logs" }, @@ -844,62 +836,58 @@ An example event for `ttp_url` looks as following: "version": "7.16.0" }, "email": { + "attachments": { + "file": { + "extension": "pdf", + "hash": { + "sha256": "cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254" + }, + "mime_type": "application/pdf", + "name": "Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf" + } + }, "direction": "inbound", "from": { "address": [ - "googlealerts-noreply@google.com" + "\u003c\u003e" ] }, - "message_id": "\u003c000000000000a02a0a05d0671c06@google.com\u003e", - "subject": "Google Alert - china", + "message_id": "\u003cCAKUQxhimsCd1bvWQVs14Amuh1+Hnw_bmSuA7ot8hy4eDa9_ziQ@mail.gmail.com\u003e", + "subject": "Test Files", "to": { "address": [ - "johndoe@example.com" + "johndoe@emample.com" ] } }, "event": { - "action": "Continue", + "action": "user_release_none", "agent_id_status": "verified", - "created": "2021-11-10T03:49:53+0000", - "dataset": "mimecast.ttp_url_logs", - "ingested": "2022-04-21T08:28:44Z", - "original": "{\"action\":\"allow\",\"actions\":\"Allow\",\"adminOverride\":\"N/A\",\"category\":\"Search Engines \\u0026 Portals\",\"creationMethod\":\"User Click\",\"date\":\"2021-11-10T03:49:53+0000\",\"emailPartsDescription\":[\"Body\"],\"fromUserEmailAddress\":\"googlealerts-noreply@google.com\",\"messageId\":\"\\u003c000000000000a02a0a05d0671c06@google.com\\u003e\",\"route\":\"inbound\",\"scanResult\":\"clean\",\"sendingIp\":\"8.8.8.8\",\"subject\":\"Google Alert - china\",\"ttpDefinition\":\"Inbound URL 'Aggressive'\",\"url\":\"https://www.google.co.za/alerts/share?hl=en\\u0026gl=US\\u0026ru=https://www.wsj.com/articles/u-s-tests-israels-iron-dome-in-guam-as-defense-against-chinese-cruise-missiles-11636455224\\u0026ss=tw\\u0026rt=U.S.+Tests+Israel%27s+Iron+Dome+in+Guam+as+Defense+Against+Chinese+Cruise+Missiles+-+WSJ\\u0026cd=KhQxNzg2NTc5NDQ3ODIzODUyNjI5NzIcZmQ4N2VjYzkxMGIxMWE4Yzpjby56YTplbjpVUw\\u0026ssp=AMJHsmW3CCK1S4TNPifSXszcyaNMwd6TDg\",\"userAwarenessAction\":\"Continue\",\"userEmailAddress\":\"johndoe@example.com\",\"userOverride\":\"None\"}" + "created": "2021-11-24T11:54:27+0000", + "dataset": "mimecast.ttp_ap_logs", + "ingested": "2022-04-21T08:27:16Z", + "original": "{\"actionTriggered\":\"user release, none\",\"date\":\"2021-11-24T11:54:27+0000\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 7 sec\",\"fileHash\":\"cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254\",\"fileName\":\"Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf\",\"fileType\":\"application/pdf\",\"messageId\":\"\\u003cCAKUQxhimsCd1bvWQVs14Amuh1+Hnw_bmSuA7ot8hy4eDa9_ziQ@mail.gmail.com\\u003e\",\"recipientAddress\":\"johndoe@emample.com\",\"result\":\"safe\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Test Files\"}" }, "input": { "type": "httpjson" }, "mimecast": { - "action": "allow", - "actions": "Allow", - "adminOverride": "N/A", - "category": "Search Engines \u0026 Portals", - "creationMethod": "User Click", - "emailPartsDescription": [ - "Body" - ], - "scanResult": "clean", - "userOverride": "None" + "details": "Safe \r\nTime taken: 0 hrs, 0 min, 7 sec", + "result": "safe" }, "related": { - "ip": [ - "8.8.8.8" + "hash": [ + "cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254" ] }, "rule": { - "name": "Inbound URL 'Aggressive'" - }, - "source": { - "ip": "8.8.8.8" + "name": "Inbound - Safe file with On-Demand Sandbox" }, "tags": [ "preserve_original_event", "forwarded", - "mimecast-ttp-url" - ], - "url": { - "original": "https://www.google.co.za/alerts/share?hl=en\u0026gl=US\u0026ru=https://www.wsj.com/articles/u-s-tests-israels-iron-dome-in-guam-as-defense-against-chinese-cruise-missiles-11636455224\u0026ss=tw\u0026rt=U.S.+Tests+Israel%27s+Iron+Dome+in+Guam+as+Defense+Against+Chinese+Cruise+Missiles+-+WSJ\u0026cd=KhQxNzg2NTc5NDQ3ODIzODUyNjI5NzIcZmQ4N2VjYzkxMGIxMWE4Yzpjby56YTplbjpVUw\u0026ssp=AMJHsmW3CCK1S4TNPifSXszcyaNMwd6TDg" - } + "mimecast-ttp-ap" + ] } ``` @@ -925,14 +913,18 @@ An example event for `ttp_url` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.direction | Direction of the message based on the sending and receiving domains | keyword | -| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | -| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | -| email.message_id.text | Multi-field of `email.message_id`. | text | -| email.subject | A brief summary of the topic of the message | keyword | -| email.subject.text | Multi-field of `email.subject`. | text | -| email.to.address | Stores the from email address to the RFC5322 From - header field. | keyword | +| email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | +| email.attachments.file.hash.sha256 | SHA256 hash. | keyword | +| email.attachments.file.mime_type | The MIME media type of the attachment. This value will typically be extracted from the `Content-Type` MIME header field. | keyword | +| email.attachments.file.name | Name of the attachment file including the file extension. | keyword | +| email.direction | The direction of the message based on the sending and receiving domains. | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | +| email.to.address | The email address of recipient | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | @@ -955,44 +947,37 @@ An example event for `ttp_url` looks as following: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| mimecast.action | The action that was taken for the click. | keyword | -| mimecast.actions | The actions that were taken. | keyword | -| mimecast.adminOverride | The action defined by the administrator for the URL. | keyword | -| mimecast.category | The category of the URL clicked. | keyword | -| mimecast.creationMethod | The description how event occurred. | keyword | -| mimecast.emailPartsDescription | An array of components of the messge where the URL was found. | keyword | -| mimecast.fromUserEmailAddress | The email of user who triggers the event. | keyword | -| mimecast.messageId | The message-id value of the message. | keyword | -| mimecast.route | The route of the email that contained the link. | keyword | -| mimecast.scanResult | The result of the URL scan. | keyword | -| mimecast.sendingIP | The IP of user who triggers the event. | keyword | +| mimecast.actionTriggered | The action triggered for the attachment. | keyword | +| mimecast.definition | The definition. | keyword | +| mimecast.details | Detailed output of the attachment sandbox processing. | keyword | +| mimecast.fileHash | The hash of the attachment. | keyword | +| mimecast.fileName | The file name of the original attachment. | keyword | +| mimecast.fileType | The file type of the attachment. | keyword | +| mimecast.messageId | The internet message id of the email. | keyword | +| mimecast.recipientAddress | The address of the user that received the attachment. | keyword | +| mimecast.result | The result of the attachment analysis - clean, malicious, unknown, or timeout. | keyword | +| mimecast.route | The route of the original email containing the attachment, either - inbound, outbound, internal, or external. | keyword | +| mimecast.senderAddress | The sender of the attachment. | keyword | | mimecast.subject | The subject of the email. | keyword | -| mimecast.ttpDefinition | The description of the definition that triggered the URL to be rewritten by Mimecast. | keyword | -| mimecast.url | The url clicked. | keyword | -| mimecast.userAwarenessAction | The action taken by the user if user awareness was applied. | keyword | -| mimecast.userEmailAddress | The email address of the user who clicked the link. | keyword | -| mimecast.userOverride | The action requested by the user. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | | rule.name | The name of the rule or signature generating the event. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | | tags | List of keywords used to tag each event. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| user.email | User email address. | keyword | -### Threat Intel Feed Malware: Customer +### TTP Impersonation Logs -This is the `mimecast.threat_intel_malware_customer` dataset. These logs contain information about messages that return identified malware threats at a customer level. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). +This is the `mimecast.ttp_ip_logs` dataset. These logs contain information about +messages containing information flagged by an Impersonation Protection +configuration. Learn more about these logs [here] +(https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-impersonation-protect-logs/). -An example event for `threat_intel_malware_customer` looks as following: +An example event for `ttp_ip` looks as following: ```json { - "@timestamp": "2021-11-19T01:28:37.099Z", + "@timestamp": "2021-11-12T15:27:04.000Z", "agent": { - "ephemeral_id": "350131de-71cb-4dba-9001-75ff27fc2e0f", + "ephemeral_id": "923dd3fb-0685-4b8f-9f21-90fc828587b1", "hostname": "docker-fleet-agent", "id": "01800603-1f81-46c1-b412-764819259d1b", "name": "docker-fleet-agent", @@ -1000,7 +985,7 @@ An example event for `threat_intel_malware_customer` looks as following: "version": "7.16.0" }, "data_stream": { - "dataset": "mimecast.threat_intel_malware_customer", + "dataset": "mimecast.ttp_ip_logs", "namespace": "ep", "type": "logs" }, @@ -1012,50 +997,64 @@ An example event for `threat_intel_malware_customer` looks as following: "snapshot": true, "version": "7.16.0" }, + "email": { + "from": { + "address": [ + "johndoe@example.com" + ] + }, + "message_id": "\u003cMN2PR16MB2719879CA4DB60C265F7FD8FB0959@MN2PR16MB2719.namprd16.prod.outlook.com\u003e", + "subject": "Don't read, just fill out!", + "to": { + "address": [ + "johndoe@example.com" + ] + } + }, "event": { + "action": "none", "agent_id_status": "verified", - "category": "threat", - "created": "2022-04-21T08:25:44.963Z", - "dataset": "mimecast.threat_intel_malware_customer", - "ingested": "2022-04-21T08:25:45Z", - "kind": "enrichment", - "original": "{\"created\":\"2021-11-19T01:28:37.099Z\",\"id\":\"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd\",\"labels\":[\"malicious-activity\"],\"modified\":\"2021-11-19T01:28:37.099Z\",\"pattern\":\"[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']\",\"type\":\"indicator\",\"valid_from\":\"2021-11-19T01:28:37.099Z\"}", - "type": "indicator" + "created": "2021-11-12T15:27:04+0000", + "dataset": "mimecast.ttp_ip_logs", + "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGMiEg8", + "ingested": "2022-04-21T08:28:03Z", + "original": "{\"action\":\"none\",\"definition\":\"IP - 1 hit (Tag email)\",\"eventTime\":\"2021-11-12T15:27:04+0000\",\"hits\":1,\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGMiEg8\",\"identifiers\":[\"internal_user_name\"],\"impersonationResults\":[{\"checkerResult\":\"hit\",\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe \\u003cjohndoe_cdw@example.com\\u003e\",\"stringSimilarToDomain\":\"John Doe\"}],\"messageId\":\"\\u003cMN2PR16MB2719879CA4DB60C265F7FD8FB0959@MN2PR16MB2719.namprd16.prod.outlook.com\\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"senderAddress\":\"johndoe@example.com\",\"senderIpAddress\":\"8.8.8.8\",\"subject\":\"Don't read, just fill out!\",\"taggedExternal\":false,\"taggedMalicious\":true}" }, "input": { "type": "httpjson" }, "mimecast": { - "id": "indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd", - "labels": [ - "malicious-activity" + "hits": 1, + "identifiers": [ + "internal_user_name" ], - "pattern": "[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']", - "type": "indicator" + "impersonationResults": [ + { + "checkerResult": "hit", + "impersonationDomainSource": "internal_user_name", + "similarDomain": "John Doe \u003cjohndoe_cdw@example.com\u003e", + "stringSimilarToDomain": "John Doe" + } + ], + "taggedExternal": false, + "taggedMalicious": true }, "related": { - "hash": [ - "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" + "ip": [ + "8.8.8.8" ] }, + "rule": { + "name": "IP - 1 hit (Tag email)" + }, + "source": { + "ip": "8.8.8.8" + }, "tags": [ "preserve_original_event", "forwarded", - "mimecast-threat-intel-feed-malware-customer", - "malicious-activity" - ], - "threat": { - "indicator": { - "file": { - "hash": { - "sha256": "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" - } - }, - "first_seen": "2021-11-19T01:28:37.099Z", - "modified_at": "2021-11-19T01:28:37.099Z", - "type": "file" - } - } + "mimecast-ttp-ip" + ] } ``` @@ -1081,7 +1080,15 @@ An example event for `threat_intel_malware_customer` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | +| email.to.address | The email address of recipient | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | +| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | | host.architecture | Operating system architecture. | keyword | @@ -1103,42 +1110,49 @@ An example event for `threat_intel_malware_customer` looks as following: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| mimecast.created | When the indicator was last created. | date | -| mimecast.hashtype | The hash type. | keyword | -| mimecast.id | The ID of the indicator. | keyword | -| mimecast.labels | The labels related to the indicator. | keyword | -| mimecast.log_type | String to get type of Threat intel feed. | keyword | -| mimecast.modified | When the indicator was last modified. | date | -| mimecast.name | Name of the file. | keyword | -| mimecast.pattern | The pattern. | keyword | -| mimecast.relationship_type | Type of the relationship. | keyword | -| mimecast.source_ref | Source of the reference. | keyword | -| mimecast.target_ref | Reference target. | keyword | -| mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | -| mimecast.valid_from | The valid from date. | date | -| mimecast.value | The value of the indicator. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| mimecast.action | The action triggered by the email. | keyword | +| mimecast.definition | The name of the policy definition that triggered the log. | keyword | +| mimecast.hits | The number of identifiers that the message triggered. | long | +| mimecast.id | A token that can be used to retrieve this log again. | keyword | +| mimecast.identifiers | The properties of the message that triggered the action - similar_internal_domain, newly_observed_domain, internal_user_name, reply_address_mismatch, and/or targeted_threat_dictionary. | keyword | +| mimecast.impersonationResults.checkerResult | Result checker. | keyword | +| mimecast.impersonationResults.impersonationDomainSource | Impersonation domain source. | keyword | +| mimecast.impersonationResults.similarDomain | Similar domain. | keyword | +| mimecast.impersonationResults.stringSimilarToDomain | The string that is suspiciously similar to a known value within the Mimecast configuration. Multiple triggers will be comma-separated. | keyword | +| mimecast.messageId | The message-id of the identified message. | keyword | +| mimecast.recipientAddress | The email address of the recipient of the email. | keyword | +| mimecast.senderAddress | The email address of the sender of the message. | keyword | +| mimecast.senderIpAddress | The source IP address of the message. | keyword | +| mimecast.subject | The subject of the email. | keyword | +| mimecast.taggedExternal | Whether the message was tagged as coming from an external address. | boolean | +| mimecast.taggedMalicious | Whether the message was tagged as malicious. | boolean | +| related.ip | All of the IPs seen on your event. | ip | +| rule.name | The name of the rule or signature generating the event. | keyword | +| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | | tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | -### Threat Intel Feed Malware: Grid +### TTP URL Logs -This is the `mimecast.threat_intel_malware_grid` dataset. These logs contain information about messages that return identified malware threats at a regional grid level. More about these logs [here](https://integrations.mimecast.com/documentation/endpoint-reference/threat-intel/get-feed/). +This is the `mimecast.ttp_url_logs` dataset. These logs contain Mimecast TTP +attachment protection logs with the following details: the category of the URL +clicked, the email address of the user who clicked the link, the url clicked, +the action taken by the user if user awareness was applied, the route of the +email that contained the link, the action defined by the administrator for the +URL, the date that the URL was clicked, url scan result, the action that was +taken for the click, the description of the definition that triggered the URL to +be rewritten by Mimecast, the action requested by the user, an array of +components of the message where the URL was found. More about these logs +[here](https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-statistics/get-ttp-url-logs/). -An example event for `threat_intel_malware_grid` looks as following: +An example event for `ttp_url` looks as following: ```json { - "@timestamp": "2021-11-19T01:28:37.099Z", + "@timestamp": "2021-11-10T03:49:53.000Z", "agent": { - "ephemeral_id": "2c512f3d-fe8b-4751-a5a0-df442fcba073", + "ephemeral_id": "e183b143-0352-44a0-a59f-5a9288714e8b", "hostname": "docker-fleet-agent", "id": "01800603-1f81-46c1-b412-764819259d1b", "name": "docker-fleet-agent", @@ -1146,7 +1160,7 @@ An example event for `threat_intel_malware_grid` looks as following: "version": "7.16.0" }, "data_stream": { - "dataset": "mimecast.threat_intel_malware_grid", + "dataset": "mimecast.ttp_url_logs", "namespace": "ep", "type": "logs" }, @@ -1158,49 +1172,62 @@ An example event for `threat_intel_malware_grid` looks as following: "snapshot": true, "version": "7.16.0" }, + "email": { + "direction": "inbound", + "from": { + "address": [ + "googlealerts-noreply@google.com" + ] + }, + "message_id": "\u003c000000000000a02a0a05d0671c06@google.com\u003e", + "subject": "Google Alert - china", + "to": { + "address": [ + "johndoe@example.com" + ] + } + }, "event": { + "action": "Continue", "agent_id_status": "verified", - "category": "threat", - "created": "2022-04-21T08:26:32.512Z", - "dataset": "mimecast.threat_intel_malware_grid", - "ingested": "2022-04-21T08:26:33Z", - "kind": "enrichment", - "original": "{\"created\":\"2021-11-19T01:28:37.099Z\",\"id\":\"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd\",\"labels\":[\"malicious-activity\"],\"modified\":\"2021-11-19T01:28:37.099Z\",\"pattern\":\"[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']\",\"type\":\"indicator\",\"valid_from\":\"2021-11-19T01:28:37.099Z\"}", - "type": "indicator" + "created": "2021-11-10T03:49:53+0000", + "dataset": "mimecast.ttp_url_logs", + "ingested": "2022-04-21T08:28:44Z", + "original": "{\"action\":\"allow\",\"actions\":\"Allow\",\"adminOverride\":\"N/A\",\"category\":\"Search Engines \\u0026 Portals\",\"creationMethod\":\"User Click\",\"date\":\"2021-11-10T03:49:53+0000\",\"emailPartsDescription\":[\"Body\"],\"fromUserEmailAddress\":\"googlealerts-noreply@google.com\",\"messageId\":\"\\u003c000000000000a02a0a05d0671c06@google.com\\u003e\",\"route\":\"inbound\",\"scanResult\":\"clean\",\"sendingIp\":\"8.8.8.8\",\"subject\":\"Google Alert - china\",\"ttpDefinition\":\"Inbound URL 'Aggressive'\",\"url\":\"https://www.google.co.za/alerts/share?hl=en\\u0026gl=US\\u0026ru=https://www.wsj.com/articles/u-s-tests-israels-iron-dome-in-guam-as-defense-against-chinese-cruise-missiles-11636455224\\u0026ss=tw\\u0026rt=U.S.+Tests+Israel%27s+Iron+Dome+in+Guam+as+Defense+Against+Chinese+Cruise+Missiles+-+WSJ\\u0026cd=KhQxNzg2NTc5NDQ3ODIzODUyNjI5NzIcZmQ4N2VjYzkxMGIxMWE4Yzpjby56YTplbjpVUw\\u0026ssp=AMJHsmW3CCK1S4TNPifSXszcyaNMwd6TDg\",\"userAwarenessAction\":\"Continue\",\"userEmailAddress\":\"johndoe@example.com\",\"userOverride\":\"None\"}" }, "input": { "type": "httpjson" }, "mimecast": { - "id": "indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd", - "labels": [ - "malicious-activity" + "action": "allow", + "actions": "Allow", + "adminOverride": "N/A", + "category": "Search Engines \u0026 Portals", + "creationMethod": "User Click", + "emailPartsDescription": [ + "Body" ], - "pattern": "[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']", - "type": "indicator" + "scanResult": "clean", + "userOverride": "None" }, "related": { - "hash": [ - "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" + "ip": [ + "8.8.8.8" ] }, + "rule": { + "name": "Inbound URL 'Aggressive'" + }, + "source": { + "ip": "8.8.8.8" + }, "tags": [ "preserve_original_event", "forwarded", - "mimecast-threat-intel-feed-malware-grid", - "malicious-activity" + "mimecast-ttp-url" ], - "threat": { - "indicator": { - "file": { - "hash": { - "sha256": "ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be" - } - }, - "first_seen": "2021-11-19T01:28:37.099Z", - "modified_at": "2021-11-19T01:28:37.099Z", - "type": "file" - } + "url": { + "original": "https://www.google.co.za/alerts/share?hl=en\u0026gl=US\u0026ru=https://www.wsj.com/articles/u-s-tests-israels-iron-dome-in-guam-as-defense-against-chinese-cruise-missiles-11636455224\u0026ss=tw\u0026rt=U.S.+Tests+Israel%27s+Iron+Dome+in+Guam+as+Defense+Against+Chinese+Cruise+Missiles+-+WSJ\u0026cd=KhQxNzg2NTc5NDQ3ODIzODUyNjI5NzIcZmQ4N2VjYzkxMGIxMWE4Yzpjby56YTplbjpVUw\u0026ssp=AMJHsmW3CCK1S4TNPifSXszcyaNMwd6TDg" } } ``` @@ -1227,6 +1254,14 @@ An example event for `threat_intel_malware_grid` looks as following: | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.direction | The direction of the message based on the sending and receiving domains. | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | +| email.to.address | The email address of recipient | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | @@ -1249,26 +1284,29 @@ An example event for `threat_intel_malware_grid` looks as following: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| mimecast.created | When the indicator was last created. | date | -| mimecast.hashtype | The hash type. | keyword | -| mimecast.id | The ID of the indicator. | keyword | -| mimecast.labels | The labels related to the indicator. | keyword | -| mimecast.log_type | String to get type of Threat intel feed. | keyword | -| mimecast.modified | When the indicator was last modified. | date | -| mimecast.name | Name of the file. | keyword | -| mimecast.pattern | The pattern. | keyword | -| mimecast.relationship_type | Type of the relationship. | keyword | -| mimecast.source_ref | Source of the reference. | keyword | -| mimecast.target_ref | Reference target. | keyword | -| mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | -| mimecast.valid_from | The valid from date. | date | -| mimecast.value | The value of the indicator. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| mimecast.action | The action that was taken for the click. | keyword | +| mimecast.actions | The actions that were taken. | keyword | +| mimecast.adminOverride | The action defined by the administrator for the URL. | keyword | +| mimecast.category | The category of the URL clicked. | keyword | +| mimecast.creationMethod | The description how event occurred. | keyword | +| mimecast.emailPartsDescription | An array of components of the messge where the URL was found. | keyword | +| mimecast.fromUserEmailAddress | The email of user who triggers the event. | keyword | +| mimecast.messageId | The message-id value of the message. | keyword | +| mimecast.route | The route of the email that contained the link. | keyword | +| mimecast.scanResult | The result of the URL scan. | keyword | +| mimecast.sendingIP | The IP of user who triggers the event. | keyword | +| mimecast.subject | The subject of the email. | keyword | +| mimecast.ttpDefinition | The description of the definition that triggered the URL to be rewritten by Mimecast. | keyword | +| mimecast.url | The url clicked. | keyword | +| mimecast.userAwarenessAction | The action taken by the user if user awareness was applied. | keyword | +| mimecast.userEmailAddress | The email address of the user who clicked the link. | keyword | +| mimecast.userOverride | The action requested by the user. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| related.user | All the user names or other user identifiers seen on the event. | keyword | +| rule.name | The name of the rule or signature generating the event. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | | tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.original.text | Multi-field of `url.original`. | match_only_text | +| user.email | User email address. | keyword | + From 79f69f049e7ab1fedb676d3fd0bc83a4bbd2bc0c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Apr 2022 22:11:38 -0400 Subject: [PATCH 16/18] Add changelog --- packages/mimecast/changelog.yml | 8 ++++++++ packages/mimecast/manifest.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml index 4457e129b2c..7b3075e2342 100644 --- a/packages/mimecast/changelog.yml +++ b/packages/mimecast/changelog.yml @@ -1,3 +1,11 @@ +- version: "0.0.11" + changes: + - description: Update integration description for consistency with other integrations. + type: enhancement + link: https://github.com/elastic/integrations/pull/3193 + - description: Add missing ECS event.* field mappings. + type: bugfix + link: https://github.com/elastic/integrations/pull/3193 - version: "0.0.10" changes: - description: Add more use cases to audit_events pipeline diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml index d7b0ac3a01e..1181a8d8112 100644 --- a/packages/mimecast/manifest.yml +++ b/packages/mimecast/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: mimecast title: "Mimecast" -version: 0.0.10 +version: 0.0.11 license: basic description: "Collect logs from the Mimecast API with Elastic Agent." type: integration From 85e223413c98ff995b998437877b1ef4665338c3 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 26 Apr 2022 08:35:43 -0400 Subject: [PATCH 17/18] siem_logs - set allow_duplicates: false for email addresses --- .../_dev/test/pipeline/test-siem-logs.log-expected.json | 1 - .../siem_logs/elasticsearch/ingest_pipeline/default.yml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json index 23e54d40c8a..352f2a08c15 100644 --- a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json @@ -240,7 +240,6 @@ "direction": "internal", "from": { "address": [ - "johndoe@example.com", "johndoe@example.com" ] }, diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml index 12584402652..7a3fd06dc66 100644 --- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml @@ -65,10 +65,12 @@ processors: - append: field: email.to.address value: "{{{mimecast.Rcpt}}}" + allow_duplicates: false if: "ctx?.mimecast?.Rcpt != null" - append: field: email.from.address value: '{{{mimecast.headerFrom}}}' + allow_duplicates: false if: ctx.mimecast?.headerFrom != null - rename: field: mimecast.RejCode @@ -88,6 +90,7 @@ processors: - append: field: email.from.address value: "{{{mimecast.Sender}}}" + allow_duplicates: false if: "ctx?.mimecast?.Sender != null" - rename: field: mimecast.Subject @@ -232,6 +235,7 @@ processors: - append: field: email.from.address value: "{{{mimecast.sender}}}" + allow_duplicates: false if: "ctx?.mimecast?.sender != null" - rename: field: mimecast.senderDomain From cfa83fd486aa70428f9ccc9e1f4d3be796d3d6bd Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 26 Apr 2022 08:44:25 -0400 Subject: [PATCH 18/18] Remove "with " from readme --- packages/mimecast/_dev/build/docs/README.md | 4 ++-- packages/mimecast/docs/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mimecast/_dev/build/docs/README.md b/packages/mimecast/_dev/build/docs/README.md index bb75b76e96d..6c909a8b42e 100644 --- a/packages/mimecast/_dev/build/docs/README.md +++ b/packages/mimecast/_dev/build/docs/README.md @@ -8,8 +8,8 @@ Authorization parameters for the Mimecast API (`Application Key`, `Application ID`, `Access Key`, and `Secret Key`) should be provided by a Mimecast representative for this integration. Under `Advanced options` you can set the time interval between two API requests as well as the API URL. A Mimecast -representative should also be able to give you with this information in case you -need to change the defaults. +representative should also be able to give you this information in case you need +to change the defaults. Note that rate limit quotas may require you to set up different credentials for the different available log types. diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md index 76ee0cba89f..65406546952 100644 --- a/packages/mimecast/docs/README.md +++ b/packages/mimecast/docs/README.md @@ -8,8 +8,8 @@ Authorization parameters for the Mimecast API (`Application Key`, `Application ID`, `Access Key`, and `Secret Key`) should be provided by a Mimecast representative for this integration. Under `Advanced options` you can set the time interval between two API requests as well as the API URL. A Mimecast -representative should also be able to give you with this information in case you -need to change the defaults. +representative should also be able to give you this information in case you need +to change the defaults. Note that rate limit quotas may require you to set up different credentials for the different available log types.