From 4a28c5fab4a645af4463fdcdfbbb5a12ec778dd1 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 25 Aug 2021 18:36:37 -0500 Subject: [PATCH 01/26] set proposal doc to target stage 2 --- rfcs/text/0010-email.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 98e1efc575..49fab70f26 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -1,8 +1,8 @@ # 0010: Email -- Stage: **1 (draft)** -- Date: **2021-08-16** +- Stage: **2 (candidate)** +- Date: **TBD** This RFC proposes a new top-level field set to facilitate email use cases, `email.*`. The `email.*` field set adds fields for the sender, recipient, message header fields, and other attributes of an email message typically seen logs produced by mail transfer agent (MTA) and email gateway applications. @@ -349,3 +349,4 @@ e.g.: * Stage 1 (formerly proposal stage): https://github.com/elastic/ecs/pull/999 * RFC ID correction: https://github.com/elastic/ecs/pull/1157 * Stage 1 (draft): https://github.com/elastic/ecs/pull/1219 +* Stage 2 (candidate): https://github.com/elastic/ecs/pull/NNNN From 9a935fc5cd6da99cc8d58f49be6015e40e94ec4b Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 25 Aug 2021 18:38:08 -0500 Subject: [PATCH 02/26] update candidate field set to match current proposal --- rfcs/text/0010/email.yml | 206 +++++++++++++++++++++++++++------------ 1 file changed, 144 insertions(+), 62 deletions(-) diff --git a/rfcs/text/0010/email.yml b/rfcs/text/0010/email.yml index 2d98a88061..040e3d9c23 100644 --- a/rfcs/text/0010/email.yml +++ b/rfcs/text/0010/email.yml @@ -1,116 +1,198 @@ ---- - name: email title: Email group: 2 - short: Fields describing an email message. + short: Fields describing an email transaction. description: > - Email fields are used for information about an email message header and body. + Fields relating to an email transaction. + + This field set focuses on the email message header, body, and attachments. Network protocols that send and receive + email messages such as SMTP are outside the scope of the `email.*` fields. type: group fields: - - name: to + - name: attachments + level: extended + type: nested + short: List of objects describing the attachments. + description: > + A list of attachment files sent along with an email message. + + - name: attachments.file.extension level: extended type: keyword - short: Recipient address(es) + short: Attachment file extension. description: > - Stores the `to` email address(es). - example: "employee@example.com" - normalize: - - array + Attachment file extension, excluding the leading dot. + example: "txt" - - name: from + - name: attachments.file.mime_type level: extended type: keyword - short: Sender address + short: MIME type of the attachment file. description: > - Stores the `from` email address. - example: "administrator@example.com" + The MIME media type of the attachment. - - name: cc + This value will typically be extracted from the `Content-Type` MIME header field. + example: "text/plain" + + - name: attachments.file.name level: extended type: keyword - short: CC recipient(s) + short: Name of the attachment file. description: > - The email address(es) of the carbon copy (CC) recipient(s). - example: '["cc.user@example.com", "cc.user2@example.com"]' + Name of the attachment file including the file extension. + example: "attachment.txt" + + - name: attachments.file.size + level: extended + type: long + short: Attachment file size. + description: > + Attachment file size in bytes. + example: 64329 + + - name: attachments.hash.md5 + level: extended + type: keyword + short: MD5 hash of the attachment. + description: > + MD5 hash of the attachment file. + example: "e25f1c98ffdacf611473af364362ec48" + + - name: attachments.hash.sha1 + level: extended + type: keyword + short: SHA-1 hash of the attachment. + description: > + SHA-1 hash of the attachment file. + example: "8c1cd40f17109b427e61d4e72ca6d9a4fc8175f3" + + - name: attachments.hash.sha256 + level: extended + type: keyword + short: SHA-256 hash of the attachment. + description: > + SHA-256 hash of the attachment file. + example: "f0366b3559f577d8732f7e9cc343a4960d202e8137dcc42f9783f3963f6abc6a" + + - name: bcc + level: extended + type: keyword + short: Email address(es) of BCC recipients + description: > + The email address(es) of the blind carbon carbon (BCC) recipients. + example: "['bcc.user1@example.com', 'bcc.user2@example.com']" + normalize: - array - - name: bcc + - name: cc level: extended type: keyword - short: BCC recipient(s) + short: Email address(es) of CC recipients description: > - The email address(es) of the blind carbon copy (BCC) recipient(s). - example: '["bcc.user@example.com", "bcc.user2@example.com"]' + The email address(es) of the carbon carbon (BCC) recipients. + example: "['cc.user1@example.com', 'cc.user2@example.com']" + normalize: - array - - name: reply_to + - name: content_type level: extended type: keyword - short: Address for replies. + short: MIME type of the email message. description: > - The address that replies should be delivered to. - example: "user@example.com" + Information about how the message is to be displayed. + + Typically a MIME type. + example: "text/plain" - - name: timestamp + - name: delivery_timestamp level: extended type: date - short: Date and time message was sent. + short: Date and time when message was delivered. description: > - The date and time that the sender authorized delivery of the message. - - For example, by pressing a "Send" button in their email client. + The date and time when the email message was received by the service or client. example: "2020-11-10T22:12:34.8196921Z" - - name: subject + - name: direction level: extended type: keyword - short: Topic of the message + short: Direction of the message. description: > - A brief summary of the topic of the message. - example: "Status update: email fields progress" - multi_fields: - - type: match_only_text - name: text + The direction of the message based on the sending and receiving domains. + example: inbound - - name: content_type + - name: from level: extended type: keyword - short: Content-Type header value + short: The sender's email address. description: > - Information about how the message is to be displayed. Typically a MIME type. - example: "multipart/mixed" + The email address of the sender, typically from the RFC 5322 `From:` header field. + example: "sender@example.com" + + - name: local_id + level: extended + type: keyword + short: Unique identifier given by the source. + description: > + Unique identifier given to the email by the source that created the event. + + Identifier is not persistent across hops. + example: "c26dbea0-80d5-463b-b93c-4e8b708219ce" - name: message_id level: extended - type: wildcard - short: Unique identifier of the message + type: keyword + short: Value from the Message-ID header. description: > - Unique identifier for the email message that refers to a particular version of a particular message. + Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email + message. + example: "<81ce15$8r2j59@mail01.example.com>" - An example would be the identifier found in the `Message-ID` email header. - example: "" + - name: origination_timestamp + level: extended + type: date + short: Date and time the email was composed. + description: > + The date and time the email message was composed. Many email clients will fill in this value + automatically when the message is sent by a user. + example: "2020-11-10T22:12:34.8196921Z" - - name: direction + - name: reply_to level: extended type: keyword - short: Direction of the email message + short: Address replies should be delivered to. description: > - The direction of the message based on the sending and received domains. + The address that replies should be delivered to based on the value in the RFC 5322 `Reply-To:` header. + example: "reply.here@example.com" - Recommended values are: - * `inbound` - From external senders to internal recipients - * `outbound` - From internal senders to external recipients - * `internal` - From internal senders to internal recipients - * `external` - From external senders to external recipients - * `unknown` - Direction is unknown + - name: subject + level: extended + type: keyword + short: The subject of the email message. + description: > + A brief summary of the topic of the message. + example: "Please see this important message." + multi_fields: + - type: match_only_text + name: text - - name: x_mailer - level: extended - type: keyword - short: The value from the X-Mailer header - description: > - The value from the `X-Mailer` header. Value captures what application was used - to draft and send the original email message. + - name: to + level: extended + type: keyword + short: Email address(es) of the recipients. + description: > + The email address(es) of the message recipients. + example: "['user1@example.com', 'user2@example.com']" + + normalize: + - array + + - name: x_mailer + level: extended + type: keyword + short: Application that drafted email. + description: > + The name of the application that was used to draft and send the original email message. + example: "Spambot v2.5" From f715bd8261668bd850bf17c3a31cb4400a02f97c Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 25 Aug 2021 18:41:03 -0500 Subject: [PATCH 03/26] set pr number in link --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 49fab70f26..ef747cfe64 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -349,4 +349,4 @@ e.g.: * Stage 1 (formerly proposal stage): https://github.com/elastic/ecs/pull/999 * RFC ID correction: https://github.com/elastic/ecs/pull/1157 * Stage 1 (draft): https://github.com/elastic/ecs/pull/1219 -* Stage 2 (candidate): https://github.com/elastic/ecs/pull/NNNN +* Stage 2 (candidate): https://github.com/elastic/ecs/pull/1593 From aeb8b7e2da67ff6584f3c1bde3024a5fbd02a1fc Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 17 Nov 2021 17:36:21 -0600 Subject: [PATCH 04/26] propose nested objects for tracking email address with display name --- rfcs/text/0010-email.md | 12 +++++-- rfcs/text/0010/email.yml | 76 ++++++++++++++++++++++++++++++++++------ 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index ef747cfe64..e37c786331 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -19,10 +19,16 @@ Stage 1: Describe at a high level how this change affects fields. Which fieldset | `email.from` | keyword | Stores the `from` email address from the RFC5322 `From:` header field. | | `email.origination_timestamp` | date | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | | `email.delivery_timestamp` | date | The date and time the email message was received by the service or client. | -| `email.to` | keyword (array) | The email address(es) of the message recipient(s) | +| `email.to` | nested | Nested object with the message recipient(s) | +| `email.to.address` | keyword | The email address of message recipient | +| `email.to.display_name` | keyword | The display name of message recipient | | `email.subject` | keyword; `.text` text multi-field | A brief summary of the topic of the message | -| `email.cc` | keyword (array) | The email address(es) of the carbon copy (CC) recipient(s) | -| `email.bcc` | keyword (array) | The email address(es) of the blind carbon copy (CC) recipient(s) | +| `email.cc` | nested | Nested object with the carbon copy (CC) recipient(s) | +| `email.cc.address` | keyword | The email address of a carbon copy (CC) recipient | +| `email.cc.display_name` | keyword | The display name of carbon copy (CC) recipient | +| `email.bcc` | nested | Nested object with the blind carbon copy (CC) recipient(s) | +| `email.bcc.address` | keyword | The email address of the blind carbon copy (CC) recipient(s) | +| `email.bcc.display_name` | keyword | The display name of the blind carbon copy (CC) recipient(s) | | `email.content_type` | keyword | Information about how the message is to be displayed. Typically a MIME type | | `email.message_id` | wildcard | Identifier from the RFC5322 `Message-ID:` header field that refers to a particular version of a particular message. | | `email.local_id` | keyword | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the `X-MS-Exchange-Organization-Network-Message-Id` id). | diff --git a/rfcs/text/0010/email.yml b/rfcs/text/0010/email.yml index 040e3d9c23..40414d8ec5 100644 --- a/rfcs/text/0010/email.yml +++ b/rfcs/text/0010/email.yml @@ -17,6 +17,9 @@ description: > A list of attachment files sent along with an email message. + normalize: + - array + - name: attachments.file.extension level: extended type: keyword @@ -78,7 +81,7 @@ - name: bcc level: extended type: keyword - short: Email address(es) of BCC recipients + short: BCC recipients description: > The email address(es) of the blind carbon carbon (BCC) recipients. example: "['bcc.user1@example.com', 'bcc.user2@example.com']" @@ -86,17 +89,56 @@ normalize: - array - - name: cc + - name: bcc + level: extended + type: nested + short: List of objects describing BCC recipients + description: > + The email address(es) of the blind carbon carbon (BCC) recipients. + normalize: + - array + + - name: bcc.address level: extended type: keyword - short: Email address(es) of CC recipients + short: Email address of BCC recipient description: > - The email address(es) of the carbon carbon (BCC) recipients. - example: "['cc.user1@example.com', 'cc.user2@example.com']" + The email address of BCC recipient + example: "bcc.user1@example.com" + + - name: bcc.display_name + level: extended + type: keyword + short: Display name of BCC recipient + description: > + The display name of the BCC recipient + example: "BCC User" + - name: cc + level: extended + type: nested + short: List of objects describing CC recipients + description: > + The email address(es) of the carbon carbon (BCC) recipients. normalize: - array + - name: cc.address + level: extended + type: keyword + short: Email address of CC recipient + description: > + The email address of CC recipient + example: "cc.user1@example.com" + + - name: cc.display_name + level: extended + type: keyword + short: Display name of CC recipient + description: > + The display name of CC recipient + example: "CC User" + - name: content_type level: extended type: keyword @@ -180,15 +222,29 @@ - name: to level: extended - type: keyword - short: Email address(es) of the recipients. + type: nested + short: List of objects describing recipients. description: > - The email address(es) of the message recipients. - example: "['user1@example.com', 'user2@example.com']" - + The email addresses of message recipients. normalize: - array + - name: to.address + level: extended + type: keyword + short: Email address of recipient + description: > + The email address of recipient + example: "user1@example.com" + + - name: to.display_name + level: extended + type: keyword + short: Display name of recipient + description: > + The display name of recipient + example: "Example User" + - name: x_mailer level: extended type: keyword From b5df3f23dbfd00b44b5034bd9d65e3e6d98a948e Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 13:48:57 -0600 Subject: [PATCH 05/26] also add display_name to from --- rfcs/text/0010-email.md | 13 ++++++++----- rfcs/text/0010/email.yml | 10 +++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index e37c786331..a3935041da 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -16,7 +16,8 @@ Stage 1: Describe at a high level how this change affects fields. Which fieldset | field | type | description | | --- | --- | --- | -| `email.from` | keyword | Stores the `from` email address from the RFC5322 `From:` header field. | +| `email.from.address` | keyword | Stores the `from` email address from the RFC5322 `From:` header field. | +| `email.from.display_name` | keyword | Stores the display name of the `from` address. | | `email.origination_timestamp` | date | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | | `email.delivery_timestamp` | date | The date and time the email message was received by the service or client. | | `email.to` | nested | Nested object with the message recipient(s) | @@ -104,7 +105,9 @@ Stage 2: Included a real world example source document. Ideally this example com "o365mc@microsoft.com" ], "to": [ - "john@testdomain.onmicrosoft.com" + { + "address": "john@testdomain.onmicrosoft.com" + } ], "subject": "Weekly digest: Microsoft service updates", "message_id": "\\u003c95689d8d5e7f429390a4e3646eef75e8-JFBVALKQOJXWILKBK4YVA7APGM3DKTLFONZWCZ3FINSW45DFOJ6EAQ2ENFTWK43UL4YTCMBYGIYHYU3NORYA====@microsoft.com\\u003e" @@ -150,9 +153,9 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-10T22:12:34.8196921Z", - "from": [ - "postmaster@testdomain.onmicrosoft.com" - ], + "from": { + "address": "postmaster@testdomain.onmicrosoft.com" + }, "to": [ "o365mc@microsoft.com" ], diff --git a/rfcs/text/0010/email.yml b/rfcs/text/0010/email.yml index 40414d8ec5..a1a50a9bc9 100644 --- a/rfcs/text/0010/email.yml +++ b/rfcs/text/0010/email.yml @@ -165,7 +165,7 @@ The direction of the message based on the sending and receiving domains. example: inbound - - name: from + - name: from.address level: extended type: keyword short: The sender's email address. @@ -173,6 +173,14 @@ The email address of the sender, typically from the RFC 5322 `From:` header field. example: "sender@example.com" + - name: from.display_name + level: extended + type: keyword + short: The sender's display name. + description: > + The display name of the sender. + example: "Sender" + - name: local_id level: extended type: keyword From 5a2cd59e6dbd42f69582a7ff8bf43dffd072c136 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 13:54:16 -0600 Subject: [PATCH 06/26] update examples to use new from, to, cc, and bcc fields --- rfcs/text/0010-email.md | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index a3935041da..92f28bf8ab 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -157,7 +157,9 @@ Stage 2: Included a real world example source document. Ideally this example com "address": "postmaster@testdomain.onmicrosoft.com" }, "to": [ - "o365mc@microsoft.com" + { + "address": "o365mc@microsoft.com" + } ], "subject": "Undeliverable: Message Center Major Change Update Notification", "message_id": "\\u003c72872e16-f4c2-4eef-a393-e5621748a0ff@AS8P19vMB1605.EURP191.PROD.OUTLOOK.COM\\u003e" @@ -191,15 +193,25 @@ Stage 2: Included a real world example source document. Ideally this example com "message_id": "20160624211145.62086.mail@evil.zz", "local_id": "c26dbea0-80d5-463b-b93c-4e8b708219ce", "to": [ - "clark.kent@pharmtech.zz", - "diana.prince@pharmtech.zz" + { + "address": "clark.kent@pharmtech.zz", + "display_name": "Clark Kent" + }, + { + "address": "diana.prince@pharmtech.zz", + "display_name": "Diana Prince" + } ], "cc": [ - "bruce.wayne@university-of-education.zz" - ], - "from": [ - "badguy@evil.zz" + { + "address": "bruce.wayne@university-of-education.zz", + "display_name": "Bruce Wayne" + } ], + "from": { + "address": "badguy@evil.zz", + "display_name": "A. Badguy" + }, "subject": "Please find a totally safe invoice attached.", "reply_to": "null", "x_mailer": "Spambot v2.5", @@ -249,11 +261,13 @@ datetime=2017-05-26T16:47:41+0100|aCode=7O7I7MvGP1mj8plHRDuHEA|acc=C0A0|SpamLimi }, "email": { "message_id": "<81ce15$8r2j59@mail01.example.com>", - "from": [ - "from@mimecast.com" - ], + "from": { + "address": "from@mimecast.com" + }, "to": [ - "auser@mimecast.com" + { + "address": "auser@mimecast.com" + } ], "subject": "message subject", "direction": "internal" From e260c2d91c7b4246b61caaa642fc3ed8a613a701 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 13:54:29 -0600 Subject: [PATCH 07/26] typo --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 92f28bf8ab..e761b6fa7f 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -306,7 +306,7 @@ The fields proposed in this document are focused on the contents of an email mes For example, users may need to compare the email address from the SMTP (envelope) sender to the `From:` header email address. -### Email metrics and observability use caes +### Email metrics and observability use cases Does the initial set of `email` fields need to consider observability and email monitoring use cases, for example spam, metrics, deliverables, and logging. From 924b4e7b4d0c75c66b7ecd9f6a681b5b7f9ab007 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 13:56:56 -0600 Subject: [PATCH 08/26] removing unneeded comments --- rfcs/text/0010-email.md | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index e761b6fa7f..80c6b8ecb3 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -1,5 +1,4 @@ # 0010: Email - - Stage: **2 (candidate)** - Date: **TBD** @@ -8,10 +7,6 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai ## Fields - - ### Email specific fields | field | type | description | @@ -51,18 +46,10 @@ Email events may benefit from an additional ECS allowed event categorization val ## Usage - - Email use cases stretch across all three Elastic solutions - Search, Observe, Protect. Whether it's searching for content within email, ensuring email infrastructure is operational or detecting email based attacks, there are many possibilities for email fields within ECS. ## Source data - - - **Email Analytics**: [Hubspot](https://legacydocs.hubspot.com/docs/methods/email/email_events_overview), Marketo, Salesforce Pardot - **Email Server**: [O365 Message Tracing](https://docs.microsoft.com/en-us/exchange/monitoring/trace-an-email-message/run-a-message-trace-and-view-results), [Postfix](https://nxlog.co/documentation/nxlog-user-guide/postfix.html) - **Email Security**: [Barracuda](https://campus.barracuda.com/product/emailsecuritygateway/doc/12193950/syslog-and-the-barracuda-email-security-gateway/), [Forcepoint](https://www.websense.com/content/support/library/email/v85/email_siem/siem_log_map.pdf), [Mimecast](https://www.mimecast.com/tech-connect/documentation/tutorials/understanding-siem-logs/), [Proofpoint](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API) @@ -296,10 +283,6 @@ The goal here is to research and understand the impact of these changes on users ## Concerns - - ### Email messages vs. protocols The fields proposed in this document are focused on the contents of an email message but not on specific fields for email protocols. Do protocols like SMTP, POP3, IMAP, etc. be represented in ECS? @@ -340,19 +323,6 @@ The following are the people that consulted on the contents of this RFC. * @devonakerr | Co-sponsor - - - ## References From 2a40ceb980ee56c54e0dc6d2e546f9e51535dfd0 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 14:49:46 -0600 Subject: [PATCH 09/26] clean up examples --- rfcs/text/0010-email.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 80c6b8ecb3..03beac2851 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -88,9 +88,9 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-08T22:12:34.8196921Z", - "from": [ - "o365mc@microsoft.com" - ], + "from": { + "address": "o365mc@microsoft.com" + }, "to": [ { "address": "john@testdomain.onmicrosoft.com" @@ -223,8 +223,8 @@ Stage 2: Included a real world example source document. Ideally this example com "action": "MSGBLK" }, "source": { - "address": 192.0.2.255, - "ip": 192.0.2.255 + "address": "192.0.2.255", + "ip": "192.0.2.255" } } ``` @@ -243,8 +243,8 @@ datetime=2017-05-26T16:47:41+0100|aCode=7O7I7MvGP1mj8plHRDuHEA|acc=C0A0|SpamLimi { "@timestamp": "2017-05-26T16:47:41+0100", "source": { - "address": 123.123.123.123, - "ip": 123.123.123.123 + "address": "123.123.123.123", + "ip": "123.123.123.123" }, "email": { "message_id": "<81ce15$8r2j59@mail01.example.com>", From b889936bd70c85c57315455141d37e716b96bd3e Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 15:06:12 -0600 Subject: [PATCH 10/26] clean up grammar --- rfcs/text/0010-email.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 03beac2851..2045e1e3c0 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -46,7 +46,7 @@ Email events may benefit from an additional ECS allowed event categorization val ## Usage -Email use cases stretch across all three Elastic solutions - Search, Observe, Protect. Whether it's searching for content within email, ensuring email infrastructure is operational or detecting email based attacks, there are many possibilities for email fields within ECS. +Email use cases stretch across all three Elastic solutions - Search, Observe, Protect. Whether it's searching for content within email, ensuring email infrastructure is operational, or detecting email-based attacks, there are many possibilities for email fields within ECS. ## Source data @@ -285,21 +285,21 @@ The goal here is to research and understand the impact of these changes on users ### Email messages vs. protocols -The fields proposed in this document are focused on the contents of an email message but not on specific fields for email protocols. Do protocols like SMTP, POP3, IMAP, etc. be represented in ECS? +The fields proposed in this document focus on an email message's content but not on specific fields for email protocols. However, should protocols like SMTP, POP3, IMAP, etc., be represented in ECS? For example, users may need to compare the email address from the SMTP (envelope) sender to the `From:` header email address. ### Email metrics and observability use cases -Does the initial set of `email` fields need to consider observability and email monitoring use cases, for example spam, metrics, deliverables, and logging. +Does the initial set of `email` fields need to consider observability and email monitoring use cases, for example, spam, metrics, deliverables, and logging? ### Additional event categorization values -Should a new event.category field (email) be created, and, if so, which `event.type` values the `email` category should be combined with? +Should a new event.category field (email) be created, and, if so, which `event.type` values should be used for the `email` category? ### Display names -Should the display name be captured separately from the email address for senders and recipients. If so, how do we accomplish this in a document while keeping the 1:1 of a display name to email address. +Should the display name be captured separately from the email address for senders and recipients? If so, how do we accomplish this in a document while keeping the 1:1 of a display name to email address? ### Spam processing details From 9d309deea11fe78d3ff8e04411970abe588bc0f9 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 15:31:24 -0600 Subject: [PATCH 11/26] capturing scope of impact --- rfcs/text/0010-email.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 2045e1e3c0..ebcb7c6c15 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -273,13 +273,9 @@ Stage 3: Add more real world example source documents so we have at least 2 tota ## Scope of impact - +This is a new field set, and the changes introduced will not affect existing ECS implementations. + +Integrations or other data sources mapping to ECS will need to map their original events to the new fields. ## Concerns From de350cf54725f22a71508d475e09aaeaff771e28 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 15:47:00 -0600 Subject: [PATCH 12/26] address concerns --- rfcs/text/0010-email.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index ebcb7c6c15..9989e63cd0 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -285,22 +285,32 @@ The fields proposed in this document focus on an email message's content but not For example, users may need to compare the email address from the SMTP (envelope) sender to the `From:` header email address. +**Resolution**: Focus on email message content in this initial phase. Additional protocol details can be added later on. + ### Email metrics and observability use cases Does the initial set of `email` fields need to consider observability and email monitoring use cases, for example, spam, metrics, deliverables, and logging? +**Resolution**: This initial field set focuses on email message content. + ### Additional event categorization values Should a new event.category field (email) be created, and, if so, which `event.type` values should be used for the `email` category? +**Resolution**: Propose to add `event.category: email` and make `info` an expected event type for the category. + ### Display names Should the display name be captured separately from the email address for senders and recipients? If so, how do we accomplish this in a document while keeping the 1:1 of a display name to email address? +**Resolution**: Use `nested` types to allows arrays of objects containing both the email address and display name for the `to`, `cc`, and `bcc` recipients. + ### Spam processing details Should fields intended to capture details around spam processing like sender policy framework (SPF), domainkeys identified mail (DKIM), or domain-based message authentication, reporting, and conformance (DMARC) be in scope for this proposal as well? +**Resolution**: This initial field set focuses on email message content. + From 5ffd9d90c939fa1d3fff062ba14b7a6c89b3726e Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Nov 2021 15:58:11 -0600 Subject: [PATCH 13/26] typo in the field table --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 9989e63cd0..bb2336ea83 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -18,7 +18,7 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | `email.to` | nested | Nested object with the message recipient(s) | | `email.to.address` | keyword | The email address of message recipient | | `email.to.display_name` | keyword | The display name of message recipient | -| `email.subject` | keyword; `.text` text multi-field | A brief summary of the topic of the message | +| `email.subject` | keyword | `.text` text multi-field | A brief summary of the topic of the message | | `email.cc` | nested | Nested object with the carbon copy (CC) recipient(s) | | `email.cc.address` | keyword | The email address of a carbon copy (CC) recipient | | `email.cc.display_name` | keyword | The display name of carbon copy (CC) recipient | From 0133a74574df3c2b306e8744aaf343dcb45f7664 Mon Sep 17 00:00:00 2001 From: djptek Date: Mon, 22 Nov 2021 10:34:59 +0100 Subject: [PATCH 14/26] Convert `from` to a Nested Object and rename `email.reply_to` to `email.reply_to.address` for consistency with the other `*.address` fields --- rfcs/text/0010-email.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index bb2336ea83..9f4b3ab5d9 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -11,8 +11,11 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | field | type | description | | --- | --- | --- | +| `email.from` | nested | Nested object with the message senders(s) | | `email.from.address` | keyword | Stores the `from` email address from the RFC5322 `From:` header field. | | `email.from.display_name` | keyword | Stores the display name of the `from` address. | +| `email.sender.address` | keyword | When the `from` field contains more than one address or the `sender` and `from` are distinct then this field is populated. | +| `email.sender.display_name` | keyword | Stores the display name of the `sender` if present. | | `email.origination_timestamp` | date | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | | `email.delivery_timestamp` | date | The date and time the email message was received by the service or client. | | `email.to` | nested | Nested object with the message recipient(s) | @@ -28,7 +31,7 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | `email.content_type` | keyword | Information about how the message is to be displayed. Typically a MIME type | | `email.message_id` | wildcard | Identifier from the RFC5322 `Message-ID:` header field that refers to a particular version of a particular message. | | `email.local_id` | keyword | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the `X-MS-Exchange-Organization-Network-Message-Id` id). | -| `email.reply_to` | keyword | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | +| `email.reply_to.address` | keyword | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | | `email.direction` | keyword | Direction of the message based on the sending and receiving domains | | `email.x_mailer` | keyword | What application was used to draft and send the original email. | | `email.attachments` | nested | Nested object of attachments on the email. | From 0ac04cf9e4cb916f39b19906f3b6b989a8fdfa95 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 22 Nov 2021 15:47:35 -0600 Subject: [PATCH 15/26] include email.reply_to.display_name --- rfcs/text/0010-email.md | 1 + 1 file changed, 1 insertion(+) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 9f4b3ab5d9..c617c685ba 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -32,6 +32,7 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | `email.message_id` | wildcard | Identifier from the RFC5322 `Message-ID:` header field that refers to a particular version of a particular message. | | `email.local_id` | keyword | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the `X-MS-Exchange-Organization-Network-Message-Id` id). | | `email.reply_to.address` | keyword | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | +| `email.reply_to.display_name` | keyword | The display name included with the `Reply-To` address. | | `email.direction` | keyword | Direction of the message based on the sending and receiving domains | | `email.x_mailer` | keyword | What application was used to draft and send the original email. | | `email.attachments` | nested | Nested object of attachments on the email. | From 69c45799ff39b29c36466107cf298c2fb33bf6de Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 22 Nov 2021 15:49:16 -0600 Subject: [PATCH 16/26] update examples to make from values into arrays --- rfcs/text/0010-email.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index c617c685ba..243d8c5d53 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -92,9 +92,11 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-08T22:12:34.8196921Z", - "from": { + "from": [ + { "address": "o365mc@microsoft.com" - }, + } + ], "to": [ { "address": "john@testdomain.onmicrosoft.com" @@ -144,9 +146,11 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-10T22:12:34.8196921Z", - "from": { - "address": "postmaster@testdomain.onmicrosoft.com" - }, + "from": [ + { + "address": "postmaster@testdomain.onmicrosoft.com" + } + ], "to": [ { "address": "o365mc@microsoft.com" @@ -199,10 +203,12 @@ Stage 2: Included a real world example source document. Ideally this example com "display_name": "Bruce Wayne" } ], - "from": { - "address": "badguy@evil.zz", - "display_name": "A. Badguy" - }, + "from": [ + { + "address": "badguy@evil.zz", + "display_name": "A. Badguy" + } + ], "subject": "Please find a totally safe invoice attached.", "reply_to": "null", "x_mailer": "Spambot v2.5", @@ -252,9 +258,11 @@ datetime=2017-05-26T16:47:41+0100|aCode=7O7I7MvGP1mj8plHRDuHEA|acc=C0A0|SpamLimi }, "email": { "message_id": "<81ce15$8r2j59@mail01.example.com>", - "from": { - "address": "from@mimecast.com" - }, + "from": [ + { + "address": "from@mimecast.com" + } + ], "to": [ { "address": "auser@mimecast.com" From c57cc32a8ddb06c222e66391e551735a25e5842a Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 22 Nov 2021 16:11:19 -0600 Subject: [PATCH 17/26] add new fields to the proposed field defs --- rfcs/text/0010/email.yml | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/rfcs/text/0010/email.yml b/rfcs/text/0010/email.yml index a1a50a9bc9..da838c6eb7 100644 --- a/rfcs/text/0010/email.yml +++ b/rfcs/text/0010/email.yml @@ -165,6 +165,15 @@ The direction of the message based on the sending and receiving domains. example: inbound + - name: from + level: extended + type: nested + short: List of objects describing sender. + description: > + The email addresses of message sender. + normalize: + - array + - name: from.address level: extended type: keyword @@ -209,7 +218,7 @@ automatically when the message is sent by a user. example: "2020-11-10T22:12:34.8196921Z" - - name: reply_to + - name: reply_to.address level: extended type: keyword short: Address replies should be delivered to. @@ -217,6 +226,28 @@ The address that replies should be delivered to based on the value in the RFC 5322 `Reply-To:` header. example: "reply.here@example.com" + - name: reply_to.display_name + level: extended + type: keyword + short: Display name included with reply-to address. + description: > + The display name included with the `Reply-To` address. + + - name: sender.address + level: extended + type: keyword + short: Address of the message sender. + description: > + Per RFC 5322, specifies the address responsible for the actual transmission of + the message. + + - name: sender.display_name + level: extended + type: keyword + short: Display name of the sending address. + description: > + The display name of the `Sender:`, if included. + - name: subject level: extended type: keyword From 5374eecaa737bd2455b297e88bb9b0aa19ea2858 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 22 Nov 2021 16:33:43 -0600 Subject: [PATCH 18/26] tweak examples --- rfcs/text/0010-email.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 243d8c5d53..0331a50f8d 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -209,8 +209,13 @@ Stage 2: Included a real world example source document. Ideally this example com "display_name": "A. Badguy" } ], + "sender": { + "address": "e99d7ed5580193f36a51f597bc2c0210@evil.zz" + }, "subject": "Please find a totally safe invoice attached.", - "reply_to": "null", + "reply_to": { + "address": "null" + }, "x_mailer": "Spambot v2.5", "attachments": [ { From 964b0b46778a1cd17f7d6b4d94e81705d17a3cee Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 30 Nov 2021 11:02:08 -0600 Subject: [PATCH 19/26] revert nested address fields --- rfcs/text/0010-email.md | 122 +++++++++++++++++---------------------- rfcs/text/0010/email.yml | 90 ++--------------------------- 2 files changed, 59 insertions(+), 153 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 0331a50f8d..4b183075c6 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -11,28 +11,18 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | field | type | description | | --- | --- | --- | -| `email.from` | nested | Nested object with the message senders(s) | | `email.from.address` | keyword | Stores the `from` email address from the RFC5322 `From:` header field. | -| `email.from.display_name` | keyword | Stores the display name of the `from` address. | | `email.sender.address` | keyword | When the `from` field contains more than one address or the `sender` and `from` are distinct then this field is populated. | -| `email.sender.display_name` | keyword | Stores the display name of the `sender` if present. | | `email.origination_timestamp` | date | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | | `email.delivery_timestamp` | date | The date and time the email message was received by the service or client. | -| `email.to` | nested | Nested object with the message recipient(s) | | `email.to.address` | keyword | The email address of message recipient | -| `email.to.display_name` | keyword | The display name of message recipient | | `email.subject` | keyword | `.text` text multi-field | A brief summary of the topic of the message | -| `email.cc` | nested | Nested object with the carbon copy (CC) recipient(s) | | `email.cc.address` | keyword | The email address of a carbon copy (CC) recipient | -| `email.cc.display_name` | keyword | The display name of carbon copy (CC) recipient | -| `email.bcc` | nested | Nested object with the blind carbon copy (CC) recipient(s) | | `email.bcc.address` | keyword | The email address of the blind carbon copy (CC) recipient(s) | -| `email.bcc.display_name` | keyword | The display name of the blind carbon copy (CC) recipient(s) | | `email.content_type` | keyword | Information about how the message is to be displayed. Typically a MIME type | | `email.message_id` | wildcard | Identifier from the RFC5322 `Message-ID:` header field that refers to a particular version of a particular message. | | `email.local_id` | keyword | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the `X-MS-Exchange-Organization-Network-Message-Id` id). | | `email.reply_to.address` | keyword | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | -| `email.reply_to.display_name` | keyword | The display name included with the `Reply-To` address. | | `email.direction` | keyword | Direction of the message based on the sending and receiving domains | | `email.x_mailer` | keyword | What application was used to draft and send the original email. | | `email.attachments` | nested | Nested object of attachments on the email. | @@ -68,7 +58,7 @@ Stage 2: Included a real world example source document. Ideally this example com ```json { - "EndDate": "2020-11-10T22:12:34.8196921Z", + "EndDate": "2021-11-10T22:12:34.8196921Z", "FromIP": "8.8.8.8", "Index": 25, "MessageId": "\\u003c95689d8d5e7f429390a4e3646eef75e8-JFBVALKQOJXWILKBK4YVA7APGM3DKTLFONZWCZ3FINSW45DFOJ6EAQ2ENFTWK43UL4YTCMBYGIYHYU3NORYA====@microsoft.com\\u003e", @@ -92,16 +82,16 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-08T22:12:34.8196921Z", - "from": [ - { - "address": "o365mc@microsoft.com" - } - ], - "to": [ - { - "address": "john@testdomain.onmicrosoft.com" - } - ], + "from": { + "address": [ + "o365mc@microsoft.com" + ] + }, + "to": { + "address": [ + "john@testdomain.onmicrosoft.com" + ] + }, "subject": "Weekly digest: Microsoft service updates", "message_id": "\\u003c95689d8d5e7f429390a4e3646eef75e8-JFBVALKQOJXWILKBK4YVA7APGM3DKTLFONZWCZ3FINSW45DFOJ6EAQ2ENFTWK43UL4YTCMBYGIYHYU3NORYA====@microsoft.com\\u003e" }, @@ -146,16 +136,16 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-10T22:12:34.8196921Z", - "from": [ - { - "address": "postmaster@testdomain.onmicrosoft.com" - } - ], - "to": [ - { - "address": "o365mc@microsoft.com" - } - ], + "from": { + "address": [ + "postmaster@testdomain.onmicrosoft.com" + ] + }, + "to": { + "address": [ + "o365mc@microsoft.com" + ] + }, "subject": "Undeliverable: Message Center Major Change Update Notification", "message_id": "\\u003c72872e16-f4c2-4eef-a393-e5621748a0ff@AS8P19vMB1605.EURP191.PROD.OUTLOOK.COM\\u003e" }, @@ -175,40 +165,34 @@ Stage 2: Included a real world example source document. Ideally this example com #### Original log ``` -<38>1 2016-06-24T21:00:08Z - ProofpointTAP - MSGBLK [tapmsg@21139 messageTime="2016-06-24T21:18:38.000Z" messageID="20160624211145.62086.mail@evil.zz" recipient="clark.kent@pharmtech.zz, diana.prince@pharmtech.zz" sender="e99d7ed5580193f36a51f597bc2c0210@evil.zz" senderIP="192.0.2.255" phishScore="46" spamScore="4" QID="r2FNwRHF004109" GUID="c26dbea0-80d5-463b-b93c-4e8b708219ce" subject="Please find a totally safe invoice attached." quarantineRule="module.sandbox.threat" quarantineFolder="Attachment Defense" policyRoutes="default_inbound,executives" modulesRun="sandbox,urldefense,spam,pdr" headerFrom="\"A. Badguy\" " headerTo="\"Clark Kent\" ; \"Diana Prince\" " headerCC="\"Bruce Wayne\" " headerReplyTo="null" toAddresses="clark.kent@pharmtech.zz,diana.prince@pharmtech.zz" ccAddresses="bruce.wayne@university-of-education.zz" fromAddress="badguy@evil.zz" replyToAddress="null" clusterId="pharmtech_hosted" messageParts="[{\"contentType\":\"text/plain\",\"disposition\":\"inline\",\"filename\":\"text.txt\",\"md5\":\"008c5926ca861023c1d2a36653fd88e2\",\"oContentType\":\"text/plain\",\"sandboxStatus\":\"unsupported\",\"sha256\":\"85738f8f9a7f1b04b5329c590ebcb9e425925c6d0984089c43a022de4f19c281\"},{\"contentType\":\"application/pdf\",\"disposition\":\"attached\",\"filename\":\"Invoice for Pharmtech.pdf\",\"md5\":\"5873c7d37608e0d49bcaa6f32b6c731f\",\"oContentType\":\"application/pdf\",\"sandboxStatus\":\"threat\",\"sha256\":\"2fab740f143fc1aa4c1cd0146d334c5593b1428f6d062b2c406e5efe8abe95ca\"}]" xmailer="Spambot v2.5"] +<38>1 2021-06-24T21:00:08Z - ProofpointTAP - MSGBLK [tapmsg@21139 messageTime="2021-06-24T21:18:38.000Z" messageID="20160624211145.62086.mail@evil.zz" recipient="clark.kent@pharmtech.zz, diana.prince@pharmtech.zz" sender="e99d7ed5580193f36a51f597bc2c0210@evil.zz" senderIP="192.0.2.255" phishScore="46" spamScore="4" QID="r2FNwRHF004109" GUID="c26dbea0-80d5-463b-b93c-4e8b708219ce" subject="Please find a totally safe invoice attached." quarantineRule="module.sandbox.threat" quarantineFolder="Attachment Defense" policyRoutes="default_inbound,executives" modulesRun="sandbox,urldefense,spam,pdr" headerFrom="\"A. Badguy\" " headerTo="\"Clark Kent\" ; \"Diana Prince\" " headerCC="\"Bruce Wayne\" " headerReplyTo="null" toAddresses="clark.kent@pharmtech.zz,diana.prince@pharmtech.zz" ccAddresses="bruce.wayne@university-of-education.zz" fromAddress="badguy@evil.zz" replyToAddress="null" clusterId="pharmtech_hosted" messageParts="[{\"contentType\":\"text/plain\",\"disposition\":\"inline\",\"filename\":\"text.txt\",\"md5\":\"008c5926ca861023c1d2a36653fd88e2\",\"oContentType\":\"text/plain\",\"sandboxStatus\":\"unsupported\",\"sha256\":\"85738f8f9a7f1b04b5329c590ebcb9e425925c6d0984089c43a022de4f19c281\"},{\"contentType\":\"application/pdf\",\"disposition\":\"attached\",\"filename\":\"Invoice for Pharmtech.pdf\",\"md5\":\"5873c7d37608e0d49bcaa6f32b6c731f\",\"oContentType\":\"application/pdf\",\"sandboxStatus\":\"threat\",\"sha256\":\"2fab740f143fc1aa4c1cd0146d334c5593b1428f6d062b2c406e5efe8abe95ca\"}]" xmailer="Spambot v2.5"] ``` #### Mapped event ```json { - "@timestamp": "2016-06-24T21:00:08Z", + "@timestamp": "2021-06-24T21:00:08Z", "email": { - "timestamp": "2016-06-24T21:18:38.000Z", - "message_id": "20160624211145.62086.mail@evil.zz", + "timestamp": "2021-06-24T21:18:38.000Z", + "message_id": "20210624211145.62086.mail@evil.zz", "local_id": "c26dbea0-80d5-463b-b93c-4e8b708219ce", - "to": [ - { - "address": "clark.kent@pharmtech.zz", - "display_name": "Clark Kent" - }, - { - "address": "diana.prince@pharmtech.zz", - "display_name": "Diana Prince" - } - ], - "cc": [ - { - "address": "bruce.wayne@university-of-education.zz", - "display_name": "Bruce Wayne" - } - ], - "from": [ - { - "address": "badguy@evil.zz", - "display_name": "A. Badguy" - } - ], + "to": { + "address": [ + "clark.kent@pharmtech.zz", + "diana.prince@pharmtech.zz" + ] + }, + "cc": { + "address": [ + "bruce.wayne@university-of-education.zz" + ] + }, + "from": { + "address": [ + "badguy@evil.zz" + ] + }, "sender": { "address": "e99d7ed5580193f36a51f597bc2c0210@evil.zz" }, @@ -249,30 +233,30 @@ Stage 2: Included a real world example source document. Ideally this example com #### Original log ``` -datetime=2017-05-26T16:47:41+0100|aCode=7O7I7MvGP1mj8plHRDuHEA|acc=C0A0|SpamLimit=0|IP=123.123.123.123|Dir=Internal|MsgId=<81ce15$8r2j59@mail01.example.com>|Subject=\message subject\|headerFrom=from@mimecast.com|Sender=from@mimecast.com|Rcpt=auser@mimecast.com|SpamInfo=[]|Act=Acc|TlsVer=TLSv1|Cphr=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA|SpamProcessingDetail={"spf":{"info":"SPF_FAIL","allow":true},"dkim":{"info":"DKIM_UNKNOWN","allow":true}}|SpamScore=1 +datetime=2021-05-26T16:47:41+0100|aCode=7O7I7MvGP1mj8plHRDuHEA|acc=C0A0|SpamLimit=0|IP=123.123.123.123|Dir=Internal|MsgId=<81ce15$8r2j59@mail01.example.com>|Subject=\message subject\|headerFrom=from@mimecast.com|Sender=from@mimecast.com|Rcpt=auser@mimecast.com|SpamInfo=[]|Act=Acc|TlsVer=TLSv1|Cphr=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA|SpamProcessingDetail={"spf":{"info":"SPF_FAIL","allow":true},"dkim":{"info":"DKIM_UNKNOWN","allow":true}}|SpamScore=1 ``` #### Mapped event ```json { - "@timestamp": "2017-05-26T16:47:41+0100", + "@timestamp": "2021-05-26T16:47:41+0100", "source": { "address": "123.123.123.123", "ip": "123.123.123.123" }, "email": { "message_id": "<81ce15$8r2j59@mail01.example.com>", - "from": [ - { - "address": "from@mimecast.com" - } - ], - "to": [ - { - "address": "auser@mimecast.com" - } - ], + "from": { + "address": [ + "from@mimecast.com" + ] + }, + "to": { + "address": [ + "auser@mimecast.com" + ] + }, "subject": "message subject", "direction": "internal" }, diff --git a/rfcs/text/0010/email.yml b/rfcs/text/0010/email.yml index da838c6eb7..181f253c4b 100644 --- a/rfcs/text/0010/email.yml +++ b/rfcs/text/0010/email.yml @@ -78,26 +78,6 @@ SHA-256 hash of the attachment file. example: "f0366b3559f577d8732f7e9cc343a4960d202e8137dcc42f9783f3963f6abc6a" - - name: bcc - level: extended - type: keyword - short: BCC recipients - description: > - The email address(es) of the blind carbon carbon (BCC) recipients. - example: "['bcc.user1@example.com', 'bcc.user2@example.com']" - - normalize: - - array - - - name: bcc - level: extended - type: nested - short: List of objects describing BCC recipients - description: > - The email address(es) of the blind carbon carbon (BCC) recipients. - normalize: - - array - - name: bcc.address level: extended type: keyword @@ -105,21 +85,6 @@ description: > The email address of BCC recipient example: "bcc.user1@example.com" - - - name: bcc.display_name - level: extended - type: keyword - short: Display name of BCC recipient - description: > - The display name of the BCC recipient - example: "BCC User" - - - name: cc - level: extended - type: nested - short: List of objects describing CC recipients - description: > - The email address(es) of the carbon carbon (BCC) recipients. normalize: - array @@ -130,14 +95,8 @@ description: > The email address of CC recipient example: "cc.user1@example.com" - - - name: cc.display_name - level: extended - type: keyword - short: Display name of CC recipient - description: > - The display name of CC recipient - example: "CC User" + normalize: + - array - name: content_type level: extended @@ -165,15 +124,6 @@ The direction of the message based on the sending and receiving domains. example: inbound - - name: from - level: extended - type: nested - short: List of objects describing sender. - description: > - The email addresses of message sender. - normalize: - - array - - name: from.address level: extended type: keyword @@ -181,14 +131,8 @@ description: > The email address of the sender, typically from the RFC 5322 `From:` header field. example: "sender@example.com" - - - name: from.display_name - level: extended - type: keyword - short: The sender's display name. - description: > - The display name of the sender. - example: "Sender" + normalize: + - array - name: local_id level: extended @@ -241,13 +185,6 @@ Per RFC 5322, specifies the address responsible for the actual transmission of the message. - - name: sender.display_name - level: extended - type: keyword - short: Display name of the sending address. - description: > - The display name of the `Sender:`, if included. - - name: subject level: extended type: keyword @@ -259,15 +196,6 @@ - type: match_only_text name: text - - name: to - level: extended - type: nested - short: List of objects describing recipients. - description: > - The email addresses of message recipients. - normalize: - - array - - name: to.address level: extended type: keyword @@ -275,14 +203,8 @@ description: > The email address of recipient example: "user1@example.com" - - - name: to.display_name - level: extended - type: keyword - short: Display name of recipient - description: > - The display name of recipient - example: "Example User" + normalize: + - array - name: x_mailer level: extended From ef4bf1327979c3b2fe500a16c6d5bb23e44b2614 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 30 Nov 2021 11:02:47 -0600 Subject: [PATCH 20/26] update display name resolution --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 4b183075c6..e35d511688 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -304,7 +304,7 @@ Should a new event.category field (email) be created, and, if so, which `event.t Should the display name be captured separately from the email address for senders and recipients? If so, how do we accomplish this in a document while keeping the 1:1 of a display name to email address? -**Resolution**: Use `nested` types to allows arrays of objects containing both the email address and display name for the `to`, `cc`, and `bcc` recipients. +**Resolution**: Previously, this proposal considered using `nested` types to allows arrays of objects containing both the email address and display name for the `to`, `cc`, and `bcc` recipients. However, after more consideration of the limitations to using `nested` fields types and limited support for `nested` fields in Kibana, that decision was reversed. ### Spam processing details From 00a1fc6edb90ee4c460748b4a2127ce4fde9c715 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 30 Nov 2021 11:05:41 -0600 Subject: [PATCH 21/26] s/Previously/Initially --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index e35d511688..35071c567c 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -304,7 +304,7 @@ Should a new event.category field (email) be created, and, if so, which `event.t Should the display name be captured separately from the email address for senders and recipients? If so, how do we accomplish this in a document while keeping the 1:1 of a display name to email address? -**Resolution**: Previously, this proposal considered using `nested` types to allows arrays of objects containing both the email address and display name for the `to`, `cc`, and `bcc` recipients. However, after more consideration of the limitations to using `nested` fields types and limited support for `nested` fields in Kibana, that decision was reversed. +**Resolution**: Initially, this proposal considered using `nested` types to allows arrays of objects containing both the email address and display name for the `to`, `cc`, and `bcc` recipients. However, after more consideration of the limitations to using `nested` fields types and limited support for `nested` fields in Kibana, that decision was reversed. ### Spam processing details From 6c106ae1e6f260d1e766bf3bd7a00d4fed976226 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 30 Nov 2021 11:14:29 -0600 Subject: [PATCH 22/26] fix reply_to --- rfcs/text/0010/email.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/rfcs/text/0010/email.yml b/rfcs/text/0010/email.yml index 181f253c4b..5049078d19 100644 --- a/rfcs/text/0010/email.yml +++ b/rfcs/text/0010/email.yml @@ -169,13 +169,8 @@ description: > The address that replies should be delivered to based on the value in the RFC 5322 `Reply-To:` header. example: "reply.here@example.com" - - - name: reply_to.display_name - level: extended - type: keyword - short: Display name included with reply-to address. - description: > - The display name included with the `Reply-To` address. + normalize: + - array - name: sender.address level: extended From cee119d293b2844b320e6815349c91f89970f3a8 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 30 Nov 2021 11:14:38 -0600 Subject: [PATCH 23/26] tidy up table --- rfcs/text/0010-email.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 35071c567c..378cb05020 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -11,18 +11,18 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | field | type | description | | --- | --- | --- | -| `email.from.address` | keyword | Stores the `from` email address from the RFC5322 `From:` header field. | -| `email.sender.address` | keyword | When the `from` field contains more than one address or the `sender` and `from` are distinct then this field is populated. | | `email.origination_timestamp` | date | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | | `email.delivery_timestamp` | date | The date and time the email message was received by the service or client. | -| `email.to.address` | keyword | The email address of message recipient | +| `email.from.address` | keyword (array) | Stores the `from` email address from the RFC5322 `From:` header field. | +| `email.sender.address` | keyword | When the `from` field contains more than one address or the `sender` and `from` are distinct then this field is populated. | +| `email.to.address` | keyword (array)| The email address of message recipient | +| `email.cc.address` | keyword (array) | The email address of a carbon copy (CC) recipient | +| `email.bcc.address` | keyword (array) | The email address of the blind carbon copy (CC) recipient(s) | +| `email.reply_to.address` | keyword (array) | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | | `email.subject` | keyword | `.text` text multi-field | A brief summary of the topic of the message | -| `email.cc.address` | keyword | The email address of a carbon copy (CC) recipient | -| `email.bcc.address` | keyword | The email address of the blind carbon copy (CC) recipient(s) | | `email.content_type` | keyword | Information about how the message is to be displayed. Typically a MIME type | | `email.message_id` | wildcard | Identifier from the RFC5322 `Message-ID:` header field that refers to a particular version of a particular message. | | `email.local_id` | keyword | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the `X-MS-Exchange-Organization-Network-Message-Id` id). | -| `email.reply_to.address` | keyword | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | | `email.direction` | keyword | Direction of the message based on the sending and receiving domains | | `email.x_mailer` | keyword | What application was used to draft and send the original email. | | `email.attachments` | nested | Nested object of attachments on the email. | From d8b9ec7cf3e8f6c23739a22bf134163ae1987764 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 30 Nov 2021 11:16:09 -0600 Subject: [PATCH 24/26] fix columns --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 378cb05020..71106fde65 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -19,7 +19,7 @@ This RFC proposes a new top-level field set to facilitate email use cases, `emai | `email.cc.address` | keyword (array) | The email address of a carbon copy (CC) recipient | | `email.bcc.address` | keyword (array) | The email address of the blind carbon copy (CC) recipient(s) | | `email.reply_to.address` | keyword (array) | The address that replies should be delivered to from the RFC 5322 `Reply-To:` header field. | -| `email.subject` | keyword | `.text` text multi-field | A brief summary of the topic of the message | +| `email.subject` | keyword (`.text` text multi-field) | A brief summary of the topic of the message | | `email.content_type` | keyword | Information about how the message is to be displayed. Typically a MIME type | | `email.message_id` | wildcard | Identifier from the RFC5322 `Message-ID:` header field that refers to a particular version of a particular message. | | `email.local_id` | keyword | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the `X-MS-Exchange-Organization-Network-Message-Id` id). | From e11eb88168b1037699a87e3f9a8590b3d32e671d Mon Sep 17 00:00:00 2001 From: djptek Date: Thu, 2 Dec 2021 09:36:36 +0100 Subject: [PATCH 25/26] remove spaces lines 85 to 89 --- rfcs/text/0010-email.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 71106fde65..65aa4e84d4 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -82,11 +82,11 @@ Stage 2: Included a real world example source document. Ideally this example com "@timestamp": 1626984241830, "email": { "timestamp": "2020-11-08T22:12:34.8196921Z", - "from": { - "address": [ - "o365mc@microsoft.com" - ] - }, + "from": { + "address": [ + "o365mc@microsoft.com" + ] + }, "to": { "address": [ "john@testdomain.onmicrosoft.com" From 7249732a2648ac95bb6e9728dbe82cb7002e8f87 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 13 Dec 2021 16:06:59 -0600 Subject: [PATCH 26/26] set date for stage 2 --- rfcs/text/0010-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/text/0010-email.md b/rfcs/text/0010-email.md index 65aa4e84d4..b160118eaa 100644 --- a/rfcs/text/0010-email.md +++ b/rfcs/text/0010-email.md @@ -1,7 +1,7 @@ # 0010: Email - Stage: **2 (candidate)** -- Date: **TBD** +- Date: **2021-12-13** This RFC proposes a new top-level field set to facilitate email use cases, `email.*`. The `email.*` field set adds fields for the sender, recipient, message header fields, and other attributes of an email message typically seen logs produced by mail transfer agent (MTA) and email gateway applications.