diff --git a/packages/mimecast/_dev/build/build.yml b/packages/mimecast/_dev/build/build.yml new file mode 100644 index 00000000000..08d85edcf9a --- /dev/null +++ b/packages/mimecast/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.12 diff --git a/packages/mimecast/_dev/build/docs/README.md b/packages/mimecast/_dev/build/docs/README.md new file mode 100644 index 00000000000..e1a86bfe5cb --- /dev/null +++ b/packages/mimecast/_dev/build/docs/README.md @@ -0,0 +1,69 @@ +# Mimecast Integration + +The Mimecast integration collects events from the Mimecast API. + +## Logs + +### AUDIT EVENTS + +This is the `mimecast.audit_events` dataset. + +{{event "audit_events"}} + +{{fields "audit_events"}} + +### DLP LOGS + +This is the `mimecast.dlp_logs` dataset. + +{{event "dlp_logs"}} + +{{fields "dlp_logs"}} + +### SIEM LOGS + +This is the `mimecast.siem_logs` dataset. + +{{event "siem_logs"}} + +{{fields "siem_logs"}} + +### TTP IMPERSONATION LOGS + +This is the `mimecast.ttp_ip_logs` dataset. + +{{event "ttp_ip_logs"}} + +{{fields "ttp_ip_logs"}} + +### TTP ATTACHMENT LOGS + +This is the `mimecast.ttp_ap_logs` dataset. + +{{event "ttp_ap_logs"}} + +{{fields "ttp_ap_logs"}} + +### TTP URL LOGS + +This is the `mimecast.ttp_url_logs` dataset. + +{{event "ttp_url_logs"}} + +{{fields "ttp_url_logs"}} + +### THREAT INTEL FEED MALWARE CUSTOMER + +This is the `mimecast.threat_intel_malware_customer` dataset. + +{{event "threat_intel_malware_customer"}} + +{{fields "threat_intel_malware_customer"}} + +### THREAT INTEL FEED MALWARE GRID + +This is the `mimecast.threat_intel_malware_grid` dataset. + +{{event "threat_intel_malware_grid"}} + +{{fields "threat_intel_malware_grid"}} \ No newline at end of file diff --git a/packages/mimecast/_dev/deploy/docker/docker-compose.yml b/packages/mimecast/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..4fb7ea2ef1a --- /dev/null +++ b/packages/mimecast/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2.3' +services: + mimecast: + image: docker.elastic.co/observability/stream:v0.6.1 + ports: + - 8080 + volumes: + - ./files:/files:ro + environment: + PORT: 8080 + command: + - http-server + - --addr=:8080 + - --config=/files/config.yml diff --git a/packages/mimecast/_dev/deploy/docker/files/config.yml b/packages/mimecast/_dev/deploy/docker/files/config.yml new file mode 100644 index 00000000000..c45e2a0d76e --- /dev/null +++ b/packages/mimecast/_dev/deploy/docker/files/config.yml @@ -0,0 +1,156 @@ +rules: + - path: /api/audit/get-audit-events + methods: ["POST"] + query_params: + pageToken: next-page + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta": {"status": 200,"pagination": {"pageSize": 10,"next": "nextToken"}},"data": [{"id": "eNqrVipOTS4tSs1MUbJSyi1Mjgw1N6tIq6p0Kko0LCj2CS1NdAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhmoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIWmLGo","auditType": "Threat Intel Feed Download","user": "johndoe@example.com","eventTime": "2021-11-16T12:03:24+0000","eventInfo": "Threat intel multiple feeds download - malware_grid_csv_20211116120324398.zip, Date: 2021-11-16, Time: 12:03:24+0000, IP: 8.8.8.8, Application: Integrations","category": "reporting_logs"},{"id": "eNqrVipOTS4tSs1MUbJSCndJ8ihLdgou9ncyz3NxcjRN80r0NgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhqoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAADXK0M","auditType": "Threat Intel Feed Download","user": "azuresentinel_api_service_account@example.com","eventTime": "2021-11-16T12:03:16+0000","eventInfo": "Threat intel multiple feeds download - malware_grid_csv_20211116120316756.zip, Date: 2021-11-16, Time: 12:03:16+0000, IP: 8.8.8.8, Application: Azure Sentinel","category": "reporting_logs"},{"id": "eNqrVipOTS4tSs1MUbJS8vYOCjX2LfV3dSuzMDcoKyvMNjZPCgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhqpKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAwiK2M","auditType": "Threat Intel Feed Download","user": "johndoe@example.com","eventTime": "2021-11-16T12:03:00+0000","eventInfo": "Threat intel multiple feeds download - malware_customer_csv_20211116120300300.zip, Date: 2021-11-16, Time: 12:03:00+0000, IP: 8.8.8.8, Application: Integrations","category": "reporting_logs"},{"id": "eNqrVipOTS4tSs1MUbJS8nbzNE3x145IjtKuqDDyNzf2sAitCAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhqqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAODxKwI","auditType": "Threat Intel Feed Download","user": "johndoe@example.com","eventTime": "2021-11-16T12:03:00+0000","eventInfo": "Threat intel multiple feeds download - malware_customer_csv_20211116120300285.zip, Date: 2021-11-16, Time: 12:03:00+0000, IP: 8.8.8.8, Application: Integrations","category": "reporting_logs"},{"id": "eNqrVipOTS4tSs1MUbJS8nfOMCkrzAv1SAwtTnFJNjJ1TE-3MAxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhioaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACRNK5A","auditType": "Review Set Action","user": "johndoe@example.com","eventTime": "2021-11-16T12:02:44+0000","eventInfo": "Viewed Review Set Details - Case: GDPR/CCPA, Review Set: Supervision - hot words, Date: 2021-11-16, Time: 12:02:44+0000, IP: 8.8.8.8, Application: mimecast-case-review","category": "case_review_logs"},{"id": "eNqrVipOTS4tSs1MUbJS8vYorCw2z4gM8A4PLc-pCMlJL4rRN3cLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlhbGJoYqyjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCafSyD",n "auditType": "Threat Intel Feed Download",n "user": "johndoe@example.com","eventTime": "2021-11-16T12:02:00+0000","eventInfo": "Threat intel multiple feeds download - malware_customer_csv_20211116120200376.zip, Date: 2021-11-16, Time: 12:02:00+0000, IP: 8.8.8.8, Application: Integrations",n "category": "reporting_logs"},{ "id": "eNqrVipOTS4tSs1MUbJSSjJNyi1xNgovr8rNTM-I0Y9yznRzc8wJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlhbGJoYqSjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCJWSxd", "auditType": "Threat Intel Feed Download", "user": "johndoe@example.com", "eventTime": "2021-11-16T12:02:00+0000", "eventInfo": "Threat intel multiple feeds download - malware_customer_csv_20211116120200267.zip, Date: 2021-11-16, Time: 12:02:00+0000, IP: 8.8.8.8, Application: Integrations", "category": "reporting_logs" }, { "id": "eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o", "auditType": "Search Action", "user": "johndoe@example.com", "eventTime": "2021-11-16T12:01:37+0000", "eventInfo": "Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review", "category": "case_review_logs" }, { "id": "eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI", "auditType": "Case Action", "user": "johndoe@example.com", "eventTime": "2021-11-16T12:01:37+0000", "eventInfo": "Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review", "category": "case_review_logs" }, { "id": "eNqrVipOTS4tSs1MUbJSKvWJ8vPMCg8tTwoMC64wMCyIKossNQxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkpqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAFqfLAw", "auditType": "Review Set Action", "user": "johndoe@example.com", "eventTime": "2021-11-16T12:01:37+0000", "eventInfo": "Viewed Review Set Details - Case: GDPR/CCPA, Review Set: Supervision - hot words, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review", "category": "case_review_logs" }], "fail": []} + - path: /api/audit/get-audit-events + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta":{"status":200,"pagination":{"pageSize":10,"totalCount": 449, "pageToken":"next-page"}},"data":[{"id":"eNqrVipOTS4tSs1MUbJSyi1Mjgw1N6tIq6p0Kko0LCj2CS1NdAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhmoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIWmLGo","auditType":"Threat Intel Feed Download","user":"johndoe@example.com","eventTime":"2021-11-16T12:03:24+0000","eventInfo":"Threat intel multiple feeds download - malware_grid_csv_20211116120324398.zip, Date: 2021-11-16, Time: 12:03:24+0000, IP: 8.8.8.8, Application: Integrations","category":"reporting_logs"},{"id":"eNqrVipOTS4tSs1MUbJSCndJ8ihLdgou9ncyz3NxcjRN80r0NgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhqoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAADXK0M","auditType":"Threat Intel Feed Download","user":"azuresentinel_api_service_account@example.com","eventTime":"2021-11-16T12:03:16+0000","eventInfo":"Threat intel multiple feeds download - malware_grid_csv_20211116120316756.zip, Date: 2021-11-16, Time: 12:03:16+0000, IP: 8.8.8.8, Application: Azure Sentinel","category":"reporting_logs"},{"id":"eNqrVipOTS4tSs1MUbJS8vYOCjX2LfV3dSuzMDcoKyvMNjZPCgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhqpKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAwiK2M","auditType":"Threat Intel Feed Download","user":"johndoe@example.com","eventTime":"2021-11-16T12:03:00+0000","eventInfo":"Threat intel multiple feeds download - malware_customer_csv_20211116120300300.zip, Date: 2021-11-16, Time: 12:03:00+0000, IP: 8.8.8.8, Application: Integrations","category":"reporting_logs"},{"id":"eNqrVipOTS4tSs1MUbJS8nbzNE3x145IjtKuqDDyNzf2sAitCAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhqqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAODxKwI","auditType":"Threat Intel Feed Download","user":"johndoe@example.com","eventTime":"2021-11-16T12:03:00+0000","eventInfo":"Threat intel multiple feeds download - malware_customer_csv_20211116120300285.zip, Date: 2021-11-16, Time: 12:03:00+0000, IP: 8.8.8.8, Application: Integrations","category":"reporting_logs"},{"id":"eNqrVipOTS4tSs1MUbJS8nfOMCkrzAv1SAwtTnFJNjJ1TE-3MAxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhioaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACRNK5A","auditType":"Review Set Action","user":"johndoe@example.com","eventTime":"2021-11-16T12:02:44+0000","eventInfo":"Viewed Review Set Details - Case: GDPR/CCPA, Review Set: Supervision - hot words, Date: 2021-11-16, Time: 12:02:44+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"},{"id":"eNqrVipOTS4tSs1MUbJS8vYorCw2z4gM8A4PLc-pCMlJL4rRN3cLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlhbGJoYqyjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCafSyD","auditType":"Threat Intel Feed Download","user":"johndoe@example.com","eventTime":"2021-11-16T12:02:00+0000","eventInfo":"Threat intel multiple feeds download - malware_customer_csv_20211116120200376.zip, Date: 2021-11-16, Time: 12:02:00+0000, IP: 8.8.8.8, Application: Integrations","category":"reporting_logs"},{"id":"eNqrVipOTS4tSs1MUbJSSjJNyi1xNgovr8rNTM-I0Y9yznRzc8wJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlhbGJoYqSjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCJWSxd","auditType":"Threat Intel Feed Download","user":"johndoe@example.com","eventTime":"2021-11-16T12:02:00+0000","eventInfo":"Threat intel multiple feeds download - malware_customer_csv_20211116120200267.zip, Date: 2021-11-16, Time: 12:02:00+0000, IP: 8.8.8.8, Application: Integrations","category":"reporting_logs"},{"id":"eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o","auditType":"Search Action","user":"johndoe@example.com","eventTime":"2021-11-16T12:01:37+0000","eventInfo":"Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"},{"id":"eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI","auditType":"Case Action","user":"johndoe@example.com","eventTime":"2021-11-16T12:01:37+0000","eventInfo":"Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"},{"id":"eNqrVipOTS4tSs1MUbJSKvWJ8vPMCg8tTwoMC64wMCyIKossNQxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkpqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAFqfLAw","auditType":"Review Set Action","user":"johndoe@example.com","eventTime":"2021-11-16T12:01:37+0000","eventInfo":"Viewed Review Set Details - Case: GDPR/CCPA, Review Set: Supervision - hot words, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"}],"fail":[]} + - path: /api/dlp/get-logs + methods: ["POST"] + query_params: + pageToken: next-page + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta": { "status": 200, "pagination": { "pageSize": 10, "totalCount": 519, "next": "nextToken" } }, data":[{ "dlpLogs": [ { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:25+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214122.A79CE10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:25+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214122.A79CE10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:24+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214121.2618D10021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:24+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214121.2618D10021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:22+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214119.9F8FF10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:22+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214119.9F8FF10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:20+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214117.CD20510021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:20+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214117.CD20510021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:18+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214115.B346F10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:18+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214115.B346F10021D@mail.emailsec.ninja>" } ] } ], "fail": []} + - path: /api/dlp/get-logs + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta": { "status": 200, "pagination": { "pageSize": 10, "totalCount": 519, "pageToken": "next-page" } },"data":[{ "dlpLogs": [ { "senderAddress": "<>", "recipientAddress": "jondoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:25+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214122.A79CE10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:25+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214122.A79CE10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:24+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214121.2618D10021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:24+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214121.2618D10021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:22+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214119.9F8FF10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:22+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214119.9F8FF10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:20+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214117.CD20510021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:20+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214117.CD20510021B@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:18+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "notification", "messageId": "<20211118214115.B346F10021D@mail.emailsec.ninja>" }, { "senderAddress": "<>", "recipientAddress": "johndoe@example.com", "subject": "Undelivered Mail Returned to Sender", "eventTime": "2021-11-18T21:41:18+0000", "route": "inbound", "policy": "Content Inspection - Watermark", "action": "hold", "messageId": "<20211118214115.B346F10021D@mail.emailsec.ninja>" } ] } ], "fail": []} + - path: /api/audit/get-siem-logs + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + {"type":"MTA","data":[{"acc":"ABC123","Sender":"johndoe@example.com","datetime":"2021-11-12T12:15:46+0000","Rcpt":"o365_service_account@example.com","RcptActType":"Jnl","aCode":"fjihpfEgM_iRwemxhe3t_w","Dir":"Internal","RcptHdrType":"Unknown"}]} + - path: /api/ttp/threat-intel/get-feed + methods: ["POST"] + query_params: + pageToken: next-page + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + {"type":"bundle","id":"bundle--0ea0ae62-5d43-4ec5-babc-1fc4479e03ba","spec_version":"2.0","objects":[{"type":"malware","id":"malware--44c9d067-12c2-439a-a314-29652b158159","created":"2021-11-19T01:28:37.099Z","modified":"2021-11-19T01:28:37.099Z","name":"Transaction notice","labels":["virus"]},{"type":"indicator","id":"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd","created":"2021-11-19T01:28:37.099Z","modified":"2021-11-19T01:28:37.099Z","labels":["malicious-activity"],"pattern":"[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']","valid_from":"2021-11-19T01:28:37.099Z"},{"type":"relationship","id":"relationship--6f2459e4-4fbd-457a-ba79-0237beb99055","created":"2021-11-19T01:28:37.099Z","modified":"2021-11-19T01:28:37.099Z","relationship_type":"indicates","source_ref":"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd","target_ref":"malware--44c9d067-12c2-439a-a314-29652b158159"}]} + - path: /api/ttp/threat-intel/get-feed + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"type":"bundle","id":"bundle--0ea0ae62-5d43-4ec5-babc-1fc4479e03ba","spec_version":"2.0","token":"next-page","objects":[{"type":"malware","id":"malware--44c9d067-12c2-439a-a314-29652b158159","created":"2021-11-19T01:28:37.099Z","modified":"2021-11-19T01:28:37.099Z","name":"Transaction notice","labels":["virus"]},{"type":"indicator","id":"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd","created":"2021-11-19T01:28:37.099Z","modified":"2021-11-19T01:28:37.099Z","labels":["malicious-activity"],"pattern":"[file:hashes.'SHA-256' = 'ec5a6c52acdc187fc6c1187f14cd685c686c2b283503a023c4a9d3a977b491be']","valid_from":"2021-11-19T01:28:37.099Z"},{"type":"relationship","id":"relationship--6f2459e4-4fbd-457a-ba79-0237beb99055","created":"2021-11-19T01:28:37.099Z","modified":"2021-11-19T01:28:37.099Z","relationship_type":"indicates","source_ref":"indicator--456ac916-4c4e-43be-b7a9-6678f6a845cd","target_ref":"malware--44c9d067-12c2-439a-a314-29652b158159"}]} + - path: /api/ttp/attachment/get-logs + methods: ["POST"] + query_params: + pageToken: next-page + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + {"fail":[],"meta":{"status":200,"pagination":{"pageSize":500,"next":"nextToken"}},"data":[{"attachmentLogs":[{"result":"safe","date":"2021-11-24T11:54:27+0000","senderAddress":"<>","fileName":"Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf","actionTriggered":"user release, none","route":"inbound","details":"Safe \r\nTime taken: 0 hrs, 0 min, 7 sec","recipientAddress":"johndoe@emample.com","fileType":"application\/pdf","subject":"Test Files","fileHash":"cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254","messageId":"","definition":"Inbound - Safe file with On-Demand Sandbox"}]}]} + - path: /api/ttp/attachment/get-logs + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"fail":[],"meta":{"status":200,"pagination":{"pageSize":500,"pageToken":"next-page","next":"nextToken"}},"data":[{"attachmentLogs":[{"result":"safe","date":"2021-11-24T11:54:27+0000","senderAddress":"<>","fileName":"Datasheet_Mimecast Targeted Threat Protection + Internal Email Protect (2).pdf","actionTriggered":"user release, none","route":"inbound","details":"Safe \r\nTime taken: 0 hrs, 0 min, 7 sec","recipientAddress":"johndoe@emample.com","fileType":"application\/pdf","subject":"Test Files","fileHash":"cabd7cb6e1822fd9e1fc9bcf144ee26ee6bfc855c4574ca967dd53dcc36a1254","messageId":"","definition":"Inbound - Safe file with On-Demand Sandbox"}]}]} + - path: /api/ttp/impersonation/get-logs + methods: ["POST"] + query_params: + pageToken: next-page + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + {"meta": { "status": 200, "pagination": { "pageSize": 10, "totalCount": 36, "next": "next" } }, "data": [ { "impersonationLogs": [ { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzCw1FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGQVEhg", "senderAddress": "johndoe@example.com", "recipientAddress": "johdoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:15+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzCw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGP6Ehc", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@exampple.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:14+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw11EqSy0qzszPU7Iy1FEqyQMrBIor1QIAY98SFg", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:14+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw01EqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGPEEhU", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:14+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw1VEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGOpEhQ", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:10+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw1FEqSy0qzszPU7ICskvywAoNDMyVagFjPRIQ", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:06+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0VEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGOOEhM", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:06+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0lEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGNYEhE", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:06+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw1lEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGNzEhI", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:05+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGMiEg8", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:04+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" } ], "resultCount": 36 } ], "fail": [] } + - path: /api/ttp/impersonation/get-logs + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta": { "status": 200, "pagination": { "pageSize": 10, "totalCount": 36, "pageToken": "next-page" } }, "data": [ { "impersonationLogs": [ { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzCw1FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGQVEhg", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:15+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzCw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGP6Ehc", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:14+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw11EqSy0qzszPU7Iy1FEqyQMrBIor1QIAY98SFg", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:14+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw01EqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGPEEhU", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:14+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw1VEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGOpEhQ", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:10+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw1FEqSy0qzszPU7ICskvywAoNDMyVagFjPRIQ", "senderAddress": "johndoe@example.com", "recipientAddress": "testsite@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:06+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0VEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGOOEhM", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:06+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0lEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGNYEhE", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:06+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw1lEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGNzEhI", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoejr@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:05+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" }, { "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjM3MzAw0FEqSy0qzszPU7Iy1FEqyQMrNDAwV6oFAGMiEg8", "senderAddress": "johndoe@example.com", "recipientAddress": "johndoe@example.com", "subject": "Don't read, just fill out!", "definition": "IP - 1 hit (Tag email)", "hits": 1, "identifiers": [ "internal_user_name" ], "action": "none", "taggedExternal": false, "taggedMalicious": true, "senderIpAddress": "8.8.8.8", "eventTime": "2021-11-12T15:27:04+0000", "impersonationResults": [ { "impersonationDomainSource": "internal_user_name", "similarDomain": "John Doe ", "stringSimilarToDomain": "John Doe", "checkerResult": "hit" } ], "messageId": "" } ], "resultCount": 36 } ], "fail": [] } + - path: /api/ttp/url/get-logs + methods: ["POST"] + query_params: + pageToken: next-page + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta":{"status":200,"pagination":{"pageSize":10,"totalCount":584,"next":"next"}},"data":[{"clickLogs":[{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts/feeds/00259755281018227146/14369994449842858162","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"N/A","date":"2021-11-10T08:55:53+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<0000000000004109b705d06b609c@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts?source=alertsmail&hl=en&gl=US&msgid=MTgzMTU0Mzc2MTA3OTY3MzIxNw&s=AB2Xq4g-GUg7dJreWJN14pFdqYo0nYsyiVX2dK8&ffu=","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T08:50:37+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<00000000000079a99a05d06b4d20@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/url?rct=j&sa=t&url=https://texassports.com/news/2021/11/10/no-5-5-mens-basketball-tops-houston-baptist-in-season-opener-92-48.aspx&ct=ga&cd=CAEYACoUMTEzMjI3MjkwNzM0OTE1Nzg0NDMyHGMzNzg3MjBmODY3MWM2MGY6Y29tOmVuOlVTOkw&usg=AFQjCNG4_460IiZmbwJkDzkFkQC5-htSxw","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - news","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T08:28:18+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<0000000000005fa4e905d06afd8f@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts/feedback?ffu=https://www.ft.com/content/4d61fac4-e3f3-401b-bca2-6e94ff47e2cc&source=alertsmail&hl=en&gl=US&msgid=MTM1OTYyMDAwNTE0MzU3NjA0NjI&s=AB2Xq4g-GUg7dJreWJN14pFdqYo0nYsyiVX2dK8","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T07:50:16+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000eb13ab05d06a76fc@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts/feedback?ffu=https://www.fox7austin.com/news/search-for-suspect-involved-in-aggravated-robbery-at-family-dollar&source=alertsmail&hl=en&gl=US&msgid=ODM0MDY5Nzg2NzI3NDkxMjUwNg&s=AB2Xq4i7OaFz4ss3vFU-wNb0DTELEKxhyDdFl54","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T06:55:34+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<0000000000001ddf7205d069b36e@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts?source=alertsmail&hl=en&gl=US&msgid=NDgxNzM4MzYwOTM2NzY1MDg2Ng","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T06:50:16+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000567c2105d069a0ce@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts/share?hl=en&gl=US&ru=https://www.ctvnews.ca/world/judge-denies-trump-s-overnight-request-for-injunction-in-executive-privilege-case-1.5658613&ss=tw&rt=Judge+denies+Trump%27s+overnight+request+for+injunction+in+executive+privilege+case+%7C+CTV+News&cd=KhM2MTQ1MDcxODY2MDQ5NTY2MDk2Mhw1MzNlMDE2OWZhYWUyMDBkOmNvbTplbjpVUzpM&ssp=AMJHsmXlnSoHb_ZABC-riiVXrxFyWhlMpQ","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - Trump","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T05:11:56+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000e78cfa05d0683fab@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts?s=AB2Xq4i7OaFz4ss3vFU-wNb0DTELEKxhyDdFl54&start=1636516479&end=1636520078&source=alertsmail&hl=en&gl=US&msgid=MTMyNTI1OTY4MzI3OTI0NDc4MTU#history","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T04:55:34+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000f2696405d0680583@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.gstati","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Unknown","sendingIp":"8.8.8.8","userAwarenessAction":"N/A","date":"2021-11-10T03:55:09+0000","actions":"Browser Isolation","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<00000000000065020205d0672f01@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts/share?hl=en&gl=US&ru=https://www.wsj.com/articles/u-s-tests-israels-iron-dome-in-guam-as-defense-against-chinese-cruise-missiles-11636455224&ss=tw&rt=U.S.+Tests+Israel%27s+Iron+Dome+in+Guam+as+Defense+Against+Chinese+Cruise+Missiles+-+WSJ&cd=KhQxNzg2NTc5NDQ3ODIzODUyNjI5NzIcZmQ4N2VjYzkxMGIxMWE4Yzpjby56YTplbjpVUw&ssp=AMJHsmW3CCK1S4TNPifSXszcyaNMwd6TDg","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T03:49:53+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000a02a0a05d0671c06@google.com>"}]}],"fail":[]} + - path: /api/ttp/url/get-logs + methods: ["POST"] + request_headers: + authorization: ["MC .*"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"meta":{"status":200,"pagination":{"pageSize":10,"totalCount":584,"next":"next"}},"data":[{"clickLogs":[{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts/feeds/00259755281018227146/14369994449842858162","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"N/A","date":"2021-11-10T08:55:53+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<0000000000004109b705d06b609c@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts?source=alertsmail&hl=en&gl=US&msgid=MTgzMTU0Mzc2MTA3OTY3MzIxNw&s=AB2Xq4g-GUg7dJreWJN14pFdqYo0nYsyiVX2dK8&ffu=","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T08:50:37+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<00000000000079a99a05d06b4d20@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/url?rct=j&sa=t&url=https://texassports.com/news/2021/11/10/no-5-5-mens-basketball-tops-houston-baptist-in-season-opener-92-48.aspx&ct=ga&cd=CAEYACoUMTEzMjI3MjkwNzM0OTE1Nzg0NDMyHGMzNzg3MjBmODY3MWM2MGY6Y29tOmVuOlVTOkw&usg=AFQjCNG4_460IiZmbwJkDzkFkQC5-htSxw","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - news","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T08:28:18+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<0000000000005fa4e905d06afd8f@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts/feedback?ffu=https://www.ft.com/content/4d61fac4-e3f3-401b-bca2-6e94ff47e2cc&source=alertsmail&hl=en&gl=US&msgid=MTM1OTYyMDAwNTE0MzU3NjA0NjI&s=AB2Xq4g-GUg7dJreWJN14pFdqYo0nYsyiVX2dK8","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T07:50:16+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000eb13ab05d06a76fc@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts/feedback?ffu=https://www.fox7austin.com/news/search-for-suspect-involved-in-aggravated-robbery-at-family-dollar&source=alertsmail&hl=en&gl=US&msgid=ODM0MDY5Nzg2NzI3NDkxMjUwNg&s=AB2Xq4i7OaFz4ss3vFU-wNb0DTELEKxhyDdFl54","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T06:55:34+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<0000000000001ddf7205d069b36e@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts?source=alertsmail&hl=en&gl=US&msgid=NDgxNzM4MzYwOTM2NzY1MDg2Ng","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T06:50:16+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000567c2105d069a0ce@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts/share?hl=en&gl=US&ru=https://www.ctvnews.ca/world/judge-denies-trump-s-overnight-request-for-injunction-in-executive-privilege-case-1.5658613&ss=tw&rt=Judge+denies+Trump%27s+overnight+request+for+injunction+in+executive+privilege+case+%7C+CTV+News&cd=KhM2MTQ1MDcxODY2MDQ5NTY2MDk2Mhw1MzNlMDE2OWZhYWUyMDBkOmNvbTplbjpVUzpM&ssp=AMJHsmXlnSoHb_ZABC-riiVXrxFyWhlMpQ","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - Trump","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T05:11:56+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000e78cfa05d0683fab@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.com/alerts?s=AB2Xq4i7OaFz4ss3vFU-wNb0DTELEKxhyDdFl54&start=1636516479&end=1636520078&source=alertsmail&hl=en&gl=US&msgid=MTMyNTI1OTY4MzI3OTI0NDc4MTU#history","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T04:55:34+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000f2696405d0680583@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.gstati","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - dollar","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Unknown","sendingIp":"8.8.8.8","userAwarenessAction":"N/A","date":"2021-11-10T03:55:09+0000","actions":"Browser Isolation","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<00000000000065020205d0672f01@google.com>"},{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"googlealerts-noreply@google.com","url":"https://www.google.co.za/alerts/share?hl=en&gl=US&ru=https://www.wsj.com/articles/u-s-tests-israels-iron-dome-in-guam-as-defense-against-chinese-cruise-missiles-11636455224&ss=tw&rt=U.S.+Tests+Israel%27s+Iron+Dome+in+Guam+as+Defense+Against+Chinese+Cruise+Missiles+-+WSJ&cd=KhQxNzg2NTc5NDQ3ODIzODUyNjI5NzIcZmQ4N2VjYzkxMGIxMWE4Yzpjby56YTplbjpVUw&ssp=AMJHsmW3CCK1S4TNPifSXszcyaNMwd6TDg","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Google Alert - china","action":"allow","adminOverride":"N/A","userOverride":"None","scanResult":"clean","category":"Search Engines & Portals","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-11-10T03:49:53+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<000000000000a02a0a05d0671c06@google.com>"}]}],"fail":[]} diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml new file mode 100644 index 00000000000..3ad7e275054 --- /dev/null +++ b/packages/mimecast/changelog.yml @@ -0,0 +1,7 @@ +# newer versions go on top + +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/2157 diff --git a/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log new file mode 100644 index 00000000000..d5e73cc12ee --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log @@ -0,0 +1,25 @@ +{"auditType":"Threat Intel Feed Download","category":"reporting_logs","eventInfo":"Threat intel multiple feeds download - malware_customer_csv_20211018094502564.zip, Date: 2021-10-18, Time: 08:45:02+0000, IP: 8.8.8.8, Application: Integrations","eventTime":"2021-10-18T08:45:02+0000","id":"eNqrVipOTS4tSs1MUbJS8im3dA5NjAxJTPP0svD1jioo9IsINgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxpbmRhoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACCXK48","user":"johndoe@example.com"} +{"id": "eNqrVipOTS4tSs1MUbJS8nbx8CoyTPFN9akM9K5KqnQyi8z2DgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxoaG5grKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADbWK70","auditType": "Threat Intel Feed Download","user": "johndoe@example","eventTime": "2021-10-10T22:51:57+0000","eventInfo": "Threat intel multiple feeds download - malware_grid_csv_20211010235157027.zip, Date: 2021-10-10, Time: 22:51:57+0000, IP: 8.8.8.8, Application: Azure Sentinel","category": "reporting_logs"} +{"id": "eNqrVipOTS4tSs1MUbJSivD0cisuyAirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A","auditType": "User Logged On","user": "johndoe@example.com","eventTime": "2021-10-11T17:17:30+0000","eventInfo": "Successful authentication for johndoe@example.com , Date: 2021-10-11, Time: 18:17:30 BST, IP: 8.8.8.8, Application: Administration Console, Method: Two Step Auth, 2FA: TOTP","category": "authentication_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSSsos9DMJTPLyMA6NcCt2TA1OCwjLcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhsqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAC8tK60","auditType":"Logon Requires Challenge","user":"johndoe@example.com","eventTime":"2021-10-11T17:17:26+0000","eventInfo":"Intermediate authentication for johndoe@example.com , Date: 2021-10-11, Time: 18:17:26 BST, IP: 8.8.8.8, Application: Administration Console, Method: Office 365, 2FA: TOTP","category":"authentication_logs"} +{ "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI", "auditType": "User Logged On", "user": "johndoe@example.com", "eventTime": "2021-10-11T16:03:38+0000", "eventInfo": "Successful authentication for johndoe@example.com , Date: 2021-10-11, Time: 17:03:38 BST, IP: 8.8.8.8, Application: Administration Console, Method: Cloud", "category": "authentication_logs"} +{ "id": "eNqrVipOTS4tSs1MUbJSCkg09A93r0rNi9FPynHJ9gwJzyrzT8sJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGJsaqyjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCMPCxu", "auditType": "Mimecast Support Login", "user": "johdoe@example.local", "eventTime": "2021-10-11T15:39:17+0000", "eventInfo": "Action Performed - johdoe@example.local logged into this account. by johdoe@example.local Date: 2021-10-11 Time: 16:39:17 +0100 IP: 8.8.8.8 Application: Administration Console", "category": "mimecast_access_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSynStcDUudE51LQtJKc-M0TfwMjas8nQLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGliZGhgYqSjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgBLJCvK","auditType":"Mimecast Support Login","user":"johndoe@example.local","eventTime":"2021-10-19T11:46:40+0000","eventInfo":"Action Performed - johdoe@example.local logged into this account. by johdoe@example.local Date: 2021-10-19 Time: 12:46:40 +0100 IP: 8.8.8.8 Application: Administration Console","category":"mimecast_access_logs"} +{"id":"eNqrVipOTS4tSs1MUbJS0nYKziswMy_18smyMDAs9w8P8PPNNAxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmxopqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAOifKw8","auditType":"Message Action","user":"johndoe@example.com","eventTime":"2021-10-11T15:36:01+0000","eventInfo":"Viewed Message - Source: Search, From: johndoe@example.com, To: johndoe@example.com, Subject: Test on Tues 28th Sept, Processed: 2021-09-28 07:59:23+0000, Viewed Content: True, Date: 2021-10-11, Time: 15:36:01+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"} +{"id":"eNqrVipOTS4tSs1MUbJS0i5MNHQtiqoo9Q53S0yu8sov8AszyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmxorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAFqzLAw","auditType":"Search Action","user":"johndoe@example.com","eventTime":"2021-10-11T15:35:53+0000","eventInfo":"Executed Search - Source: Search, Search Criteria: {\"keywords\":\"test\",\"mailboxes\":[\"johndoe@example.com\"],\"route\":\"ALL\",\"start\":\"2021-04-11T16:34:45+0100\",\"end\":\"2021-10-11T16:34:45+0100\"}, Date: 2021-10-11, Time: 15:35:53+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSMk9PdXYMzywJrLLMzdT2TfVN8S8zNgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkbGFmoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACyMK6M","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-11T14:46:10+0000","eventInfo":"Creating the auditLog entry for failed authentication, emailAddress :com.example.sdk.address.Address@4a3bcd11[accountCode=ABC123,accountId=75,internal=false,emailAddress=johndoe@gmail.com,domainName=gmail.com,name=johndoe@gmail.com,aliasFor=0,type=0,journalService=false,id=275078533,aliases={},alternateAddresses={},alternateAliases={}] remote IP : 8.8.8.8 application : LFS","category":"authentication_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSKnU29DVI9XJJMs6wMC9LqnAMccoxcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkZGZqoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPQMKys","auditType":"Completed Directory Sync","user":"","eventTime":"2021-10-11T13:21:06+0000","eventInfo":"No changes found.","category":"account_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSSiwLM8srLCvJzg8s8HbydCpz0Y6oCAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaG5ooKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAHTYLDo","auditType":"Case Action","user":"johndoe@example.com","eventTime":"2021-10-12T09:19:53+0000","eventInfo":"Viewed Case - Case: Class Action, Date: 2021-10-12, Time: 09:19:53+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for johndoe@example.com , Date: 2021-10-12, Time: 09:47:55 BST, IP: 8.8.8.8, Application: mimecast-moa, Method: Office 365, Reason: Wrong password","category":"authentication_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSSnJMinKNMMtyDg3xKw2rDM91DC-JdAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRooaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAEQYK9w","auditType":"Existing Archive Task Changed","user":"johdoe@example.com","eventTime":"2021-10-12T08:47:54+0000","eventInfo":"Successfully updated 3 'Sync and Recover' tasks associated with legacy connection (\"365\") to new migrated connector (\"Sync and Recover - 365\"), Date: 2021-10-12, Time: 08:47:54+0000, IP: 8.8.8.8, Application: Administration Console","category":"archive_service_logs"} +{"id":"eNoVzc0KgkAUQOF3uVsFuZma7qQ0UqiFqChuZH7M0iZmHMOid8_2h-98QDGiJespBDBgYwn-4orcHMrr_JqUWdjFBb8YThbF5bE6le_ardLGitJqnHF39w7YGuLsL5g8l7wAE1pN-2kQ3V-00bdt3KBrAtFqEiOTRFC2rvZbN_ScNZ-ZVL14QIDfH41XLGM","auditType":"Connectors Management","user":"johndoe@example.com","eventTime":"2021-10-12T08:47:53+0000","eventInfo":"Connector creation for Microsoft O365\nName: Sync and Recover - 365, Description: null, Product: Sync and Recover, App (provider): Microsoft O365\nSuccess: true, Date: 2021-10-12, Time: 08:47:53+0000, IP: 8.8.8.8, Application: Administration Console","category":"integrations_and_apis"} +{"id":"eNqrVipOTS4tSs1MUbJSynAJ8yuoyA4z9ygMNyv21C42MC9IDwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkbmFhoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADk2K8U","auditType":"Page Data Exports","user":"johndoe@example.com","eventTime":"2021-10-12T02:27:18+0000","eventInfo":"[Export type : Download,Name :watchlist_view,Requested By :johdoe@example.com,Export time :Tue Oct 12 03:27:18 BST 2021,IP Address :8.8.8.8,Columns exported :Name|Email|Department|Number of Videos|,File name : export_at_watchlist_view_1634005638160.xlsx,File Size: 6864,File type : .xlsx], Date: 2021-10-12, Time: 02:27:18+0000, IP: 8.8.8.8, Application: mimecast-matfe","category":"account_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSMi8zSc3J8M4Od_NwjdHPMDYzdfGO8MkJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGppaKajlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAmqSuF","auditType":"Custom Report Definition Created","user":"johndoe@example.local","eventTime":"2021-10-11T19:53:41+0000","eventInfo":"Action Performed - Custom Report Definition Created with name \"Terri test\" and description \"all user - per email report\" by johndoe@example.local Date: 2021-10-11 Time: 20:53:41 +0100 IP: 8.8.8.8 Application: Administration Console","category":"reporting_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSCij080lzDChMMjXw8o3IjnCLDIrRT8wJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGpiYaqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgBNvCvh","auditType":"Folder Log Entry","user":"johndoe@example.com","eventTime":"2021-10-11T18:23:10+0000","eventInfo":"Action Performed - Deleted New Folder by johndoe@example.com Date: 2021-10-11 Time: 19:23:10 +0100 IP: 8.8.8.8 Application: Administration Console","category":"profile_group_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSCtF28jc2DDLwd_d1NM7ULnLzdnPzdwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiCAQ6SsmlxSX5ualFyfkpqUCbnE3MHM1NgcrLUouKM_PzlKwMawGTZipR","auditType":"User Password Changed","user":"johndoe@example.com","eventTime":"2021-10-12T19:56:55+0000","eventInfo":"Password reset for user: johndoe@example.com User Password Changed, Remote IP is null","category":"user_account_and_role_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSSgwpLctzzah00TbMTTawdC4NDPAzzwlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiaGBhoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADOfK6w","auditType":"Remediation Incident Adjustment","user":"johndoe@example.com","eventTime":"2021-10-12T19:49:30+0000","eventInfo":"Remediation Incident Created - TR-C46A75-01420-M, type : manual, search criteria : {\"fileHash\":\"9e6011844705292d5abfe0aa38d8aff02f6d8f69689c2e7cb2338f9484774bb3\",\"start\":\"2021-09-12T19:48:59+0000\",\"end\":\"2021-10-12T19:48:59+0000\"}, Date: 2021-10-12, Time: 19:49:30+0000, IP: 8.8.8.8, Application: Administration Console","category":"account_logs"} +{"id":"eNoVzdEKgjAYQOF3-W8Vaps69S7KooSEJGXSzdAVMtdi04FF757dH77zASvayYi-gxQIcbI0HEtcRI5aRS7SxkN1L7ywzPb1gR3rdxOx_LbKcqYciiXdIe7pczKj02u-VuADn7p-HPTjDxKUkGCdUOxDO9lRK2Fa3YnltA2iDQ2X3Alje_2EFH1_LYQrrw","auditType":"Archive Mailbox Restore","user":"johndoe@example.com","eventTime":"2021-10-12T19:20:01+0000","eventInfo":"Archive mailbox restore created. Restored data from johdoe@example.com to johndoe@example.com by johndoe@example.com, Date: 2021-10-12, Time: 19:20:01+0000, IP: 8.8.8.8, Application: Administration Console","category":"archive_service_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSigzJC_ZNzg-vcjYKcwz3icotC0nVdgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYG5kqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAD-SK84","auditType":"Archive Mailbox Restore","user":"johndoejr@example.com","eventTime":"2021-10-12T18:19:33+0000","eventInfo":"Archive mailbox restore created. Restored data from johndoe@example.com to johndoejr@example.com by johndoejr@example.com, Date: 2021-10-12, Time: 18:19:33+0000, IP: 8.8.8.8, Application: Administration Console","category":"archive_service_logs"} +{"id":"eNqrVipOTS4tSs1MUbJScjMvyjIxr6yoLDY2qQopLq3yDnM1dwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYGZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE5dK-0","auditType":"Archive Mailbox Export Download","user":"johndoe@example.com","eventTime":"2021-10-12T17:55:14+0000","eventInfo":"Mailbox export downloaded. Download filename (HTML Report recovery id): eNqrVipOTS4tSs1MUbJSyo3RDw81rTCpynMpdiuICMopyihxynZztcisDMoN9zWLSCrPzAjz9PALNzFwySrLMNQ2yUs38g9zS860cHKNMExR0lFKLi0uyc9NLUrOT0kFGulsYuZobgoUL0pNzi9LLarULUksztYFWWdpaKqjBBQqzszPU7IyrAUAsSEteA by johndoe@example.com, Date: 2021-10-12, Time: 17:55:14+0000, IP: 8.8.8.8, Application: Administration Console","category":"archive_service_logs"} +{"id":"eNqrVipOTS4tSs1MUbJSitH39gl1cS509PT1MSnw90l0CinPCQgLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsYmBsYqqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAxASul","auditType":"Review Set Action","user":"johndoe@example.com","eventTime":"2021-10-12T17:07:00+0000","eventInfo":"Viewed Review Set Details - Case: Class Action, Review Set: Contracts, Date: 2021-10-12, Time: 17:07:00+0000, IP: 8.8.8.8, Application: mimecast-case-review","category":"case_review_logs"} +{"id":"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38","auditType":"Remediation Incident Adjustment","user":"johndoe@example.com","eventTime":"2021-10-12T15:38:05+0000","eventInfo":"Restore Remediation Incident Created - TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 8.8.8.8, Application: Administration Console","category":"account_logs"} \ No newline at end of file diff --git a/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json new file mode 100644 index 00000000000..544423efd4b --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json @@ -0,0 +1,1307 @@ +{ + "expected": [ + { + "@timestamp": "2021-10-18T08:45:02.000Z", + "file": { + "name": "Threat intel multiple feeds download - malware_customer_csv_20211018094502564.zip", + "extension": "zip" + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "threat-intel-feed-download", + "ingested": "2021-11-25T11:34:08.372326900Z", + "original": "{\"auditType\":\"Threat Intel Feed Download\",\"category\":\"reporting_logs\",\"eventInfo\":\"Threat intel multiple feeds download - malware_customer_csv_20211018094502564.zip, Date: 2021-10-18, Time: 08:45:02+0000, IP: 8.8.8.8, Application: Integrations\",\"eventTime\":\"2021-10-18T08:45:02+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJS8im3dA5NjAxJTPP0svD1jioo9IsINgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxpbmRhoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACCXK48\",\"user\":\"johndoe@example.com\"}", + "id": "eNqrVipOTS4tSs1MUbJS8im3dA5NjAxJTPP0svD1jioo9IsINgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxpbmRhoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACCXK48", + "created": "2021-10-18T08:45:02.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Integrations", + "category": "reporting_logs", + "eventInfo": "Threat intel multiple feeds download - malware_customer_csv_20211018094502564.zip, Date: 2021-10-18, Time: 08:45:02+0000, IP: 8.8.8.8, Application: Integrations" + } + }, + { + "@timestamp": "2021-10-10T22:51:57.000Z", + "file": { + "name": "Threat intel multiple feeds download - malware_grid_csv_20211010235157027.zip", + "extension": "zip" + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "threat-intel-feed-download", + "ingested": "2021-11-25T11:34:08.372329600Z", + "original": "{\"id\": \"eNqrVipOTS4tSs1MUbJS8nbx8CoyTPFN9akM9K5KqnQyi8z2DgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxoaG5grKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADbWK70\",\"auditType\": \"Threat Intel Feed Download\",\"user\": \"johndoe@example\",\"eventTime\": \"2021-10-10T22:51:57+0000\",\"eventInfo\": \"Threat intel multiple feeds download - malware_grid_csv_20211010235157027.zip, Date: 2021-10-10, Time: 22:51:57+0000, IP: 8.8.8.8, Application: Azure Sentinel\",\"category\": \"reporting_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJS8nbx8CoyTPFN9akM9K5KqnQyi8z2DgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxoaG5grKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADbWK70", + "created": "2021-10-10T22:51:57.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example", + "domain": "example" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Azure Sentinel", + "category": "reporting_logs", + "eventInfo": "Threat intel multiple feeds download - malware_grid_csv_20211010235157027.zip, Date: 2021-10-10, Time: 22:51:57+0000, IP: 8.8.8.8, Application: Azure Sentinel" + } + }, + { + "@timestamp": "2021-10-11T17:17:30.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "user-logged-on", + "ingested": "2021-11-25T11:34:08.372330600Z", + "original": "{\"id\": \"eNqrVipOTS4tSs1MUbJSivD0cisuyAirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A\",\"auditType\": \"User Logged On\",\"user\": \"johndoe@example.com\",\"eventTime\": \"2021-10-11T17:17:30+0000\",\"eventInfo\": \"Successful authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-11, Time: 18:17:30 BST, IP: 8.8.8.8, Application: Administration Console, Method: Two Step Auth, 2FA: TOTP\",\"category\": \"authentication_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSivD0cisuyAirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A", + "created": "2021-10-11T07:17:30.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console, Method: Two Step Auth, 2FA: TOTP", + "category": "authentication_logs", + "eventInfo": "Successful authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-11, Time: 18:17:30 BST, IP: 8.8.8.8, Application: Administration Console, Method: Two Step Auth, 2FA: TOTP" + } + }, + { + "@timestamp": "2021-10-11T17:17:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "logon-requires-challenge", + "ingested": "2021-11-25T11:34:08.372331600Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSSsos9DMJTPLyMA6NcCt2TA1OCwjLcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhsqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAC8tK60\",\"auditType\":\"Logon Requires Challenge\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-11T17:17:26+0000\",\"eventInfo\":\"Intermediate authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-11, Time: 18:17:26 BST, IP: 8.8.8.8, Application: Administration Console, Method: Office 365, 2FA: TOTP\",\"category\":\"authentication_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSSsos9DMJTPLyMA6NcCt2TA1OCwjLcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhsqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAC8tK60", + "created": "2021-10-11T07:17:26.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console, Method: Office 365, 2FA: TOTP", + "category": "authentication_logs", + "eventInfo": "Intermediate authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-11, Time: 18:17:26 BST, IP: 8.8.8.8, Application: Administration Console, Method: Office 365, 2FA: TOTP" + } + }, + { + "@timestamp": "2021-10-11T16:03:38.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "user-logged-on", + "ingested": "2021-11-25T11:34:08.372332600Z", + "original": "{ \"id\": \"eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI\", \"auditType\": \"User Logged On\", \"user\": \"johndoe@example.com\", \"eventTime\": \"2021-10-11T16:03:38+0000\", \"eventInfo\": \"Successful authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-11, Time: 17:03:38 BST, IP: 8.8.8.8, Application: Administration Console, Method: Cloud\", \"category\": \"authentication_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI", + "created": "2021-10-11T06:03:38.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console, Method: Cloud", + "category": "authentication_logs", + "eventInfo": "Successful authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-11, Time: 17:03:38 BST, IP: 8.8.8.8, Application: Administration Console, Method: Cloud" + } + }, + { + "@timestamp": "2021-10-11T15:39:17.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johdoe", + "johdoe@example.local" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "mimecast-support-login", + "ingested": "2021-11-25T11:34:08.372333600Z", + "original": "{ \"id\": \"eNqrVipOTS4tSs1MUbJSCkg09A93r0rNi9FPynHJ9gwJzyrzT8sJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGJsaqyjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCMPCxu\", \"auditType\": \"Mimecast Support Login\", \"user\": \"johdoe@example.local\", \"eventTime\": \"2021-10-11T15:39:17+0000\", \"eventInfo\": \"Action Performed - johdoe@example.local logged into this account. by johdoe@example.local\u003cjohdoe@example.local\u003e Date: 2021-10-11 Time: 16:39:17 +0100 IP: 8.8.8.8 Application: Administration Console\", \"category\": \"mimecast_access_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSCkg09A93r0rNi9FPynHJ9gwJzyrzT8sJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGJsaqyjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCMPCxu", + "created": "2021-10-11T16:39:17.000Z" + }, + "user": { + "name": "johdoe", + "email": "johdoe@example.local", + "domain": "example.local" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "mimecast_access_logs", + "eventInfo": "Action Performed - johdoe@example.local logged into this account. by johdoe@example.local\u003cjohdoe@example.local\u003e Date: 2021-10-11 Time: 16:39:17 +0100 IP: 8.8.8.8 Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-19T11:46:40.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johdoe", + "johdoe@example.local" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "mimecast-support-login", + "ingested": "2021-11-25T11:34:08.372334500Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSynStcDUudE51LQtJKc-M0TfwMjas8nQLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGliZGhgYqSjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgBLJCvK\",\"auditType\":\"Mimecast Support Login\",\"user\":\"johndoe@example.local\",\"eventTime\":\"2021-10-19T11:46:40+0000\",\"eventInfo\":\"Action Performed - johdoe@example.local logged into this account. by johdoe@example.local\u003cjohdoe@example.local\u003e Date: 2021-10-19 Time: 12:46:40 +0100 IP: 8.8.8.8 Application: Administration Console\",\"category\":\"mimecast_access_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSynStcDUudE51LQtJKc-M0TfwMjas8nQLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGliZGhgYqSjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgBLJCvK", + "created": "2021-10-19T12:46:40.000Z" + }, + "user": { + "name": "johdoe", + "email": "johdoe@example.local", + "domain": "example.local" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "mimecast_access_logs", + "eventInfo": "Action Performed - johdoe@example.local logged into this account. by johdoe@example.local\u003cjohdoe@example.local\u003e Date: 2021-10-19 Time: 12:46:40 +0100 IP: 8.8.8.8 Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-11T15:36:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "message-action", + "ingested": "2021-11-25T11:34:08.372335500Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJS0nYKziswMy_18smyMDAs9w8P8PPNNAxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmxopqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAOifKw8\",\"auditType\":\"Message Action\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-11T15:36:01+0000\",\"eventInfo\":\"Viewed Message - Source: Search, From: \u003cJohn Done\u003e johndoe@example.com, To: \u003cjohndoe@example.com\u003e johndoe@example.com, Subject: Test on Tues 28th Sept, Processed: 2021-09-28 07:59:23+0000, Viewed Content: True, Date: 2021-10-11, Time: 15:36:01+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"category\":\"case_review_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJS0nYKziswMy_18smyMDAs9w8P8PPNNAxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmxopqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAOifKw8", + "created": "2021-10-11T15:36:01.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "email": { + "origination_timestamp": "2021-09-28 07:59:23+0000", + "from": { + "address": "johndoe@example.com" + }, + "to": { + "address": "johndoe@example.com" + }, + "subject": "Test on Tues 28th Sept" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "name": {}, + "eventInfo": "Viewed Message - Source: Search, From: \u003cJohn Done\u003e johndoe@example.com, To: \u003cjohndoe@example.com\u003e johndoe@example.com, Subject: Test on Tues 28th Sept, Processed: 2021-09-28 07:59:23+0000, Viewed Content: True, Date: 2021-10-11, Time: 15:36:01+0000, IP: 8.8.8.8, Application: mimecast-case-review", + "application": "mimecast-case-review", + "category": "case_review_logs" + } + }, + { + "@timestamp": "2021-10-11T15:35:53.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "search-action", + "ingested": "2021-11-25T11:34:08.372336400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJS0i5MNHQtiqoo9Q53S0yu8sov8AszyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmxorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAFqzLAw\",\"auditType\":\"Search Action\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-11T15:35:53+0000\",\"eventInfo\":\"Executed Search - Source: Search, Search Criteria: {\\\"keywords\\\":\\\"test\\\",\\\"mailboxes\\\":[\\\"johndoe@example.com\\\"],\\\"route\\\":\\\"ALL\\\",\\\"start\\\":\\\"2021-04-11T16:34:45+0100\\\",\\\"end\\\":\\\"2021-10-11T16:34:45+0100\\\"}, Date: 2021-10-11, Time: 15:35:53+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"category\":\"case_review_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJS0i5MNHQtiqoo9Q53S0yu8sov8AszyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmxorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAFqzLAw", + "created": "2021-10-11T15:35:53.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "mimecast-case-review", + "category": "case_review_logs", + "eventInfo": "Executed Search - Source: Search, Search Criteria: {\"keywords\":\"test\",\"mailboxes\":[\"johndoe@example.com\"],\"route\":\"ALL\",\"start\":\"2021-04-11T16:34:45+0100\",\"end\":\"2021-10-11T16:34:45+0100\"}, Date: 2021-10-11, Time: 15:35:53+0000, IP: 8.8.8.8, Application: mimecast-case-review" + } + }, + { + "@timestamp": "2021-10-11T14:46:10.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "logon-authentication-failed", + "ingested": "2021-11-25T11:34:08.372337400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMk9PdXYMzywJrLLMzdT2TfVN8S8zNgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkbGFmoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACyMK6M\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-11T14:46:10+0000\",\"eventInfo\":\"Creating the auditLog entry for failed authentication, emailAddress :com.example.sdk.address.Address@4a3bcd11[accountCode=ABC123,accountId=75,internal=false,emailAddress=johndoe@gmail.com,domainName=gmail.com,name=johndoe@gmail.com,aliasFor=0,type=0,journalService=false,id=275078533,aliases={},alternateAddresses={},alternateAliases={}] remote IP : 8.8.8.8 application : LFS\",\"category\":\"authentication_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSMk9PdXYMzywJrLLMzdT2TfVN8S8zNgxL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkbGFmoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWACyMK6M" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "eventInfo": "Creating the auditLog entry for failed authentication, emailAddress :com.example.sdk.address.Address@4a3bcd11[accountCode=ABC123,accountId=75,internal=false,emailAddress=johndoe@gmail.com,domainName=gmail.com,name=johndoe@gmail.com,aliasFor=0,type=0,journalService=false,id=275078533,aliases={},alternateAddresses={},alternateAliases={}] remote IP : 8.8.8.8 application : LFS", + "application": "LFS", + "category": "authentication_logs", + "email": { + "metadata": "accountCode=ABC123,accountId=75,internal=false,emailAddress=johndoe@gmail.com,domainName=gmail.com,name=johndoe@gmail.com,aliasFor=0,type=0,journalService=false,id=275078533,aliases={},alternateAddresses={},alternateAliases={}", + "address": "com.example.sdk.address.Address@4a3bcd11" + } + } + }, + { + "@timestamp": "2021-10-11T13:21:06.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "" + ] + }, + "event": { + "action": "completed-directory-sync", + "ingested": "2021-11-25T11:34:08.372338400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSKnU29DVI9XJJMs6wMC9LqnAMccoxcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkZGZqoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPQMKys\",\"auditType\":\"Completed Directory Sync\",\"user\":\"\",\"eventTime\":\"2021-10-11T13:21:06+0000\",\"eventInfo\":\"No changes found.\",\"category\":\"account_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSKnU29DVI9XJJMs6wMC9LqnAMccoxcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkZGZqoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPQMKys" + }, + "user": { + "email": "" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "category": "account_logs", + "eventInfo": "No changes found." + } + }, + { + "@timestamp": "2021-10-12T09:19:53.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "case-action", + "ingested": "2021-11-25T11:34:08.372339500Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSSiwLM8srLCvJzg8s8HbydCpz0Y6oCAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaG5ooKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAHTYLDo\",\"auditType\":\"Case Action\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T09:19:53+0000\",\"eventInfo\":\"Viewed Case - Case: Class Action, Date: 2021-10-12, Time: 09:19:53+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"category\":\"case_review_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSSiwLM8srLCvJzg8s8HbydCpz0Y6oCAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaG5ooKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAHTYLDo", + "created": "2021-10-12T09:19:53.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "mimecast-case-review", + "category": "case_review_logs", + "eventInfo": "Viewed Case - Case: Class Action, Date: 2021-10-12, Time: 09:19:53+0000, IP: 8.8.8.8, Application: mimecast-case-review" + } + }, + { + "@timestamp": "2021-10-12T08:47:55.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "reason": "Reason: Wrong password", + "action": "logon-authentication-failed", + "ingested": "2021-11-25T11:34:08.372340500Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-12, Time: 09:47:55 BST, IP: 8.8.8.8, Application: mimecast-moa, Method: Office 365, Reason: Wrong password\",\"category\":\"authentication_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg", + "created": "2021-10-11T22:47:55.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "mimecast-moa", + "category": "authentication_logs", + "eventInfo": "Failed authentication for johndoe@example.com \u003cJohn Doe\u003e, Date: 2021-10-12, Time: 09:47:55 BST, IP: 8.8.8.8, Application: mimecast-moa, Method: Office 365, Reason: Wrong password" + } + }, + { + "@timestamp": "2021-10-12T08:47:54.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johdoe", + "johdoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "existing-archive-task-changed", + "ingested": "2021-11-25T11:34:08.372341400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSSnJMinKNMMtyDg3xKw2rDM91DC-JdAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRooaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAEQYK9w\",\"auditType\":\"Existing Archive Task Changed\",\"user\":\"johdoe@example.com\",\"eventTime\":\"2021-10-12T08:47:54+0000\",\"eventInfo\":\"Successfully updated 3 'Sync and Recover' tasks associated with legacy connection (\\\"365\\\") to new migrated connector (\\\"Sync and Recover - 365\\\"), Date: 2021-10-12, Time: 08:47:54+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"archive_service_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSSnJMinKNMMtyDg3xKw2rDM91DC-JdAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRooaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAEQYK9w", + "created": "2021-10-12T08:47:54.000Z" + }, + "user": { + "name": "johdoe", + "email": "johdoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "archive_service_logs", + "eventInfo": "Successfully updated 3 'Sync and Recover' tasks associated with legacy connection (\"365\") to new migrated connector (\"Sync and Recover - 365\"), Date: 2021-10-12, Time: 08:47:54+0000, IP: 8.8.8.8, Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T08:47:53.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "connectors-management", + "ingested": "2021-11-25T11:34:08.372342400Z", + "original": "{\"id\":\"eNoVzc0KgkAUQOF3uVsFuZma7qQ0UqiFqChuZH7M0iZmHMOid8_2h-98QDGiJespBDBgYwn-4orcHMrr_JqUWdjFBb8YThbF5bE6le_ardLGitJqnHF39w7YGuLsL5g8l7wAE1pN-2kQ3V-00bdt3KBrAtFqEiOTRFC2rvZbN_ScNZ-ZVL14QIDfH41XLGM\",\"auditType\":\"Connectors Management\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T08:47:53+0000\",\"eventInfo\":\"Connector creation for Microsoft O365\\nName: Sync and Recover - 365, Description: null, Product: Sync and Recover, App (provider): Microsoft O365\\nSuccess: true, Date: 2021-10-12, Time: 08:47:53+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"integrations_and_apis\"}", + "id": "eNoVzc0KgkAUQOF3uVsFuZma7qQ0UqiFqChuZH7M0iZmHMOid8_2h-98QDGiJespBDBgYwn-4orcHMrr_JqUWdjFBb8YThbF5bE6le_ardLGitJqnHF39w7YGuLsL5g8l7wAE1pN-2kQ3V-00bdt3KBrAtFqEiOTRFC2rvZbN_ScNZ-ZVL14QIDfH41XLGM", + "created": "2021-10-12T08:47:53.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "integrations_and_apis", + "eventInfo": "Connector creation for Microsoft O365\nName: Sync and Recover - 365, Description: null, Product: Sync and Recover, App (provider): Microsoft O365\nSuccess: true, Date: 2021-10-12, Time: 08:47:53+0000, IP: 8.8.8.8, Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T02:27:18.000Z", + "file": { + "size": 6864, + "name": "export_at_watchlist_view_1634005638160.xlsx", + "extension": ".xlsx" + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johdoe", + "johdoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "page-data-exports", + "ingested": "2021-11-25T11:34:08.372343300Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSynAJ8yuoyA4z9ygMNyv21C42MC9IDwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkbmFhoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADk2K8U\",\"auditType\":\"Page Data Exports\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T02:27:18+0000\",\"eventInfo\":\"[Export type : Download,Name :watchlist_view,Requested By :johdoe@example.com,Export time :Tue Oct 12 03:27:18 BST 2021,IP Address :8.8.8.8,Columns exported :Name|Email|Department|Number of Videos|,File name : export_at_watchlist_view_1634005638160.xlsx,File Size: 6864,File type : .xlsx], Date: 2021-10-12, Time: 02:27:18+0000, IP: 8.8.8.8, Application: mimecast-matfe\",\"category\":\"account_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSynAJ8yuoyA4z9ygMNyv21C42MC9IDwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkbmFhoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADk2K8U", + "created": "2021-10-12T02:27:18.000Z" + }, + "user": { + "name": "johdoe", + "email": "johdoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "mimecast-matfe", + "category": "account_logs", + "eventInfo": "[Export type : Download,Name :watchlist_view,Requested By :johdoe@example.com,Export time :Tue Oct 12 03:27:18 BST 2021,IP Address :8.8.8.8,Columns exported :Name|Email|Department|Number of Videos|,File name : export_at_watchlist_view_1634005638160.xlsx,File Size: 6864,File type : .xlsx], Date: 2021-10-12, Time: 02:27:18+0000, IP: 8.8.8.8, Application: mimecast-matfe" + } + }, + { + "@timestamp": "2021-10-11T19:53:41.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.local" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "custom-report-definition-created", + "ingested": "2021-11-25T11:34:08.372344400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMi8zSc3J8M4Od_NwjdHPMDYzdfGO8MkJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGppaKajlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAmqSuF\",\"auditType\":\"Custom Report Definition Created\",\"user\":\"johndoe@example.local\",\"eventTime\":\"2021-10-11T19:53:41+0000\",\"eventInfo\":\"Action Performed - Custom Report Definition Created with name \\\"Terri test\\\" and description \\\"all user - per email report\\\" by johndoe@example.local\u003cjohndoe@example.local\u003e Date: 2021-10-11 Time: 20:53:41 +0100 IP: 8.8.8.8 Application: Administration Console\",\"category\":\"reporting_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSMi8zSc3J8M4Od_NwjdHPMDYzdfGO8MkJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGppaKajlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAmqSuF", + "created": "2021-10-11T20:53:41.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.local", + "domain": "example.local" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "reporting_logs", + "eventInfo": "Action Performed - Custom Report Definition Created with name \"Terri test\" and description \"all user - per email report\" by johndoe@example.local\u003cjohndoe@example.local\u003e Date: 2021-10-11 Time: 20:53:41 +0100 IP: 8.8.8.8 Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-11T18:23:10.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "John Doe" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "folder-log-entry", + "ingested": "2021-11-25T11:34:08.372345400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSCij080lzDChMMjXw8o3IjnCLDIrRT8wJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGpiYaqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgBNvCvh\",\"auditType\":\"Folder Log Entry\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-11T18:23:10+0000\",\"eventInfo\":\"Action Performed - Deleted New Folder by johndoe@example.com\u003cJohn Doe\u003e Date: 2021-10-11 Time: 19:23:10 +0100 IP: 8.8.8.8 Application: Administration Console\",\"category\":\"profile_group_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSCij080lzDChMMjXw8o3IjnCLDIrRT8wJS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsZGpiYaqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgBNvCvh", + "created": "2021-10-11T19:23:10.000Z" + }, + "user": { + "email": "John Doe" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "profile_group_logs", + "eventInfo": "Action Performed - Deleted New Folder by johndoe@example.com\u003cJohn Doe\u003e Date: 2021-10-11 Time: 19:23:10 +0100 IP: 8.8.8.8 Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T19:56:55.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ] + }, + "event": { + "action": "user-password-changed", + "ingested": "2021-11-25T11:34:08.372346400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSCtF28jc2DDLwd_d1NM7ULnLzdnPzdwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiCAQ6SsmlxSX5ualFyfkpqUCbnE3MHM1NgcrLUouKM_PzlKwMawGTZipR\",\"auditType\":\"User Password Changed\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T19:56:55+0000\",\"eventInfo\":\"Password reset for user: johndoe@example.com User Password Changed, Remote IP is null\",\"category\":\"user_account_and_role_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSCtF28jc2DDLwd_d1NM7ULnLzdnPzdwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiCAQ6SsmlxSX5ualFyfkpqUCbnE3MHM1NgcrLUouKM_PzlKwMawGTZipR" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "category": "user_account_and_role_logs", + "eventInfo": "Password reset for user: johndoe@example.com User Password Changed, Remote IP is null" + } + }, + { + "@timestamp": "2021-10-12T19:49:30.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "remediation-incident-adjustment", + "ingested": "2021-11-25T11:34:08.372347400Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSSgwpLctzzah00TbMTTawdC4NDPAzzwlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiaGBhoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADOfK6w\",\"auditType\":\"Remediation Incident Adjustment\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T19:49:30+0000\",\"eventInfo\":\"Remediation Incident Created - TR-C46A75-01420-M, type : manual, search criteria : {\\\"fileHash\\\":\\\"9e6011844705292d5abfe0aa38d8aff02f6d8f69689c2e7cb2338f9484774bb3\\\",\\\"start\\\":\\\"2021-09-12T19:48:59+0000\\\",\\\"end\\\":\\\"2021-10-12T19:48:59+0000\\\"}, Date: 2021-10-12, Time: 19:49:30+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"account_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSSgwpLctzzah00TbMTTawdC4NDPAzzwlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiaGBhoqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADOfK6w", + "type": "type : manual", + "created": "2021-10-12T19:49:30.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "account_logs", + "eventInfo": "Remediation Incident Created - TR-C46A75-01420-M, type : manual, search criteria : {\"fileHash\":\"9e6011844705292d5abfe0aa38d8aff02f6d8f69689c2e7cb2338f9484774bb3\",\"start\":\"2021-09-12T19:48:59+0000\",\"end\":\"2021-10-12T19:48:59+0000\"}, Date: 2021-10-12, Time: 19:49:30+0000, IP: 8.8.8.8, Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T19:20:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "archive-mailbox-restore", + "ingested": "2021-11-25T11:34:08.372348400Z", + "original": "{\"id\":\"eNoVzdEKgjAYQOF3-W8Vaps69S7KooSEJGXSzdAVMtdi04FF757dH77zASvayYi-gxQIcbI0HEtcRI5aRS7SxkN1L7ywzPb1gR3rdxOx_LbKcqYciiXdIe7pczKj02u-VuADn7p-HPTjDxKUkGCdUOxDO9lRK2Fa3YnltA2iDQ2X3Alje_2EFH1_LYQrrw\",\"auditType\":\"Archive Mailbox Restore\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T19:20:01+0000\",\"eventInfo\":\"Archive mailbox restore created. Restored data from johdoe@example.com to johndoe@example.com by johndoe@example.com, Date: 2021-10-12, Time: 19:20:01+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"archive_service_logs\"}", + "id": "eNoVzdEKgjAYQOF3-W8Vaps69S7KooSEJGXSzdAVMtdi04FF757dH77zASvayYi-gxQIcbI0HEtcRI5aRS7SxkN1L7ywzPb1gR3rdxOx_LbKcqYciiXdIe7pczKj02u-VuADn7p-HPTjDxKUkGCdUOxDO9lRK2Fa3YnltA2iDQ2X3Alje_2EFH1_LYQrrw", + "created": "2021-10-12T19:20:01.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "archive_service_logs", + "eventInfo": "Archive mailbox restore created. Restored data from johdoe@example.com to johndoe@example.com by johndoe@example.com, Date: 2021-10-12, Time: 19:20:01+0000, IP: 8.8.8.8, Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T18:19:33.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoejr", + "johndoejr@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "archive-mailbox-restore", + "ingested": "2021-11-25T11:34:08.372349300Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSigzJC_ZNzg-vcjYKcwz3icotC0nVdgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYG5kqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAD-SK84\",\"auditType\":\"Archive Mailbox Restore\",\"user\":\"johndoejr@example.com\",\"eventTime\":\"2021-10-12T18:19:33+0000\",\"eventInfo\":\"Archive mailbox restore created. Restored data from johndoe@example.com to johndoejr@example.com by johndoejr@example.com, Date: 2021-10-12, Time: 18:19:33+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"archive_service_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSigzJC_ZNzg-vcjYKcwz3icotC0nVdgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYG5kqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAD-SK84", + "created": "2021-10-12T18:19:33.000Z" + }, + "user": { + "name": "johndoejr", + "email": "johndoejr@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "archive_service_logs", + "eventInfo": "Archive mailbox restore created. Restored data from johndoe@example.com to johndoejr@example.com by johndoejr@example.com, Date: 2021-10-12, Time: 18:19:33+0000, IP: 8.8.8.8, Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T17:55:14.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "archive-mailbox-export-download", + "ingested": "2021-11-25T11:34:08.372350300Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJScjMvyjIxr6yoLDY2qQopLq3yDnM1dwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYGZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE5dK-0\",\"auditType\":\"Archive Mailbox Export Download\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T17:55:14+0000\",\"eventInfo\":\"Mailbox export downloaded. Download filename (HTML Report recovery id): eNqrVipOTS4tSs1MUbJSyo3RDw81rTCpynMpdiuICMopyihxynZztcisDMoN9zWLSCrPzAjz9PALNzFwySrLMNQ2yUs38g9zS860cHKNMExR0lFKLi0uyc9NLUrOT0kFGulsYuZobgoUL0pNzi9LLarULUksztYFWWdpaKqjBBQqzszPU7IyrAUAsSEteA by johndoe@example.com, Date: 2021-10-12, Time: 17:55:14+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"archive_service_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJScjMvyjIxr6yoLDY2qQopLq3yDnM1dwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYGZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE5dK-0", + "created": "2021-10-12T17:55:14.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "archive_service_logs", + "eventInfo": "Mailbox export downloaded. Download filename (HTML Report recovery id): eNqrVipOTS4tSs1MUbJSyo3RDw81rTCpynMpdiuICMopyihxynZztcisDMoN9zWLSCrPzAjz9PALNzFwySrLMNQ2yUs38g9zS860cHKNMExR0lFKLi0uyc9NLUrOT0kFGulsYuZobgoUL0pNzi9LLarULUksztYFWWdpaKqjBBQqzszPU7IyrAUAsSEteA by johndoe@example.com, Date: 2021-10-12, Time: 17:55:14+0000, IP: 8.8.8.8, Application: Administration Console" + } + }, + { + "@timestamp": "2021-10-12T17:07:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "review-set-action", + "ingested": "2021-11-25T11:34:08.372351300Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSitH39gl1cS509PT1MSnw90l0CinPCQgLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsYmBsYqqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAxASul\",\"auditType\":\"Review Set Action\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T17:07:00+0000\",\"eventInfo\":\"Viewed Review Set Details - Case: Class Action, Review Set: Contracts, Date: 2021-10-12, Time: 17:07:00+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"category\":\"case_review_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJSitH39gl1cS509PT1MSnw90l0CinPCQgLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsYmBsYqqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAxASul", + "created": "2021-10-12T17:07:00.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "mimecast-case-review", + "category": "case_review_logs", + "eventInfo": "Viewed Review Set Details - Case: Class Action, Review Set: Contracts, Date: 2021-10-12, Time: 17:07:00+0000, IP: 8.8.8.8, Application: mimecast-case-review" + } + }, + { + "@timestamp": "2021-10-12T15:38:05.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "client": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "ip": "8.8.8.8" + }, + "event": { + "action": "remediation-incident-adjustment", + "ingested": "2021-11-25T11:34:08.372352300Z", + "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38\",\"auditType\":\"Remediation Incident Adjustment\",\"user\":\"johndoe@example.com\",\"eventTime\":\"2021-10-12T15:38:05+0000\",\"eventInfo\":\"Restore Remediation Incident Created - TR-C46A75-01419-R, type : restore, search criteria : {\\\"unremediateCode\\\":\\\"TR-C46A75-01419-M\\\",\\\"from\\\":\\\"gmail.com\\\",\\\"start\\\":\\\"2021-10-10T15:33:49+0000\\\",\\\"end\\\":\\\"2021-10-12T15:33:49+0000\\\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 8.8.8.8, Application: Administration Console\",\"category\":\"account_logs\"}", + "id": "eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38", + "type": "type : restore", + "created": "2021-10-12T15:38:05.000Z" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "application": "Administration Console", + "category": "account_logs", + "eventInfo": "Restore Remediation Incident Created - TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 8.8.8.8, Application: Administration Console" + } + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/audit_events/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/audit_events/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..c128c658d01 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/audit/get-audit-events diff --git a/packages/mimecast/data_stream/audit_events/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/audit_events/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..7d7a875d1a2 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/agent/stream/httpjson.yml.hbs @@ -0,0 +1,51 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: +- set: + target: body.meta.pagination.pageSize + value: 500 +- set: + target: body.data + value: '[{"endDateTime": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "startDateTime":"[[.cursor.next_date]]"}]' + default: '[{"endDateTime": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "startDateTime":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json +- set: + target: header.x-mc-app-id + value: {{app_id}} +- set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' +- set: + target: header.x-mc-req-id + value: '[[uuid]]' +- set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/audit/get-audit-events:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + target: body.data +response.pagination: +- set: + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + fail_on_template_error: true +cursor: + next_date: + value: '[[.first_event.eventTime]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..f75b58aca00 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,243 @@ +--- +description: Pipeline for processing sample logs +processors: + # # Generic event/ecs fields we always want to populate + - set: + field: event.ingested + value: "{{ _ingest.timestamp }}" + - set: + field: ecs.version + value: "1.12.0" + - rename: + field: message + target_field: event.original + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.eventTime == null + - date: + description: Use 'mimecast.eventTime' as the '@timestamp' + field: mimecast.eventTime + timezone: UTC + formats: + - yyyy-MM-dd'T'HH:mm:ssZ + + ### + + # Convert 'mimecast.auditType' to a bone-cased event action. + # ie: User Log On -> user-log-on + - lowercase: + field: mimecast.auditType + ignore_missing: true + - gsub: + field: mimecast.auditType + pattern: " " + replacement: "-" + ignore_missing: true + - rename: + field: mimecast.auditType + target_field: event.action + ignore_missing: true + ### + + # User fields + - rename: + field: mimecast.user + target_field: user.email + ignore_missing: true + - rename: + field: mimecast.id + target_field: event.id + ignore_missing: true + ### + # Here we want to add as much categorization information as possible + # We can do this by parsing mimecast.eventInfo differently based on + # what event.action is, etc. + ### + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.filename}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "threat-intel-feed-download" || ctx?.event?.action == "existing-archive-task-changed" || ctx?.event?.action == "case-action" || ctx?.event?.action == "user-logged-on" || ctx?.event?.action == "logon-requires-challenge"' + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{event.type}, %{mimecast.search}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "remediation-incident-adjustment"' + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{mimecast.type}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "review-set-action"' + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "archive-mailbox-export-download" || ctx?.event?.action == "archive-mailbox-restore"' #logon-authentication-failed + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{mimecast.description}, %{mimecast.product}, %{mimecast.provider}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "connectors-management"' + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{mimecast.criteria}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "search-action"' + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{?key}: <%{mimecast.name.from}> %{email.from.address}, %{?key}: <%{mimecast.name.to}> %{email.to.address}, %{?key}: %{email.subject}, %{?key}: %{email.origination_timestamp}, %{?key}: %{mimecast.viewed}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action == "message-action"' + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}, %{mimecast.application_method}, %{event.reason}" + if: 'ctx?.event?.action=="logon-authentication-failed"' + ignore_missing: true + ignore_failure: true + - dissect: + field: mimecast.eventInfo + pattern: "%{mimecast.info}, %{?key}:%{mimecast.email.address}[%{mimecast.email.metadata}] %{?key}: %{client.ip} %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action=="logon-authentication-failed"' + ignore_missing: true + ignore_failure: true + - dissect: + field: mimecast.eventInfo + pattern: "%{?drop->} - %{mimecast.info}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action=="folder-log-entry" || ctx?.event?.action=="custom-report-definition-created"' + - dissect: + field: mimecast.eventInfo + pattern: "%{?drop->} - %{mimecast.info}. %{mimecast.byuser}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action=="mimecast-support-login"' + - dissect: + field: mimecast.eventInfo + pattern: "[%{?key} : %{mimecast.export_type},%{?key} :%{mimecast.export_name},%{?key} :%{user.email},%{?key} :%{mimecast.weekday} %{mimecast.month} %{mimecast.monthday} %{mimecast.time} %{mimecast.timezone} %{mimecast.year},%{?key} :%{client.ip},%{?key} :%{mimecast.columns_exported},%{?key} : %{file.name},%{?key}: %{file.size},%{?key} : %{file.extension}], %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" + if: 'ctx?.event?.action=="page-data-exports"' + - convert: + field: file.size + type: long + ignore_missing: true + - split: + field: user.email + separator: "@" + target_field: user.parts + if: 'ctx?.user?.email != null' + - set: + field: user.name + copy_from: user.parts.0 + if: 'ctx?.user?.parts !=null && ctx?.user?.parts.length > 1' + - set: + field: user.domain + copy_from: user.parts.1 + if: 'ctx?.user?.parts !=null && ctx?.user?.parts.length > 1' + - rename: + field: mimecast.filename + target_field: file.name + ignore_missing: true + if: 'ctx?.mimecast?.filename != null && ctx?.event?.action == "threat-intel-feed-download"' + - split: + field: file.name + separator: "\\." + target_field: file.parts + if: 'ctx?.file?.name != null && ctx?.event?.action == "threat-intel-feed-download"' + - script: + lang: painless + source: | + ctx.file.extension = ctx.file.parts[ctx.file.parts.length-1]; + if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' + - set: + field: event.created + value: "{{mimecast.date}} {{mimecast.time}}" + if: 'ctx?.mimecast?.date != null && ctx?.mimecast?.time != null' + - date: + field: event.created + target_field: event.created + timezone: UTC + formats: + - yyyy-MM-dd HH:mm:ssZ + - yyyy-MM-dd HH:mm:ss z + - yyyy-MM-dd HH:mm:ss + if: 'ctx?.event?.created != null' + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + - append: + field: related.ip + value: "{{client.ip}}" + allow_duplicates: false + if: 'ctx?.client?.ip !=null' + - append: + field: related.user + value: "{{user.name}}" + allow_duplicates: false + if: 'ctx?.user?.name !=null' + - append: + field: related.user + value: "{{user.email}}" + allow_duplicates: false + if: ctx?.user?.email != null + # Cleanup + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast.eventTime + - user.parts + - mimecast.date + - mimecast.time + - file.parts + - mimecast.info + - mimecast.type + - mimecast.search + - mimecast.description + - mimecast.product + - mimecast.provider + - mimecast.filename + - mimecast.criteria + - mimecast.aplication_method + - mimecast.name.to + - mimecast.name.from + - mimecast.viewed + - mimecast.application_method + - mimecast.timezone + - mimecast.byuser + - mimecast.export_type + - mimecast.export_name + - mimecast.weekday + - mimecast.month + - mimecast.monthday + - mimecast.year + - mimecast.columns_exported + - mimecast.as.asn + - mimecast.organization_name + ignore_missing: true + - remove: + description: Remove 'event.original' if 'preserve_original_event' is not set. + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + - remove: + description: Remove 'source.ip' if 'auditType' is not set. + field: source.ip + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + + + ### + +# Error handling +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/mimecast/data_stream/audit_events/fields/agent.yml b/packages/mimecast/data_stream/audit_events/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/audit_events/fields/base-fields.yml b/packages/mimecast/data_stream/audit_events/fields/base-fields.yml new file mode 100644 index 00000000000..d705e5e80a0 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.audit_events +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/audit_events/fields/ecs.yml b/packages/mimecast/data_stream/audit_events/fields/ecs.yml new file mode 100644 index 00000000000..2a6db9bbf6f --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/fields/ecs.yml @@ -0,0 +1,72 @@ +- external: ecs + name: event.original +- external: ecs + name: event.action +- external: ecs + name: user.email +- external: ecs + name: event.id +- external: ecs + name: tags +- external: ecs + name: ecs.version +- external: ecs + name: client.ip +- external: ecs + name: file.name +- external: ecs + name: user.name +- external: ecs + name: user.domain +- external: ecs + name: file.extension +- external: ecs + name: client.geo.city_name +- external: ecs + name: client.geo.continent_name +- external: ecs + name: client.geo.country_iso_code +- external: ecs + name: client.geo.country_name +- description: Longitude and latitude. + level: core + name: client.geo.location + type: geo_point +- external: ecs + name: client.geo.region_iso_code +- external: ecs + name: client.geo.region_name +- description: Client ASN number. + name: client.as.asn + type: long +- descriiption: Client Organization name. + name: client.as.organization_name + type: keyword +- external: ecs + name: client.as.number +- external: ecs + name: client.as.organization.name +- description: The email address(es) of the message recipient(s) + type: keyword + name: email.to.address +- description: Stores the from email address from the RFC5322 From - header field. + type: keyword + name: email.from.address +- description: A brief summary of the topic of the message + type: keyword + name: email.subject + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. + type: date + name: email.origination_timestamp +- external: ecs + name: file.size +- external: ecs + name: related.ip +- external: ecs + name: related.user diff --git a/packages/mimecast/data_stream/audit_events/fields/field.yml b/packages/mimecast/data_stream/audit_events/fields/field.yml new file mode 100644 index 00000000000..be7e5f2a870 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/fields/field.yml @@ -0,0 +1,18 @@ +- name: mimecast + type: group + fields: + - name: category + type: keyword + description: The category of the event. + - name: eventInfo + type: keyword + description: The detailed event information. + - name: application + type: keyword + description: The Mimecast unique id of the event. + - name: email.metadata + type: keyword + description: The email meta data from audit info. + - name: email.address + type: keyword + description: Email address from event info. diff --git a/packages/mimecast/data_stream/audit_events/manifest.yml b/packages/mimecast/data_stream/audit_events/manifest.yml new file mode 100644 index 00000000000..bf4c8534560 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/manifest.yml @@ -0,0 +1,77 @@ +title: "Audit Events Mimecast Logs" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: Audit events + description: Collect audit Events Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: text + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/audit/get-audit-events + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-audit-events + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/audit_events/sample_event.json b/packages/mimecast/data_stream/audit_events/sample_event.json new file mode 100644 index 00000000000..473cd0af7f3 --- /dev/null +++ b/packages/mimecast/data_stream/audit_events/sample_event.json @@ -0,0 +1,78 @@ +{ + "@timestamp": "2021-11-16T12:01:37.000Z", + "agent": { + "ephemeral_id": "57841034-22ed-4fcd-bcfd-0a9518249e2d", + "hostname": "docker-fleet-agent", + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.16.0" + }, + "client": { + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "geo": { + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 37.751, + "lon": -97.822 + } + }, + "ip": "8.8.8.8" + }, + "data_stream": { + "dataset": "mimecast.audit_events", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "snapshot": true, + "version": "7.16.0" + }, + "event": { + "action": "case-action", + "agent_id_status": "verified", + "created": "2021-11-16T12:01:37.000Z", + "dataset": "mimecast.audit_events", + "id": "eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI", + "ingested": "2021-11-24T15:39:11Z", + "original": "{\"auditType\":\"Case Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI\",\"user\":\"johndoe@example.com\"}" + }, + "input": { + "type": "httpjson" + }, + "mimecast": { + "application": "mimecast-case-review", + "category": "case_review_logs", + "eventInfo": "Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review" + }, + "related": { + "ip": [ + "8.8.8.8" + ], + "user": [ + "johndoe", + "johndoe@example.com" + ] + }, + "tags": [ + "preserve_original_event", + "forwarded", + "mimecast-audit-events" + ], + "user": { + "domain": "example.com", + "email": "johndoe@example.com", + "name": "johndoe" + } +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log new file mode 100644 index 00000000000..30571cfc4c8 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log @@ -0,0 +1,10 @@ +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:25+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"hold","messageId":"<20211015204122.2CA6DFCAE2@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:25+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"notification","messageId":"<20211015204122.2CA6DFCAE2@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:22+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"hold","messageId":"<20211015204119.F16C2FCC60@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:22+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"notification","messageId":"<20211015204119.F16C2FCC60@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:21+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"notification","messageId":"<20211015204118.05EA6FCAE2@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:21+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"hold","messageId":"<20211015204118.05EA6FCAE2@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:19+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"notification","messageId":"<20211015204116.6A8CFFCC60@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:19+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"hold","messageId":"<20211015204116.6A8CFFCC60@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:17+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"hold","messageId":"<20211015204114.8AE40FCAE2@mail.emailsec.ninja>"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","subject":"Undelivered Mail Returned to Sender","eventTime":"2021-10-15T20:41:17+0000","route":"inbound","policy":"Content Inspection - Watermark","action":"notification","messageId":"<20211015204114.8AE40FCAE2@mail.emailsec.ninja>"} \ No newline at end of file diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json new file mode 100644 index 00000000000..7bba78b2f50 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json @@ -0,0 +1,294 @@ +{ + "expected": [ + { + "@timestamp": "2021-10-15T20:41:25.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "hold", + "ingested": "2021-11-25T11:34:10.753237800Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:25+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"hold\",\"messageId\":\"\u003c20211015204122.2CA6DFCAE2@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:25+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204122.2CA6DFCAE2@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:25.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "notification", + "ingested": "2021-11-25T11:34:10.753244800Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:25+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"notification\",\"messageId\":\"\u003c20211015204122.2CA6DFCAE2@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:25+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204122.2CA6DFCAE2@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:22.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "hold", + "ingested": "2021-11-25T11:34:10.753246400Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:22+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"hold\",\"messageId\":\"\u003c20211015204119.F16C2FCC60@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:22+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204119.F16C2FCC60@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:22.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "notification", + "ingested": "2021-11-25T11:34:10.753248100Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:22+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"notification\",\"messageId\":\"\u003c20211015204119.F16C2FCC60@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:22+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204119.F16C2FCC60@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:21.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "notification", + "ingested": "2021-11-25T11:34:10.753249600Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:21+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"notification\",\"messageId\":\"\u003c20211015204118.05EA6FCAE2@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:21+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204118.05EA6FCAE2@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:21.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "hold", + "ingested": "2021-11-25T11:34:10.753250900Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:21+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"hold\",\"messageId\":\"\u003c20211015204118.05EA6FCAE2@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:21+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204118.05EA6FCAE2@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:19.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "notification", + "ingested": "2021-11-25T11:34:10.753252100Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:19+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"notification\",\"messageId\":\"\u003c20211015204116.6A8CFFCC60@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:19+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204116.6A8CFFCC60@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:19.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "hold", + "ingested": "2021-11-25T11:34:10.753253400Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:19+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"hold\",\"messageId\":\"\u003c20211015204116.6A8CFFCC60@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:19+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204116.6A8CFFCC60@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:17.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "hold", + "ingested": "2021-11-25T11:34:10.753254700Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:17+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"hold\",\"messageId\":\"\u003c20211015204114.8AE40FCAE2@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:17+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204114.8AE40FCAE2@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-10-15T20:41:17.000Z", + "ecs": { + "version": "1.12.0" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "event": { + "action": "notification", + "ingested": "2021-11-25T11:34:10.753256Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Undelivered Mail Returned to Sender\",\"eventTime\":\"2021-10-15T20:41:17+0000\",\"route\":\"inbound\",\"policy\":\"Content Inspection - Watermark\",\"action\":\"notification\",\"messageId\":\"\u003c20211015204114.8AE40FCAE2@mail.emailsec.ninja\u003e\"}", + "created": "2021-10-15T20:41:17+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211015204114.8AE40FCAE2@mail.emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Undelivered Mail Returned to Sender", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/dlp_logs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..ccc006a822c --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/dlp/get-logs diff --git a/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..0f2331d944e --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs @@ -0,0 +1,50 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: + - set: + target: body.data + value: '[{"to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[.cursor.eventTime]]"}]' + default: '[{"to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: {{app_id}} + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: "[[uuid]]" + - set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/dlp/get-logs:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + target: body.data + split: + target: body.dlpLogs +response.pagination: +- set: + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + fail_on_template_error: true +cursor: +next_date: + value: '[[.first_event.eventTime]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..38695f2a016 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,87 @@ +--- +description: Pipeline for processing sample logs +processors: + # Generic event/ecs fields we always want to populated + - set: + field: event.ingested + value: "{{ _ingest.timestamp }}" + - set: + field: ecs.version + value: "1.12.0" + - rename: + field: message + target_field: event.original + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.eventTime == null + - date: + description: Use 'mimecast.eventTime' as the '@timestamp' + field: mimecast.eventTime + timezone: UTC + formats: + - yyyy-MM-dd'T'HH:mm:ssZ + + ### + + # Convert 'mimecast.auditType' to a bone-cased event action. + # ie: User Log On -> user-log-on + - rename: + field: mimecast.senderAddress + target_field: email.from.address + ignore_missing: true + if: 'ctx?.mimecast?.senderAddress !=null' + - rename: + field: mimecast.recipientAddress + target_field: email.to.address + ignore_missing: true + if: 'ctx?.mimecast?.recipientAddress !=null' + - rename: + field: mimecast.action + target_field: event.action + ignore_missing: true + if: 'ctx?.mimecast?.action !=null' + - rename: + field: mimecast.subject + target_field: email.subject + ignore_missing: true + if: 'ctx?.mimecast?.subject !=null' + - rename: + field: mimecast.messageId + target_field: email.message_id + ignore_missing: true + if: 'ctx?.mimecast?.messageId !=null' + - rename: + field: mimecast.route + target_field: email.direction + ignore_missing: true + if: 'ctx?.mimecast?.route !=null' + - rename: + field: mimecast.policy + target_field: rule.name + ignore_missing: true + if: 'ctx?.mimecast?.policy !=null' + - set: + field: event.created + value: "{{mimecast.eventTime}}" + if: 'ctx?.mimecast?.eventTime != null' + + # Cleanup + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast + ignore_missing: true + - remove: + description: Remove 'event.original' if 'preserve_original_event' is not set. + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + +# Error handling +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/mimecast/data_stream/dlp_logs/fields/agent.yml b/packages/mimecast/data_stream/dlp_logs/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/dlp_logs/fields/base-fields.yml b/packages/mimecast/data_stream/dlp_logs/fields/base-fields.yml new file mode 100644 index 00000000000..7b49c3eae72 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.dlp_logs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml new file mode 100644 index 00000000000..b540179f326 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml @@ -0,0 +1,36 @@ +- external: ecs + name: event.original +- external: ecs + name: event.action +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. + type: wildcard + name: email.message_id + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: Direction of the message based on the sending and receiving domains + type: keyword + name: email.direction +- external: ecs + name: rule.name +- external: ecs + name: tags +- external: ecs + name: ecs.version +- description: The email address(es) of the message recipient(s) + type: keyword + name: email.to.address +- description: Stores the from email address from the RFC5322 From - header field. + type: keyword + name: email.from.address +- description: A brief summary of the topic of the message + type: keyword + name: email.subject + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false diff --git a/packages/mimecast/data_stream/dlp_logs/fields/field.yml b/packages/mimecast/data_stream/dlp_logs/fields/field.yml new file mode 100644 index 00000000000..36a1bbebc9c --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/fields/field.yml @@ -0,0 +1,24 @@ +- name: mimecast + type: group + fields: + - name: senderAddress + type: keyword + description: Email address of the sender. + - name: action + type: keyword + description: The action taken against the message. + - name: messageId + type: keyword + description: The message-id value of the message. + - name: subject + type: keyword + description: The message subject. + - name: route + type: keyword + description: The message direction. Possible values are inbound, outbound or internal. + - name: policy + type: keyword + description: The name of a DLP or Content Examination configuration that triggered the message. + - name: recipientAddress + type: keyword + description: Email address of the recipient. diff --git a/packages/mimecast/data_stream/dlp_logs/manifest.yml b/packages/mimecast/data_stream/dlp_logs/manifest.yml new file mode 100644 index 00000000000..2d228f27fb2 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/manifest.yml @@ -0,0 +1,77 @@ +title: "DLP Mimecast Logs" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: DLP Logs + description: Collect DLP Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: text + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/dlp/get-logs + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-dlp-logs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/dlp_logs/sample_event.json b/packages/mimecast/data_stream/dlp_logs/sample_event.json new file mode 100644 index 00000000000..88b952d6767 --- /dev/null +++ b/packages/mimecast/data_stream/dlp_logs/sample_event.json @@ -0,0 +1,54 @@ +{ + "@timestamp": "2021-11-18T21:41:18.000Z", + "agent": { + "ephemeral_id": "1aef981f-3448-4d12-bd5a-723ac1cdcc81", + "hostname": "docker-fleet-agent", + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.16.0" + }, + "data_stream": { + "dataset": "mimecast.dlp_logs", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "snapshot": true, + "version": "7.16.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211118214115.B346F10021D@mail.emailsec.ninja\u003e", + "subject": "Undelivered Mail Returned to Sender", + "to": { + "address": "johndoe@example.com" + } + }, + "event": { + "action": "notification", + "agent_id_status": "verified", + "created": "2021-11-18T21:41:18+0000", + "dataset": "mimecast.dlp_logs", + "ingested": "2021-11-24T15:39:49Z", + "original": "{\"action\":\"notification\",\"eventTime\":\"2021-11-18T21:41:18+0000\",\"messageId\":\"\\u003c20211118214115.B346F10021D@mail.emailsec.ninja\\u003e\",\"policy\":\"Content Inspection - Watermark\",\"recipientAddress\":\"johndoe@example.com\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Undelivered Mail Returned to Sender\"}" + }, + "input": { + "type": "httpjson" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "mimecast-dlp-logs" + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log new file mode 100644 index 00000000000..a369af94999 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log @@ -0,0 +1,6 @@ +{"Act":"Hld","AttCnt":0,"AttNames":null,"AttSize":0,"Content-Disposition":"attachment; filename=\"process_20211018093329655.json\"","Hld":"Spm","MsgId":"\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\u003e","MsgSize":157436,"Sender":"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu","Subject":"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!","aCode":"HhuwRf_AOcuJZINE2ZgcKw","acc":"ABC123","datetime":"2021-10-18T09:02:43+0100"} +{"acc":"ABC123","Delivered":false,"IP":"8.8.8.8","RejType":"Recipient email address is possibly incorrect","RejCode":"550","AttCnt":0,"Dir":"Inbound","ReceiptAck":null,"MsgId":null,"Subject":null,"Latency":505,"Sender":"<>","datetime":"2021-10-19T07:06:40+0100","Rcpt":"johndoe@example.com","AttSize":0,"Attempt":1,"RejInfo":"5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]","TlsVer":"TLSv1.2","Cphr":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","Snt":125,"aCode":"29be076e-44cd-354d-a7c2-083d4a312371","UseTls":"Yes","Route":"Office365","Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""} +{"acc":"ABC123","Sender":"postmaster@twotoeight.com","datetime":"2021-10-19T07:04:55+0100","AttSize":0,"Content-Disposition":"attachment; filename=\"process_20211018093329655.json\"","Act":"Acc","aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","AttCnt":0,"AttNames":null,"MsgSize":49025,"MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":"You have new held messages"} +{"acc":"ABC123","Delivered":true,"IP":"8.8.8.8","AttCnt":0,"Dir":"Internal","ReceiptAck":"250 SmtpInternalThread-19194240-1634623495703@uk-mta-151.uk.mimecast.lan Received OK [61dfe7da-4c6d-34e1-9667-69b04f0d564f.uk151]","MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":null,"Latency":1090,"Sender":"johndoe@example.com","datetime":"2021-10-19T07:04:55+0100","Rcpt":"johndoejr@example.com","AttSize":0,"Attempt":1,"Snt":51666,"aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","UseTls":"No", "Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""},{"acc":"ABC123","Delivered":false,"IP":"8.8.8.8","RejType":"Recipient email address is possibly incorrect","RejCode":"550","AttCnt":0,"Dir":"Internal","ReceiptAck":null,"MsgId":"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>","Subject":"You have new held messages","Latency":1534,"Sender":"johndoe@example.com","datetime":"2021-10-19T07:04:56+0100","Rcpt":"johndoejr@example.com","AttSize":0,"Attempt":1,"RejInfo":"5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]","TlsVer":"TLSv1.2","Cphr":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","Snt":147,"aCode":"61dfe7da-4c6d-34e1-9667-69b04f0d564f","UseTls":"Yes","Route":"Office365","Content-Disposition":"attachment; filename=\"delivery_20211018093329655.json\""} +{"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:09:18+0000","Rcpt":"o365_service_account@example.com","RcptActType":"Jnl","aCode":"CYSuuaBUMjOpk3k1Xhvy_Q","Dir":"Internal","RcptHdrType":"Unknown", "Content-Disposition":"attachment; filename=\"jrnl_20211018093329655.json\""} +{"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:10:19+0000","Rcpt":"johndoejr@example.com","Act":"Acc","IP":"8.8.8.8","aCode":"3dbe9918-f91f-3043-b61f-d3164badfe50","Dir":"Internal","Subject":"You have new held messages","MsgId":"<140943948-1636373419265@uk-mta-286.uk.mimecast.lan>","headerFrom":"johndoe@example.com", "Content-Disposition":"attachment; filename=\"receipt_20211018093329655.json\""} \ No newline at end of file diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json new file mode 100644 index 00000000000..315913399b6 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json @@ -0,0 +1,242 @@ +{ + "expected": [ + { + "@timestamp": "2021-10-18T08:02:43.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "reason": "Spm", + "action": "Hld", + "ingested": "2021-11-25T11:34:11.459620200Z", + "original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":null,\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Hld\":\"Spm\",\"MsgId\":\"\\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\\u003e\",\"MsgSize\":157436,\"Sender\":\"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu\",\"Subject\":\"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!\",\"aCode\":\"HhuwRf_AOcuJZINE2ZgcKw\",\"acc\":\"ABC123\",\"datetime\":\"2021-10-18T09:02:43+0100\"}", + "created": "2021-10-18T09:02:43+0100", + "outcome": "unknown" + }, + "email": { + "message_id": "\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\u003e", + "from": { + "address": "bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu" + }, + "attachments": { + "file": { + "size": 0 + } + }, + "local_id": "HhuwRf_AOcuJZINE2ZgcKw", + "subject": "Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!", + "message_size": 157436 + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "ABC123", + "log_type": "process", + "AttCnt": 0 + } + }, + { + "rule": { + "name": "Office365" + }, + "source": { + "ip": "8.8.8.8" + }, + "error": { + "type": "Recipient email address is possibly incorrect", + "code": "550" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-10-19T06:06:40.000Z", + "ecs": { + "version": "1.12.0" + }, + "tls": { + "cipher": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "established": true, + "version": "TLSv1.2" + }, + "event": { + "reason": "5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]", + "ingested": "2021-11-25T11:34:11.459623100Z", + "original": "{\"acc\":\"ABC123\",\"Delivered\":false,\"IP\":\"8.8.8.8\",\"RejType\":\"Recipient email address is possibly incorrect\",\"RejCode\":\"550\",\"AttCnt\":0,\"Dir\":\"Inbound\",\"ReceiptAck\":null,\"MsgId\":null,\"Subject\":null,\"Latency\":505,\"Sender\":\"\u003c\u003e\",\"datetime\":\"2021-10-19T07:06:40+0100\",\"Rcpt\":\"johndoe@example.com\",\"AttSize\":0,\"Attempt\":1,\"RejInfo\":\"5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]\",\"TlsVer\":\"TLSv1.2\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"Snt\":125,\"aCode\":\"29be076e-44cd-354d-a7c2-083d4a312371\",\"UseTls\":\"Yes\",\"Route\":\"Office365\",\"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"}", + "created": "2021-10-19T07:06:40+0100", + "outcome": "failure" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "attachments": { + "file": { + "size": 0 + } + }, + "to": { + "address": "johndoe@example.com" + }, + "local_id": "29be076e-44cd-354d-a7c2-083d4a312371", + "direction": "Inbound" + }, + "mimecast": { + "acc": "ABC123", + "Snt": 125, + "log_type": "delivery", + "AttCnt": 0, + "Attempt": 1, + "Latency": 505 + } + }, + { + "@timestamp": "2021-10-19T06:04:55.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "action": "Acc", + "ingested": "2021-11-25T11:34:11.459624200Z", + "original": "{\"acc\":\"ABC123\",\"Sender\":\"postmaster@twotoeight.com\",\"datetime\":\"2021-10-19T07:04:55+0100\",\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Act\":\"Acc\",\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"AttCnt\":0,\"AttNames\":null,\"MsgSize\":49025,\"MsgId\":\"\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e\",\"Subject\":\"You have new held messages\"}", + "created": "2021-10-19T07:04:55+0100", + "outcome": "unknown" + }, + "email": { + "message_id": "\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e", + "from": { + "address": "postmaster@twotoeight.com" + }, + "attachments": { + "file": { + "size": 0 + } + }, + "local_id": "61dfe7da-4c6d-34e1-9667-69b04f0d564f", + "subject": "You have new held messages", + "message_size": 49025 + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "ABC123", + "log_type": "process", + "AttCnt": 0 + } + }, + { + "@timestamp": "2021-10-19T06:04:55.000Z", + "ecs": { + "version": "1.12.0" + }, + "tls": { + "established": false + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "ingested": "2021-11-25T11:34:11.459625200Z", + "original": "{\"acc\":\"ABC123\",\"Delivered\":true,\"IP\":\"8.8.8.8\",\"AttCnt\":0,\"Dir\":\"Internal\",\"ReceiptAck\":\"250 SmtpInternalThread-19194240-1634623495703@uk-mta-151.uk.mimecast.lan Received OK [61dfe7da-4c6d-34e1-9667-69b04f0d564f.uk151]\",\"MsgId\":\"\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e\",\"Subject\":null,\"Latency\":1090,\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-10-19T07:04:55+0100\",\"Rcpt\":\"johndoejr@example.com\",\"AttSize\":0,\"Attempt\":1,\"Snt\":51666,\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"UseTls\":\"No\", \"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"},{\"acc\":\"ABC123\",\"Delivered\":false,\"IP\":\"8.8.8.8\",\"RejType\":\"Recipient email address is possibly incorrect\",\"RejCode\":\"550\",\"AttCnt\":0,\"Dir\":\"Internal\",\"ReceiptAck\":null,\"MsgId\":\"\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e\",\"Subject\":\"You have new held messages\",\"Latency\":1534,\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-10-19T07:04:56+0100\",\"Rcpt\":\"johndoejr@example.com\",\"AttSize\":0,\"Attempt\":1,\"RejInfo\":\"5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]\",\"TlsVer\":\"TLSv1.2\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"Snt\":147,\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"UseTls\":\"Yes\",\"Route\":\"Office365\",\"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"}", + "created": "2021-10-19T07:04:55+0100", + "outcome": "success" + }, + "email": { + "message_id": "\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e", + "from": { + "address": "johndoe@example.com" + }, + "attachments": { + "file": { + "size": 0 + } + }, + "to": { + "address": "johndoejr@example.com" + }, + "local_id": "61dfe7da-4c6d-34e1-9667-69b04f0d564f", + "direction": "Internal" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "ABC123", + "log_type": "delivery", + "Attempt": 1, + "Snt": 51666, + "AttCnt": 0, + "ReceiptAck": "250 SmtpInternalThread-19194240-1634623495703@uk-mta-151.uk.mimecast.lan Received OK [61dfe7da-4c6d-34e1-9667-69b04f0d564f.uk151]", + "Latency": 1090 + } + }, + { + "@timestamp": "2021-11-08T12:09:18.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-11-25T11:34:11.459630600Z", + "original": "{\"acc\":\"C46A75\",\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-11-08T12:09:18+0000\",\"Rcpt\":\"o365_service_account@example.com\",\"RcptActType\":\"Jnl\",\"aCode\":\"CYSuuaBUMjOpk3k1Xhvy_Q\",\"Dir\":\"Internal\",\"RcptHdrType\":\"Unknown\", \"Content-Disposition\":\"attachment; filename=\\\"jrnl_20211018093329655.json\\\"\"}", + "created": "2021-11-08T12:09:18+0000", + "outcome": "unknown" + }, + "email": { + "from": { + "address": "johndoe@example.com" + }, + "to": { + "address": "o365_service_account@example.com" + }, + "local_id": "CYSuuaBUMjOpk3k1Xhvy_Q", + "direction": "Internal" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "C46A75", + "log_type": "jrnl", + "RcptActType": "Jnl", + "RcptHdrType": "Unknown" + } + }, + { + "@timestamp": "2021-11-08T12:10:19.000Z", + "ecs": { + "version": "1.12.0" + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "action": "Acc", + "ingested": "2021-11-25T11:34:11.459631700Z", + "original": "{\"acc\":\"C46A75\",\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-11-08T12:10:19+0000\",\"Rcpt\":\"johndoejr@example.com\",\"Act\":\"Acc\",\"IP\":\"8.8.8.8\",\"aCode\":\"3dbe9918-f91f-3043-b61f-d3164badfe50\",\"Dir\":\"Internal\",\"Subject\":\"You have new held messages\",\"MsgId\":\"\u003c140943948-1636373419265@uk-mta-286.uk.mimecast.lan\u003e\",\"headerFrom\":\"johndoe@example.com\", \"Content-Disposition\":\"attachment; filename=\\\"receipt_20211018093329655.json\\\"\"}", + "created": "2021-11-08T12:10:19+0000", + "outcome": "unknown" + }, + "email": { + "header_from": "johndoe@example.com", + "local_id": "3dbe9918-f91f-3043-b61f-d3164badfe50", + "subject": "You have new held messages", + "message_id": "\u003c140943948-1636373419265@uk-mta-286.uk.mimecast.lan\u003e", + "from": { + "address": "johndoe@example.com" + }, + "to": { + "address": "johndoejr@example.com" + }, + "direction": "Internal" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "C46A75", + "log_type": "receipt" + } + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/siem_logs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..a41bccfe834 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/audit/get-siem-logs diff --git a/packages/mimecast/data_stream/siem_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/siem_logs/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..f9f6fe91f38 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/agent/stream/httpjson.yml.hbs @@ -0,0 +1,55 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: + - set: + target: body.data + value: '[{"type":"MTA","fileFormat":"json", "compress":false, "token": "[[.cursor.next_token]]"}]' + default: '[{"type":"MTA","fileFormat":"json", "compress":false}]' + value_type: json + - set: + target: header.x-mc-app-id + value: {{app_id}} + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: "[[uuid]]" + - set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/audit/get-siem-logs:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true + - set: + target: header.Accept + value: '*/*' +response.decode_as: application/json +response.split: + transforms: + - set: + target: body.Content-Disposition + value: '[[.last_response.header.Get "Content-Disposition"]]' + target: body.data +response.pagination: + - set: + target: body.data + value: '[{"type":"MTA","fileFormat":"json", "compress":false, "token": "[[.last_response.header.Get "mc-siem-token"]]"}]' + value_type: json +cursor: + next_token: + value: '[[.last_response.header.Get "mc-siem-token"]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..82e91c807cf --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,334 @@ +--- +description: Pipeline for processing sample logs +processors: + # Generic event/ecs fields we always want to populated + - set: + field: event.ingested + value: "{{ _ingest.timestamp }}" + - set: + field: ecs.version + value: "1.12.0" + - rename: + field: message + target_field: event.original + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.datetime == null + - date: + description: Use 'mimecast.datetime' as the '@timestamp' + field: mimecast.datetime + timezone: UTC + formats: + - yyyy-MM-dd'T'HH:mm:ssZ + + ### RECEIPT LOGS + - rename: + field: mimecast.aCode + target_field: email.local_id + ignore_missing: true + if: 'ctx?.mimecast?.aCode !=null' + - rename: + field: mimecast.Act + target_field: event.action + ignore_missing: true + if: 'ctx?.mimecast?.Act !=null' + - rename: + field: mimecast.Cphr + target_field: tls.cipher + ignore_missing: true + if: 'ctx?.mimecast?.Cphr !=null' + - rename: + field: mimecast.Dir + target_field: email.direction + ignore_missing: true + if: 'ctx?.mimecast?.Dir !=null' + - rename: + field: mimecast.Error + target_field: error.message + ignore_missing: true + if: 'ctx?.mimecast?.Error !=null' + - rename: + field: mimecast.IP + target_field: source.ip + ignore_missing: true + if: 'ctx?.mimecast?.IP !=null' + - rename: + field: mimecast.MsgId + target_field: email.message_id + ignore_missing: true + if: 'ctx?.mimecast?.MsgId !=null' + - rename: + field: mimecast.Rcpt + target_field: email.to.address + ignore_missing: true + if: 'ctx?.mimecast?.Rcpt !=null' + - rename: + field: mimecast.headerFrom + target_field: email.header_from + ignore_missing: true + if: 'ctx?.mimecast?.headerFrom !=null' + - rename: + field: mimecast.RejCode + target_field: error.code + ignore_missing: true + if: 'ctx?.mimecast?.RejCode !=null' + - rename: + field: mimecast.RejInfo + target_field: event.reason + ignore_missing: true + if: 'ctx?.mimecast?.RejInfo !=null' + - rename: + field: mimecast.RejType + target_field: error.type + ignore_missing: true + if: 'ctx?.mimecast?.RejType !=null' + - rename: + field: mimecast.Sender + target_field: email.from.address + ignore_missing: true + if: 'ctx?.mimecast?.Sender !=null' + - rename: + field: mimecast.Subject + target_field: email.subject + ignore_missing: true + if: 'ctx?.mimecast?.Subject !=null' + - rename: + field: mimecast.TlsVer + target_field: tls.version + ignore_missing: true + if: 'ctx?.mimecast?.TlsVer !=null' + ### PROCESS LOGS + - rename: + field: mimecast.AttSize + target_field: email.attachments.file.size + ignore_missing: true + if: 'ctx?.mimecast?.AttSize !=null' + - rename: + field: mimecast.AttNames + target_field: email.attachments.file.name + ignore_missing: true + if: 'ctx?.mimecast?.AttNames !=null' + - rename: + field: mimecast.Hld + target_field: event.reason + ignore_missing: true + if: 'ctx?.mimecast?.Hld !=null' + - rename: + field: mimecast.MsgSize + target_field: email.message_size + ignore_missing: true + if: 'ctx?.mimecast?.MsgSize !=null' + ### DELIVERY LOGS + - rename: + field: mimecast.Err + target_field: error.message + ignore_missing: true + if: 'ctx?.mimecast?.Err !=null' + - rename: + field: mimecast.Route + target_field: rule.name + ignore_missing: true + if: 'ctx?.mimecast?.Route !=null' + - rename: + field: mimecast.UseTls + target_field: tls.established + ignore_missing: true + if: 'ctx?.mimecast?.UseTls !=null' + ### AV LOGS + - rename: + field: mimecast.fileExt + target_field: email.attachments.file.extension + ignore_missing: true + if: 'ctx?.mimecast?.fileExt !=null' + - rename: + field: mimecast.fileMime + target_field: email.attachments.file.mime_type + ignore_missing: true + if: 'ctx?.mimecast?.fileMime !=null' + - rename: + field: mimecast.md5 + target_field: email.attachments.hash.md5 + ignore_missing: true + if: 'ctx?.mimecast?.md5 !=null' + - rename: + field: mimecast.Recipient + target_field: email.to.address + ignore_missing: true + if: 'ctx?.mimecast?.Recipient !=null' + - rename: + field: mimecast.SenderDomain + target_field: source.domain + ignore_missing: true + if: 'ctx?.mimecast?.SenderDomain !=null' + - rename: + field: mimecast.sha1 + target_field: email.attachments.hash.sha1 + ignore_missing: true + if: 'ctx?.mimecast?.sha1 !=null' + - rename: + field: mimecast.sha256 + target_field: email.attachments.hash.sha256 + ignore_missing: true + if: 'ctx?.mimecast?.sha256 !=null' + - rename: + field: mimecast.Size + target_field: email.attachments.file.size + ignore_missing: true + if: 'ctx?.mimecast?.Size !=null' + - rename: + field: mimecast.fileName + target_field: email.attachments.file.name + ignore_missing: true + if: 'ctx?.mimecast?.fileName !=null' + ### SPAM EVENT THREAD LOGS + - rename: + field: mimecast.SourceIP + target_field: source.ip + ignore_missing: true + if: 'ctx?.mimecast?.SourceIP !=null' + ### SIEM Email Protect Logs + - rename: + field: mimecast.URL + target_field: url.full + ignore_missing: true + if: 'ctx?.mimecast?.URL !=null' + ### SIEM Impersonation logs + - rename: + field: mimecast.Action + target_field: event.action + ignore_missing: true + if: 'ctx?.mimecast?.Action !=null' + - rename: + field: mimecast.Definition + target_field: rule.name + ignore_missing: true + if: 'ctx?.mimecast?.Definition !=null' + - rename: + field: mimecast.NewDomain + target_field: source.domain + ignore_missing: true + if: 'ctx?.mimecast?.NewDomain !=null' + ### SIEM TTP Url Logs + - rename: + field: mimecast.reason + target_field: event.reason + ignore_missing: true + if: 'ctx?.mimecast?.reason !=null' + - rename: + field: mimecast.recipient + target_field: email.to.address + ignore_missing: true + if: 'ctx?.mimecast?.recipient !=null' + - rename: + field: mimecast.route + target_field: email.direction + ignore_missing: true + if: 'ctx?.mimecast?.route !=null' + - rename: + field: mimecast.sender + target_field: email.from.address + ignore_missing: true + if: 'ctx?.mimecast?.sender !=null' + - rename: + field: mimecast.senderDomain + target_field: source.domain + ignore_missing: true + if: 'ctx?.mimecast?.senderDomain !=null' + - rename: + field: mimecast.sourceIp + target_field: source.ip + ignore_missing: true + if: 'ctx?.mimecast?.sourceIp !=null' + - rename: + field: mimecast.subject + target_field: email.subject + ignore_missing: true + if: 'ctx?.mimecast?.subject !=null' + - rename: + field: mimecast.url + target_field: url.full + ignore_missing: true + if: 'ctx?.mimecast?.url !=null' + - rename: + field: mimecast.action + target_field: event.action + ignore_missing: true + if: 'ctx?.mimecast?.action !=null' + - dissect: + field: mimecast.Content-Disposition + pattern: "%{?drop->}=\"%{mimecast.log_type}_%{?drop->}" + ignore_missing: true + - set: + field: event.created + value: "{{mimecast.datetime}}" + if: 'ctx?.mimecast?.datetime != null' + - set: + field: tls.established + value: false + if: 'ctx?.tls?.established == "No"' + - set: + field: tls.established + value: true + if: 'ctx?.tls?.established == "Yes"' + - rename: + field: mimecast.Delivered + target_field: event.outcome + ignore_missing: true + if: 'ctx?.mimecast?.Delivered !=null' + - set: + field: event.outcome + value: "success" + if: 'ctx?.event?.outcome ==true' + - set: + field: event.outcome + value: "failure" + if: 'ctx?.event?.outcome ==false' + - set: + field: event.outcome + value: "unknown" + if: 'ctx?.event?.outcome ==null' + # Cleanup + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast.eventTime + - mimecast.Content-Disposition + - mimecast.datetime + ignore_missing: true + - remove: + description: Remove 'mimecast.RecieptApk' if null + field: mimecast.ReceiptAck + if: 'ctx?.mimecast?.ReceiptAck == null' + ignore_missing: true + - remove: + description: Remove 'mimecast.AttNames' if null + field: mimecast.AttNames + if: 'ctx?.mimecast?.AttNames == null' + ignore_missing: true + - remove: + description: Remove 'mimecast.MsgId' if null + field: mimecast.MsgId + if: 'ctx?.mimecast?.MsgId == null' + ignore_missing: true + - remove: + description: Remove 'mimecast.Subject' if null + field: mimecast.Subject + if: 'ctx?.mimecast?.Subject == null' + ignore_missing: true + - remove: + description: Remove 'event.original' if 'preserve_original_event' is not set. + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + + + ### + +# Error handling +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/mimecast/data_stream/siem_logs/fields/agent.yml b/packages/mimecast/data_stream/siem_logs/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/siem_logs/fields/base-fields.yml b/packages/mimecast/data_stream/siem_logs/fields/base-fields.yml new file mode 100644 index 00000000000..aa70102a062 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.siem_logs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml new file mode 100644 index 00000000000..9dd7efeec0d --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml @@ -0,0 +1,97 @@ +- external: ecs + name: event.original +- external: ecs + name: event.action +- external: ecs + name: user.email +- external: ecs + name: event.id +- external: ecs + name: tags +- external: ecs + name: ecs.version +- description: Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the X-MS-Exchange-Organization-Network-Message-Id id). + type: keyword + name: email.local_id +- external: ecs + name: event.action +- external: ecs + name: tls.cipher +- description: Direction of the message based on the sending and receiving domains. + type: keyword + name: email.direction +- external: ecs + name: error.message +- external: ecs + name: source.ip +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. + type: wildcard + name: email.message_id + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: The email address(es) of the message recipient(s). + type: keyword + name: email.to.address +- description: The sender address found in the from header of the email. + type: keyword + name: email.header_from +- external: ecs + name: error.code +- external: ecs + name: event.reason +- external: ecs + name: error.type +- description: Stores the from email address from the RFC5322 From - header field. + type: keyword + name: email.from.address +- description: A brief summary of the topic of the message + type: keyword + name: email.subject + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false +- external: ecs + name: tls.version +- description: Attachment file size in bytes. + type: long + name: email.attachments.file.size +- description: Name of the attachment file including the extension. + type: keyword + name: email.attachments.file.name +- description: The total size of the email.The total size of the email. + type: long + name: email.message_size +- external: ecs + name: tls.established +- external: ecs + name: rule.name +- description: Attachment file extension, excluding the leading dot. + type: keyword + name: email.attachments.file.extension +- description: MIME type of the attachment file. + type: keyword + name: email.attachments.file.mime_type +- external: ecs + name: source.domain +- description: SHA-1 hash of the file attachment. + type: keyword + name: email.attachments.hash.sha1 +- description: SHA-256 hash of the file attachment. + type: keyword + name: email.attachments.hash.sha256 +- description: MD5 hash of the file attachment. + type: keyword + name: email.attachments.hash.md5 +- description: Name of the attachment file including the extension. + type: keyword + name: email.attachments.file.name +- external: ecs + name: url.full +- external: ecs + name: event.outcome diff --git a/packages/mimecast/data_stream/siem_logs/fields/field.yml b/packages/mimecast/data_stream/siem_logs/fields/field.yml new file mode 100644 index 00000000000..24eae8c46c1 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/fields/field.yml @@ -0,0 +1,136 @@ +- name: mimecast + type: group + fields: + ### Receipt logs + - name: acc + type: keyword + description: The Mimecast account code for your account. + - name: SpamInfo + type: keyword + description: Information from Mimecast Spam scanners for messages found to be Spam. + - name: SpamLimit + type: long + description: The Spam limit defined for the given sender and recipient. + - name: SpamProcessingDetail + type: keyword + description: The Spam processing details for DKIM, SPF, DMARC. + - name: SpamScore + type: long + description: The Spam score the email was given. + - name: Virus + type: keyword + description: The name of the virus found on the email, if applicable. + - name: MsgId + type: keyword + description: The internet message id of the email. + - name: Subject + type: keyword + description: The subject of the email, limited to 150 characters. + #### Process logs + - name: AttCnt + type: long + description: The number of attachments on the email. + - name: IPInternalName + type: keyword + description: For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detected to be from an internal user name. + - name: IPNewDomain + type: keyword + description: For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detected to be from a new domain. + - name: IPReplyMismatch + type: keyword + description: For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detetced to have a mismatch in the reply to address. + - name: IPSimilarDomain + type: keyword + description: For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detetced to be from a similar domain to any domain you have registered as an Internal Domain. + - name: IPThreadDict + type: keyword + description: For emails subject to Targeted Threat Protection - Impersonation Protect, if the content of the email was detected to contain words in the Mimecast threat dictionary. + - name: MsgSize + type: long + description: The total size of the email. + - name: AttNames + type: keyword + description: The filenames of all attachments on the email. + ### Delivery logs + - name: Attempt + type: long + description: The count of attempts that the Mimecast MTA has made to deliver the email. + - name: Latency + type: long + description: The time in milliseconds that the delivery attempt took. + - name: ReceiptAck + type: keyword + description: The receipt acknowledgment message received by Mimecast from the receiving mail server. + - name: Snt + type: long + description: The amount of data in bytes that were delivered. + ### AV Logs + - name: CustomerIP + type: keyword + description: The source IP is one of the accounts authorised IPs or one of the authorised IPs belonging to an Umbrella Account, if the Account uses an Umbrella Account. + - name: MimecastIP + type: keyword + description: The source IP is one of the Mimecast' IPs e.g. Mimecast Personal Portal. + - name: SenderDomainInternal + type: keyword + description: The sender domain is a registered internal domain. + ### Spam Event Thread Logs + - name: ScanResultInfo + ### Siem Email Protect Logs + + type: keyword + description: The reason that the click was blocked. + - name: UrlCategory + type: keyword + description: The category of the URL that was clicked. + ### Siem Impersonation Logs + - name: CustomName + type: keyword + description: The message has matched a custom name. + - name: CustomThreatDictionary + type: keyword + description: The content of the email was detected to contain words in a custom threat dictionary. + - name: Hits + type: keyword + description: Number of items flagged for the message. + - name: InternalName + type: keyword + description: The email was detected to be from an internal user name. + - name: ReplyMismatch + type: keyword + description: The reply address does not correspond to the senders address. + - name: SimilarCustomExternalDomain + type: keyword + description: The senders domain is similar to a custom external domain list. + - name: SimilarInternalDomain + type: keyword + description: The senders domain is similar to a registered internal domain. + - name: SimilarMimecastExternalDomain + type: keyword + description: The senders domain is similar to a Mimecast managed list of domains. + - name: TaggedExternal + type: keyword + description: The message has been tagged as originating from a external source. + - name: TaggedMalicious + type: keyword + description: The message has been tagged as malicious. + - name: ThreatDictionary + type: keyword + description: The content of the email was detected to contain words in the Mimecast threat dictionary. + ### SIEM TTP Url Protect Logs + - name: urlCategory + type: keyword + description: The category of the URL that was clicked. + - name: credentialTheft + type: keyword + description: The info about credential theft. + ### SIEM Journal Logs + - name: RcptHdrType + type: keyword + description: Type of the receipt header. + - name: RcptActType + type: keyword + description: Action after reception. + - name: log_type + type: keyword + description: String to get type of SIEM log. diff --git a/packages/mimecast/data_stream/siem_logs/manifest.yml b/packages/mimecast/data_stream/siem_logs/manifest.yml new file mode 100644 index 00000000000..652c5f69437 --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/manifest.yml @@ -0,0 +1,77 @@ +title: "SIEM Mimecast Logs" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: SIEM logs + description: Collect SIEM Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 0.5s + - name: api_url + type: password + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/audit/get-siem-logs + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-siem-logs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/siem_logs/sample_event.json b/packages/mimecast/data_stream/siem_logs/sample_event.json new file mode 100644 index 00000000000..01ef03c371e --- /dev/null +++ b/packages/mimecast/data_stream/siem_logs/sample_event.json @@ -0,0 +1,36 @@ +{ + "@timestamp": "2021-10-18T08:02:43.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "reason": "Spm", + "action": "Hld", + "ingested": "2021-11-25T11:34:11.459620200Z", + "original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":null,\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Hld\":\"Spm\",\"MsgId\":\"\\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\\u003e\",\"MsgSize\":157436,\"Sender\":\"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu\",\"Subject\":\"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!\",\"aCode\":\"HhuwRf_AOcuJZINE2ZgcKw\",\"acc\":\"ABC123\",\"datetime\":\"2021-10-18T09:02:43+0100\"}", + "created": "2021-10-18T09:02:43+0100", + "outcome": "unknown" + }, + "email": { + "message_id": "\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\u003e", + "from": { + "address": "bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu" + }, + "attachments": { + "file": { + "size": 0 + } + }, + "local_id": "HhuwRf_AOcuJZINE2ZgcKw", + "subject": "Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!", + "message_size": 157436 + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "ABC123", + "log_type": "process", + "AttCnt": 0 + } +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-threat-intel-malware-customer.log b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-threat-intel-malware-customer.log new file mode 100644 index 00000000000..d7f3fe0cd18 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-threat-intel-malware-customer.log @@ -0,0 +1,21 @@ +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--656138d6-faef-4a9d-907a-d6932bc459cb", "created": "2021-10-29T15:07:26.653Z", "modified": "2021-10-29T15:07:26.653Z", "name": "Business Proposal_Final", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", "created": "2021-10-29T15:07:26.653Z", "modified": "2021-10-29T15:07:26.653Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", "valid_from": "2021-10-29T15:07:26.653Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--4505a917-12f9-4c24-8729-3efe5aa3b3f6", "created": "2021-10-29T15:07:26.653Z", "modified": "2021-10-29T15:07:26.653Z", "relationship_type": "indicates", "source_ref": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", "target_ref": "malware--656138d6-faef-4a9d-907a-d6932bc459cb" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--261da827-4f7b-4607-a856-8aa34a3cb000", "created": "2021-10-29T15:07:22.595Z", "modified": "2021-10-29T15:07:22.595Z", "name": "Urgent info! - TwoToEight", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3", "created": "2021-10-29T15:07:22.595Z", "modified": "2021-10-29T15:07:22.595Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = '6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb']", "valid_from": "2021-10-29T15:07:22.595Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--14fcbc1d-f20a-418f-b368-2c17ac4b8c1a", "created": "2021-10-29T15:07:22.595Z", "modified": "2021-10-29T15:07:22.595Z", "relationship_type": "indicates", "source_ref": "indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3", "target_ref": "malware--261da827-4f7b-4607-a856-8aa34a3cb000" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--2e6bcc79-7be4-4abb-9b37-01c2c2bfd509", "created": "2021-10-29T15:07:17.538Z", "modified": "2021-10-29T15:07:17.538Z", "name": "RE: Read: Data base forms", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976", "created": "2021-10-29T15:07:17.538Z", "modified": "2021-10-29T15:07:17.538Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = '8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668']", "valid_from": "2021-10-29T15:07:17.538Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--1b489f44-35c2-49a5-b9d5-9320dba53fa5", "created": "2021-10-29T15:07:17.538Z", "modified": "2021-10-29T15:07:17.538Z", "relationship_type": "indicates", "source_ref": "indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976", "target_ref": "malware--2e6bcc79-7be4-4abb-9b37-01c2c2bfd509" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--9bbe2b25-411f-4a98-beb1-fb7440b36d54", "created": "2021-10-29T15:07:14.044Z", "modified": "2021-10-29T15:07:14.044Z", "name": "VM: Caller 908-999-4562", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c", "created": "2021-10-29T15:07:14.044Z", "modified": "2021-10-29T15:07:14.044Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047']", "valid_from": "2021-10-29T15:07:14.044Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--4f282fef-e96d-44d2-ab1d-4244a3a0643f", "created": "2021-10-29T15:07:14.044Z", "modified": "2021-10-29T15:07:14.044Z", "relationship_type": "indicates", "source_ref": "indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c", "target_ref": "malware--9bbe2b25-411f-4a98-beb1-fb7440b36d54" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--10310709-f696-47e7-bb0e-73fc2dcd2c79", "created": "2021-10-29T15:07:07.295Z", "modified": "2021-10-29T15:07:07.295Z", "name": "Application pack - TwoToEight", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505", "created": "2021-10-29T15:07:07.295Z", "modified": "2021-10-29T15:07:07.295Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = '5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283']", "valid_from": "2021-10-29T15:07:07.295Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--ad6f72cf-40ee-4d26-9427-d4cd8149c677", "created": "2021-10-29T15:07:07.295Z", "modified": "2021-10-29T15:07:07.295Z", "relationship_type": "indicates", "source_ref": "indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505", "target_ref": "malware--10310709-f696-47e7-bb0e-73fc2dcd2c79" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--83b7aa17-3fd8-4881-8190-13705790c69d", "created": "2021-10-29T15:07:00.555Z", "modified": "2021-10-29T15:07:00.555Z", "name": "RE: Read: Data base forms", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0", "created": "2021-10-29T15:07:00.555Z", "modified": "2021-10-29T15:07:00.555Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c']", "valid_from": "2021-10-29T15:07:00.555Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--4e4bfde2-12af-4a7c-962b-b66e54cfca1b", "created": "2021-10-29T15:07:00.555Z", "modified": "2021-10-29T15:07:00.555Z", "relationship_type": "indicates", "source_ref": "indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0", "target_ref": "malware--83b7aa17-3fd8-4881-8190-13705790c69d" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "malware", "id": "malware--e64907e6-9129-4ffa-b426-277c4c691898", "created": "2021-10-29T15:07:00.259Z", "modified": "2021-10-29T15:07:00.259Z", "name": "VM: Caller 908-999-4562", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--3816deef-ba8f-40c4-ba11-a862b4322b11", "created": "2021-10-29T15:07:00.259Z", "modified": "2021-10-29T15:07:00.259Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd']", "valid_from": "2021-10-29T15:07:00.259Z" } +{ "Content-Disposition":"attachment; filename=\"malware_customer_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--61b1b486-507d-4631-963f-71a3af1f3253", "created": "2021-10-29T15:07:00.259Z", "modified": "2021-10-29T15:07:00.259Z", "relationship_type": "indicates", "source_ref": "indicator--3816deef-ba8f-40c4-ba11-a862b4322b11", "target_ref": "malware--e64907e6-9129-4ffa-b426-277c4c691898" } \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-threat-intel-malware-customer.log-expected.json b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-threat-intel-malware-customer.log-expected.json new file mode 100644 index 00000000000..eff4f98cf01 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/pipeline/test-threat-intel-malware-customer.log-expected.json @@ -0,0 +1,319 @@ +{ + "expected": [ + null, + { + "@timestamp": "2021-10-29T15:07:26.653Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:26.653Z", + "file": { + "hash": { + "sha256": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + } + }, + "modified_at": "2021-10-29T15:07:26.653Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218116Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--18c62174-0d31-4653-afe6-d104c57b6b2c\", \"created\": \"2021-10-29T15:07:26.653Z\", \"modified\": \"2021-10-29T15:07:26.653Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']\", \"valid_from\": \"2021-10-29T15:07:26.653Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", + "log_type": "malware_customer", + "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:22.595Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:22.595Z", + "file": { + "hash": { + "sha256": "6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb" + } + }, + "modified_at": "2021-10-29T15:07:22.595Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218119200Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3\", \"created\": \"2021-10-29T15:07:22.595Z\", \"modified\": \"2021-10-29T15:07:22.595Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = '6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb']\", \"valid_from\": \"2021-10-29T15:07:22.595Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = '6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb']", + "log_type": "malware_customer", + "id": "indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:17.538Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:17.538Z", + "file": { + "hash": { + "sha256": "8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668" + } + }, + "modified_at": "2021-10-29T15:07:17.538Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218122100Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976\", \"created\": \"2021-10-29T15:07:17.538Z\", \"modified\": \"2021-10-29T15:07:17.538Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = '8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668']\", \"valid_from\": \"2021-10-29T15:07:17.538Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = '8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668']", + "log_type": "malware_customer", + "id": "indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:14.044Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:14.044Z", + "file": { + "hash": { + "sha256": "df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047" + } + }, + "modified_at": "2021-10-29T15:07:14.044Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218124900Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c\", \"created\": \"2021-10-29T15:07:14.044Z\", \"modified\": \"2021-10-29T15:07:14.044Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047']\", \"valid_from\": \"2021-10-29T15:07:14.044Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047']", + "log_type": "malware_customer", + "id": "indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:07.295Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:07.295Z", + "file": { + "hash": { + "sha256": "5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283" + } + }, + "modified_at": "2021-10-29T15:07:07.295Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218128Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505\", \"created\": \"2021-10-29T15:07:07.295Z\", \"modified\": \"2021-10-29T15:07:07.295Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = '5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283']\", \"valid_from\": \"2021-10-29T15:07:07.295Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = '5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283']", + "log_type": "malware_customer", + "id": "indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:00.555Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:00.555Z", + "file": { + "hash": { + "sha256": "bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c" + } + }, + "modified_at": "2021-10-29T15:07:00.555Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218130900Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0\", \"created\": \"2021-10-29T15:07:00.555Z\", \"modified\": \"2021-10-29T15:07:00.555Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c']\", \"valid_from\": \"2021-10-29T15:07:00.555Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c']", + "log_type": "malware_customer", + "id": "indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:00.259Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:00.259Z", + "file": { + "hash": { + "sha256": "e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd" + } + }, + "modified_at": "2021-10-29T15:07:00.259Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:12.218133800Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--3816deef-ba8f-40c4-ba11-a862b4322b11\", \"created\": \"2021-10-29T15:07:00.259Z\", \"modified\": \"2021-10-29T15:07:00.259Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd']\", \"valid_from\": \"2021-10-29T15:07:00.259Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd']", + "log_type": "malware_customer", + "id": "indicator--3816deef-ba8f-40c4-ba11-a862b4322b11", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..dd139574250 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/ttp/threat-intel/get-feed diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..e2dae615a00 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/httpjson.yml.hbs @@ -0,0 +1,52 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: +- set: + target: body.data + value: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "start":"[[formatDate (.cursor.next_date) "2006-01-02T15:04:05+0700"]]"}]' + default: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "start":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json +- set: + target: header.x-mc-app-id + value: {{app_id}} +- set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' +- set: + target: header.x-mc-req-id + value: "[[uuid]]" +- set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/ttp/threat-intel/get-feed:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + transforms: + - set: + target: body.Content-Disposition + value: '[[.last_response.header.Get "Content-Disposition"]]' + target: body.objects +response.pagination: +- set: + target: body.data + value: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"token": "[[.last_response.header.Get "x-mc-threat-feed-next-token"]]","end": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "start":"[[.cursor.next_date]]"}]' + value_type: json +cursor: + next_date: + value: '[[.first_event.created]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..218edaf06a9 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,226 @@ +--- +description: Pipeline for parsing Mimecast - Threat Intel Feed Malware Customer Logs +processors: + #################### + # Event ECS fields # + #################### + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - set: + field: ecs.version + value: "1.12" + - set: + field: event.kind + value: enrichment + - set: + field: event.category + value: threat + - set: + field: event.type + value: indicator + + ###################### + # General ECS fields # + ###################### + - rename: + field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original + target_field: mimecast + - drop: + if: 'ctx.mimecast?.type != "indicator"' + - fingerprint: + fields: + - mimecast.id + target_field: "_id" + ignore_missing: true + + ##################### + # Threat ECS Fields # + ##################### + ## File indicator operations + - date: + field: mimecast.created + formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + if: "ctx.mimecast?.created != null" + - date: + field: mimecast.modified + target_field: threat.indicator.modified_at + formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + if: "ctx.mimecast?.modified != null" + - date: + field: mimecast.valid_from + target_field: threat.indicator.first_seen + formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + if: "ctx.mimecast?.valid_from != null" + - dissect: + field: mimecast.pattern + pattern: "[%{_tmp.threattype}:%{mimecast.pattern_value}.'%{mimecast.pattern_hash_type}' = '%{_tmp.threatvalue}']" + if: 'ctx.mimecast?.pattern != null' + - set: + field: mimecast.value + value: "{{_tmp.threatvalue}}" + if: 'ctx?._tmp?.threatvalue != null' + - set: + field: mimecast.hashtype + value: "{{mimecast.pattern_hash_type}}" + if: 'ctx?.mimecast?.pattern_hash_type != null' + - rename: + field: _tmp.threattype + target_field: threat.indicator.type + ignore_missing: true + ignore_failure: true + - rename: + field: mimecast.value + target_field: threat.indicator.file.hash.sha256 + ignore_missing: true + if: 'ctx?.mimecast?.hashtype == "SHA-256"' + - rename: + field: mimecast.value + target_field: threat.indicator.file.hash.sha1 + ignore_missing: true + if: 'ctx?.mimecast?.hashtype == "SHA-1"' + - rename: + field: mimecast.value + target_field: threat.indicator.file.hash.md5 + ignore_missing: true + if: 'ctx?.mimecast?.hashtype == "MD-5"' + - append: + field: related.hash + value: "{{threat.indicator.file.hash.sha256}}" + allow_duplicates: false + if: 'ctx?.mimecast?.hashtype == "SHA-256"' + - append: + field: related.hash + value: "{{threat.indicator.file.hash.sha1}}" + allow_duplicates: false + if: 'ctx?.mimecast?.hashtype == "SHA-1"' + - append: + field: related.hash + value: "{{threat.indicator.file.hash.md5}}" + allow_duplicates: false + if: 'ctx?.mimecast?.hashtype == "MD-5"' + - set: + field: threat.indicator.type + value: unknown + if: ctx.threat?.indicator?.type == null + - foreach: + field: mimecast.labels + ignore_missing: true + processor: + append: + field: tags + value: "{{_ingest._value}}" + allow_duplicates: false + - grok: + field: mimecast.description + patterns: + - "^%{GREEDYDATA}Source: %{GREEDYDATA:threat.indicator.provider}" + ignore_missing: true + ignore_failure: true + - dissect: + field: mimecast.Content-Disposition + pattern: "%{?drop->}=\"%{mimecast.logtype_part.1}_%{mimecast.logtype_part.2}_%{?drop->}" + ignore_missing: true + - set: + field: mimecast.log_type + value: "{{mimecast.logtype_part.1}}_{{mimecast.logtype_part.2}}" + if: 'ctx?.mimecast?.logtype_part?.1 != null && ctx?.mimecast?.logtype_part?.2 != null' + - rename: + field: mimecast.name + target_field: email.attachments.file.name + ignore_missing: true + if: 'ctx?.mimecast?.name != null' + - split: + field: email.attachments.file.name + separator: "\\." + target_field: file.parts + if: 'ctx?.email?.attachments?.file?.name != null' + - script: + lang: painless + source: | + ctx.email.attachments.file.name = ctx.file.parts[0]; + if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' + - script: + lang: painless + source: | + ctx.email.attachments.file.extension = ctx.file.parts[ctx.file.parts.length-1]; + if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' + ###################### + # Cleanup processors # + ###################### + - script: + lang: painless + if: ctx?.threatintel != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - remove: + field: + - mimecast.created + - message + - _tmp + - mimecast.pattern_value + - mimecast.pattern_hash_type + - mimecast.hashtype + - mimecast.value + - mimecast.Content-Disposition + - mimecast.logtype_part + - mimecast.modified + - mimecast.valid_from + - file.parts + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/base-fields.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/base-fields.yml new file mode 100644 index 00000000000..ea3b377344f --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.threat_intel_malware_customer +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml new file mode 100644 index 00000000000..fcb3df2855e --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml @@ -0,0 +1,22 @@ +- external: ecs + name: message +- external: ecs + name: tags +- external: ecs + name: event.original +- external: ecs + name: ecs.version +- external: ecs + name: threat.indicator.type +- external: ecs + name: threat.indicator.first_seen +- external: ecs + name: threat.indicator.modified_at +- external: ecs + name: threat.indicator.file.hash.sha256 +- external: ecs + name: threat.indicator.file.hash.sha1 +- external: ecs + name: threat.indicator.file.hash.md5 +- external: ecs + name: related.hash diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/field.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/field.yml new file mode 100644 index 00000000000..08f642eceef --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/field.yml @@ -0,0 +1,73 @@ +- name: mimecast + type: group + fields: + - name: id + type: keyword + description: > + The ID of the indicator. + + - name: value + type: keyword + description: > + The value of the indicator. + + - name: modified + type: date + description: > + When the indicator was last modified. + + - name: created + type: date + description: > + When the indicator was last created. + + - name: labels + type: keyword + description: > + The labels related to the indicator. + + - name: type + type: keyword + description: > + The indicator type, can for example be "domain, email, FileHash-SHA256". + + - name: valid_from + type: date + description: > + The valid from date. + + - name: pattern + type: keyword + description: > + The pattern. + + - name: hashtype + type: keyword + description: > + The hash type. + + - name: log_type + type: keyword + description: > + String to get type of Threat intel feed. + + - name: name + type: keyword + description: > + Name of the file. + + - name: relationship_type + type: keyword + description: > + Type of the relationship. + + - name: source_ref + type: keyword + description: > + Source of the reference. + + - name: target_ref + type: keyword + description: > + Reference target. + diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/manifest.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/manifest.yml new file mode 100644 index 00000000000..e172edd9860 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/manifest.yml @@ -0,0 +1,77 @@ +title: "Threat Intel Feed - Malware Customer" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: Threat Intel Feed - Malware Customer Logs + description: Collect Threat Intel Feed - Malware Customer Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: password + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/ttp/threat-intel/get-feed + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-threat-intel-feed-malware-customer + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/sample_event.json b/packages/mimecast/data_stream/threat_intel_malware_customer/sample_event.json new file mode 100644 index 00000000000..7627d4d8cde --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/sample_event.json @@ -0,0 +1,41 @@ +{ + "@timestamp": "2021-10-29T15:07:26.653Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:26.653Z", + "file": { + "hash": { + "sha256": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + } + }, + "modified_at": "2021-10-29T15:07:26.653Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-17T13:42:34.324885300Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--18c62174-0d31-4653-afe6-d104c57b6b2c\", \"created\": \"2021-10-29T15:07:26.653Z\", \"modified\": \"2021-10-29T15:07:26.653Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']\", \"valid_from\": \"2021-10-29T15:07:26.653Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", + "log_type": "malware_customer", + "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-threat-intel-malware-grid.log b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-threat-intel-malware-grid.log new file mode 100644 index 00000000000..d8a993a4abe --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-threat-intel-malware-grid.log @@ -0,0 +1,21 @@ +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--656138d6-faef-4a9d-907a-d6932bc459cb", "created": "2021-10-29T15:07:26.653Z", "modified": "2021-10-29T15:07:26.653Z", "name": "Business Proposal_Final.pdf", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", "created": "2021-10-29T15:07:26.653Z", "modified": "2021-10-29T15:07:26.653Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", "valid_from": "2021-10-29T15:07:26.653Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--4505a917-12f9-4c24-8729-3efe5aa3b3f6", "created": "2021-10-29T15:07:26.653Z", "modified": "2021-10-29T15:07:26.653Z", "relationship_type": "indicates", "source_ref": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", "target_ref": "malware--656138d6-faef-4a9d-907a-d6932bc459cb" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--261da827-4f7b-4607-a856-8aa34a3cb000", "created": "2021-10-29T15:07:22.595Z", "modified": "2021-10-29T15:07:22.595Z", "name": "Urgent info! - TwoToEight.docx", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3", "created": "2021-10-29T15:07:22.595Z", "modified": "2021-10-29T15:07:22.595Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = '6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb']", "valid_from": "2021-10-29T15:07:22.595Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--14fcbc1d-f20a-418f-b368-2c17ac4b8c1a", "created": "2021-10-29T15:07:22.595Z", "modified": "2021-10-29T15:07:22.595Z", "relationship_type": "indicates", "source_ref": "indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3", "target_ref": "malware--261da827-4f7b-4607-a856-8aa34a3cb000" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--2e6bcc79-7be4-4abb-9b37-01c2c2bfd509", "created": "2021-10-29T15:07:17.538Z", "modified": "2021-10-29T15:07:17.538Z", "name": "RE: Read: Data base forms.ip.zip", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976", "created": "2021-10-29T15:07:17.538Z", "modified": "2021-10-29T15:07:17.538Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = '8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668']", "valid_from": "2021-10-29T15:07:17.538Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--1b489f44-35c2-49a5-b9d5-9320dba53fa5", "created": "2021-10-29T15:07:17.538Z", "modified": "2021-10-29T15:07:17.538Z", "relationship_type": "indicates", "source_ref": "indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976", "target_ref": "malware--2e6bcc79-7be4-4abb-9b37-01c2c2bfd509" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--9bbe2b25-411f-4a98-beb1-fb7440b36d54", "created": "2021-10-29T15:07:14.044Z", "modified": "2021-10-29T15:07:14.044Z", "name": "VM: Caller 908-999-4562", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c", "created": "2021-10-29T15:07:14.044Z", "modified": "2021-10-29T15:07:14.044Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047']", "valid_from": "2021-10-29T15:07:14.044Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--4f282fef-e96d-44d2-ab1d-4244a3a0643f", "created": "2021-10-29T15:07:14.044Z", "modified": "2021-10-29T15:07:14.044Z", "relationship_type": "indicates", "source_ref": "indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c", "target_ref": "malware--9bbe2b25-411f-4a98-beb1-fb7440b36d54" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--10310709-f696-47e7-bb0e-73fc2dcd2c79", "created": "2021-10-29T15:07:07.295Z", "modified": "2021-10-29T15:07:07.295Z", "name": "Application pack - TwoToEight.zip", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505", "created": "2021-10-29T15:07:07.295Z", "modified": "2021-10-29T15:07:07.295Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = '5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283']", "valid_from": "2021-10-29T15:07:07.295Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--ad6f72cf-40ee-4d26-9427-d4cd8149c677", "created": "2021-10-29T15:07:07.295Z", "modified": "2021-10-29T15:07:07.295Z", "relationship_type": "indicates", "source_ref": "indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505", "target_ref": "malware--10310709-f696-47e7-bb0e-73fc2dcd2c79" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--83b7aa17-3fd8-4881-8190-13705790c69d", "created": "2021-10-29T15:07:00.555Z", "modified": "2021-10-29T15:07:00.555Z", "name": "RE: Read: Data base forms", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0", "created": "2021-10-29T15:07:00.555Z", "modified": "2021-10-29T15:07:00.555Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c']", "valid_from": "2021-10-29T15:07:00.555Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--4e4bfde2-12af-4a7c-962b-b66e54cfca1b", "created": "2021-10-29T15:07:00.555Z", "modified": "2021-10-29T15:07:00.555Z", "relationship_type": "indicates", "source_ref": "indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0", "target_ref": "malware--83b7aa17-3fd8-4881-8190-13705790c69d" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "malware", "id": "malware--e64907e6-9129-4ffa-b426-277c4c691898", "created": "2021-10-29T15:07:00.259Z", "modified": "2021-10-29T15:07:00.259Z", "name": "VM: Caller 908-999-4562", "labels": [ "virus" ] } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "indicator", "id": "indicator--3816deef-ba8f-40c4-ba11-a862b4322b11", "created": "2021-10-29T15:07:00.259Z", "modified": "2021-10-29T15:07:00.259Z", "labels": [ "malicious-activity" ], "pattern": "[file:hashes.'SHA-256' = 'e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd']", "valid_from": "2021-10-29T15:07:00.259Z" } +{ "Content-Disposition":"attachment; filename=\"malware_grid_stix_20211028161801144.stix\"","type": "relationship", "id": "relationship--61b1b486-507d-4631-963f-71a3af1f3253", "created": "2021-10-29T15:07:00.259Z", "modified": "2021-10-29T15:07:00.259Z", "relationship_type": "indicates", "source_ref": "indicator--3816deef-ba8f-40c4-ba11-a862b4322b11", "target_ref": "malware--e64907e6-9129-4ffa-b426-277c4c691898" } \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-threat-intel-malware-grid.log-expected.json b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-threat-intel-malware-grid.log-expected.json new file mode 100644 index 00000000000..4197cdfe2b7 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/pipeline/test-threat-intel-malware-grid.log-expected.json @@ -0,0 +1,319 @@ +{ + "expected": [ + null, + { + "@timestamp": "2021-10-29T15:07:26.653Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:26.653Z", + "file": { + "hash": { + "sha256": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + } + }, + "modified_at": "2021-10-29T15:07:26.653Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427883400Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--18c62174-0d31-4653-afe6-d104c57b6b2c\", \"created\": \"2021-10-29T15:07:26.653Z\", \"modified\": \"2021-10-29T15:07:26.653Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']\", \"valid_from\": \"2021-10-29T15:07:26.653Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", + "log_type": "malware_grid", + "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:22.595Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:22.595Z", + "file": { + "hash": { + "sha256": "6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb" + } + }, + "modified_at": "2021-10-29T15:07:22.595Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427886400Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3\", \"created\": \"2021-10-29T15:07:22.595Z\", \"modified\": \"2021-10-29T15:07:22.595Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = '6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb']\", \"valid_from\": \"2021-10-29T15:07:22.595Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = '6a6cd489550ddc08871e14dec73f782bf2405378e9f4adeaa61f1574bea4dbbb']", + "log_type": "malware_grid", + "id": "indicator--d70d0fc0-7fbe-4acc-9830-230a97ecdab3", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:17.538Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:17.538Z", + "file": { + "hash": { + "sha256": "8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668" + } + }, + "modified_at": "2021-10-29T15:07:17.538Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427889200Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976\", \"created\": \"2021-10-29T15:07:17.538Z\", \"modified\": \"2021-10-29T15:07:17.538Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = '8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668']\", \"valid_from\": \"2021-10-29T15:07:17.538Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = '8042d56337b7e7be79688ca861b3c4ba928f95b0824f598ca79e63882dea0668']", + "log_type": "malware_grid", + "id": "indicator--571f0b0a-7206-4a1f-9c5d-9c04e46e0976", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:14.044Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:14.044Z", + "file": { + "hash": { + "sha256": "df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047" + } + }, + "modified_at": "2021-10-29T15:07:14.044Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427892100Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c\", \"created\": \"2021-10-29T15:07:14.044Z\", \"modified\": \"2021-10-29T15:07:14.044Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047']\", \"valid_from\": \"2021-10-29T15:07:14.044Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'df086e1fe742f0992ecd2aec8a3a4b5be6023cca5ef8caf1da3d5b67e9359047']", + "log_type": "malware_grid", + "id": "indicator--90b29bf9-ea1a-423b-8542-4c4590f4038c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:07.295Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:07.295Z", + "file": { + "hash": { + "sha256": "5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283" + } + }, + "modified_at": "2021-10-29T15:07:07.295Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427895300Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505\", \"created\": \"2021-10-29T15:07:07.295Z\", \"modified\": \"2021-10-29T15:07:07.295Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = '5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283']\", \"valid_from\": \"2021-10-29T15:07:07.295Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = '5dbdcba2a373949359459e0e94954896bc06565d745dd36ee2b013dee1dcc283']", + "log_type": "malware_grid", + "id": "indicator--a84c0ac6-f99e-4d1e-b552-74e9023d1505", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:00.555Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:00.555Z", + "file": { + "hash": { + "sha256": "bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c" + } + }, + "modified_at": "2021-10-29T15:07:00.555Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427898300Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0\", \"created\": \"2021-10-29T15:07:00.555Z\", \"modified\": \"2021-10-29T15:07:00.555Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c']\", \"valid_from\": \"2021-10-29T15:07:00.555Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'bcb910f6ab3144c97ca15845741c94479a8925f545ddf59e74253f50d862d10c']", + "log_type": "malware_grid", + "id": "indicator--1fe76455-3ec3-4319-a34c-e4e8e8236ec0", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null, + null, + { + "@timestamp": "2021-10-29T15:07:00.259Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": [ + "e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd" + ] + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:00.259Z", + "file": { + "hash": { + "sha256": "e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd" + } + }, + "modified_at": "2021-10-29T15:07:00.259Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-25T11:34:13.427901100Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--3816deef-ba8f-40c4-ba11-a862b4322b11\", \"created\": \"2021-10-29T15:07:00.259Z\", \"modified\": \"2021-10-29T15:07:00.259Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd']\", \"valid_from\": \"2021-10-29T15:07:00.259Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'e87c5de5f07b36806521334cd25e756b66aa8376d2d52faf269b1878c62cf3dd']", + "log_type": "malware_grid", + "id": "indicator--3816deef-ba8f-40c4-ba11-a862b4322b11", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } + }, + null + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..dd139574250 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/ttp/threat-intel/get-feed diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..b0107c1a95d --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/httpjson.yml.hbs @@ -0,0 +1,52 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: +- set: + target: body.data + value: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "start":"[[formatDate (.cursor.next_date) "2006-01-02T15:04:05+0700"]]"}]' + default: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "start":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json +- set: + target: header.x-mc-app-id + value: {{app_id}} +- set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' +- set: + target: header.x-mc-req-id + value: "[[uuid]]" +- set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/ttp/threat-intel/get-feed:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + transforms: + - set: + target: body.Content-Disposition + value: '[[.last_response.header.Get "Content-Disposition"]]' + target: body.objects +response.pagination: +- set: + target: body.data + value: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"token": "[[.last_response.header.Get "x-mc-threat-feed-next-token"]]","end": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "start":"[[.cursor.next_date]]"}]' + value_type: json +cursor: + next_date: + value: '[[.first_event.created]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..fb7097bb438 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,225 @@ +--- +description: Pipeline for parsing Mimecast - Threat Intel Feed Malware Grid Logs +processors: + #################### + # Event ECS fields # + #################### + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - set: + field: ecs.version + value: "1.12" + - set: + field: event.kind + value: enrichment + - set: + field: event.category + value: threat + - set: + field: event.type + value: indicator + + ###################### + # General ECS fields # + ###################### + - rename: + field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original + target_field: mimecast + - drop: + if: 'ctx.mimecast?.type != "indicator"' + - fingerprint: + fields: + - mimecast.id + target_field: "_id" + ignore_missing: true + + ##################### + # Threat ECS Fields # + ##################### + ## File indicator operations + - date: + field: mimecast.created + formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + if: "ctx.mimecast?.created != null" + - date: + field: mimecast.modified + target_field: threat.indicator.modified_at + formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + if: "ctx.mimecast?.modified != null" + - date: + field: mimecast.valid_from + target_field: threat.indicator.first_seen + formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSZ" + - "yyyy-MM-dd'T'HH:mm:ss.SSSz" + - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + if: "ctx.mimecast?.valid_from != null" + - dissect: + field: mimecast.pattern + pattern: "[%{_tmp.threattype}:%{mimecast.pattern_value}.'%{mimecast.pattern_hash_type}' = '%{_tmp.threatvalue}']" + if: 'ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - set: + field: mimecast.value + value: "{{_tmp.threatvalue}}" + if: 'ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - set: + field: mimecast.hashtype + value: "{{mimecast.pattern_hash_type}}" + if: 'ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - rename: + field: _tmp.threattype + target_field: threat.indicator.type + ignore_missing: true + if: 'ctx?.mimecast?.pattern != null' + - rename: + field: mimecast.value + target_field: threat.indicator.file.hash.sha256 + ignore_missing: true + if: 'ctx?.mimecast?.hashtype == "SHA-256" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - rename: + field: mimecast.value + target_field: threat.indicator.file.hash.sha1 + ignore_missing: true + if: 'ctx?.mimecast?.hashtype == "SHA-1" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - rename: + field: mimecast.value + target_field: threat.indicator.file.hash.md5 + ignore_missing: true + if: 'ctx?.mimecast?.hashtype == "MD-5" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - append: + field: related.hash + value: "{{threat.indicator.file.hash.sha256}}" + allow_duplicates: false + if: 'ctx?.mimecast?.hashtype == "SHA-256" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - append: + field: related.hash + value: "{{threat.indicator.file.hash.sha1}}" + allow_duplicates: false + if: 'ctx?.mimecast?.hashtype == "SHA-1" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - append: + field: related.hash + value: "{{threat.indicator.file.hash.md5}}" + allow_duplicates: false + if: 'ctx?.mimecast?.hashtype == "MD-5" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' + - set: + field: threat.indicator.type + value: unknown + if: 'ctx?.threat?.indicator?.type == null' + - foreach: + field: mimecast.labels + ignore_missing: true + processor: + append: + field: tags + value: "{{_ingest._value}}" + allow_duplicates: false + - grok: + field: mimecast.description + patterns: + - "^%{GREEDYDATA}Source: %{GREEDYDATA:threat.indicator.provider}" + ignore_missing: true + ignore_failure: true + - dissect: + field: mimecast.Content-Disposition + pattern: "%{?drop->}=\"%{mimecast.logtype_part.1}_%{mimecast.logtype_part.2}_%{?drop->}" + ignore_missing: true + - set: + field: mimecast.log_type + value: "{{mimecast.logtype_part.1}}_{{mimecast.logtype_part.2}}" + if: 'ctx?.mimecast?.logtype_part != null' + - rename: + field: mimecast.name + target_field: email.attachments.file.name + ignore_missing: true + - split: + field: email.attachments.file.name + separator: "\\." + target_field: file.parts + if: 'ctx?.email?.attachments?.file?.name != null' + - script: + lang: painless + source: | + ctx.email.attachments.file.name = ctx.file.parts[0]; + if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' + - script: + lang: painless + source: | + ctx.email.attachments.file.extension = ctx.file.parts[ctx.file.parts.length-1]; + if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' + ###################### + # Cleanup processors # + ###################### + - script: + lang: painless + if: ctx?.threatintel != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - remove: + field: + - mimecast.created + - message + - _tmp + - mimecast.pattern_value + - mimecast.pattern_hash_type + - mimecast.hashtype + - mimecast.value + - mimecast.Content-Disposition + - mimecast.logtype_part + - mimecast.modified + - mimecast.valid_from + - file.parts + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" \ No newline at end of file diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/base-fields.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/base-fields.yml new file mode 100644 index 00000000000..f6a6e618f8f --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.threat_intel_malware_grid +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml new file mode 100644 index 00000000000..fcb3df2855e --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml @@ -0,0 +1,22 @@ +- external: ecs + name: message +- external: ecs + name: tags +- external: ecs + name: event.original +- external: ecs + name: ecs.version +- external: ecs + name: threat.indicator.type +- external: ecs + name: threat.indicator.first_seen +- external: ecs + name: threat.indicator.modified_at +- external: ecs + name: threat.indicator.file.hash.sha256 +- external: ecs + name: threat.indicator.file.hash.sha1 +- external: ecs + name: threat.indicator.file.hash.md5 +- external: ecs + name: related.hash diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/field.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/field.yml new file mode 100644 index 00000000000..1bd3638cb5c --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/field.yml @@ -0,0 +1,53 @@ +- name: mimecast + type: group + fields: + - name: id + type: keyword + description: The ID of the indicator. + - name: value + type: keyword + description: The value of the indicator. + - name: modified + type: date + description: When the indicator was last modified. + - name: created + type: date + description: When the indicator was last created. + - name: labels + type: keyword + description: The labels related to the indicator. + - name: type + type: keyword + description: The indicator type, can for example be "domain, email, FileHash-SHA256". + - name: valid_from + type: date + description: The valid from date. + - name: pattern + type: keyword + description: The pattern. + - name: hashtype + type: keyword + description: The hash type. + - name: log_type + type: keyword + description: String to get type of Threat intel feed. + - name: name + type: keyword + description: > + Name of the file. + + - name: relationship_type + type: keyword + description: > + Type of the relationship. + + - name: source_ref + type: keyword + description: > + Source of the reference. + + - name: target_ref + type: keyword + description: > + Reference target. + diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/manifest.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/manifest.yml new file mode 100644 index 00000000000..c14bb673bc1 --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/manifest.yml @@ -0,0 +1,77 @@ +title: "Threat Intel Feed - Malware Grid" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: Threat Intel Feed - Malware Grid Logs + description: Collect Threat Intel Feed - Malware Grid Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: text + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/ttp/threat-intel/get-feed + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-threat-intel-feed-malware-grid + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/sample_event.json b/packages/mimecast/data_stream/threat_intel_malware_grid/sample_event.json new file mode 100644 index 00000000000..23becc0e29b --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/sample_event.json @@ -0,0 +1,40 @@ +{ + "@timestamp": "2021-10-29T15:07:26.653Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:26.653Z", + "file": { + "hash": { + "sha256": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + } + }, + "modified_at": "2021-10-29T15:07:26.653Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-17T13:42:35.248902200Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--18c62174-0d31-4653-afe6-d104c57b6b2c\", \"created\": \"2021-10-29T15:07:26.653Z\", \"modified\": \"2021-10-29T15:07:26.653Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']\", \"valid_from\": \"2021-10-29T15:07:26.653Z\" }", + "category": "threat", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", + "log_type": "malware_grid", + "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log new file mode 100644 index 00000000000..4d5fbf0c8d2 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log @@ -0,0 +1,3 @@ +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","fileName":"numbers.pdf","fileType":"application\/pdf","result":"safe","actionTriggered":"user release, none","date":"2021-10-14T18:54:32+0000","details":"Safe \r\nTime taken: 0 hrs, 0 min, 4 sec","route":"inbound","messageId":"<20200806044148.F35F813B435@mail.brianjthronton.com>","subject":"Important Updated Numbers from the Center for Disease Control","fileHash":"eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3","definition":"Inbound - Safe file with On-Demand Sandbox"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","fileName":"Titus-Test Doc - Classification - InternalUseOnly.docx","fileType":"application\/vnd.openxmlformats-officedocument.wordprocessingml.document","result":"safe","actionTriggered":"user release, none","date":"2021-10-14T11:24:23+0000","details":"Safe \r\nTime taken: 0 hrs, 0 min, 5 sec","route":"inbound","messageId":"","subject":"FW: Titus classification work","fileHash":"2fb26be55ac710e4d9f80677ba24ae212dbb36bd934a0569fe521839e9f5d16e","definition":"Inbound - Safe file with On-Demand Sandbox"} +{"senderAddress":"<>","recipientAddress":"johndoe@example.com","fileName":"Titus classification v0.3.pptx","fileType":"application\/vnd.openxmlformats-officedocument.presentationml","result":"safe","actionTriggered":"user release, none","date":"2021-10-14T11:24:23+0000","details":"Safe \r\nTime taken: 0 hrs, 0 min, 5 sec","route":"inbound","messageId":"","subject":"FW: Titus classification work","fileHash":"111b86e1244ce6389efb60cddc001d594d334c540e85f9976be467a4ce472973","definition":"Inbound - Safe file with On-Demand Sandbox"} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json new file mode 100644 index 00000000000..fb94b0a115a --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json @@ -0,0 +1,142 @@ +{ + "expected": [ + { + "@timestamp": "2021-10-14T18:54:32.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hash": [ + "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3" + ] + }, + "rule": { + "name": "Inbound - Safe file with On-Demand Sandbox" + }, + "event": { + "action": "user_release_none", + "ingested": "2021-11-25T11:34:14.425308900Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"fileName\":\"numbers.pdf\",\"fileType\":\"application\\/pdf\",\"result\":\"safe\",\"actionTriggered\":\"user release, none\",\"date\":\"2021-10-14T18:54:32+0000\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 4 sec\",\"route\":\"inbound\",\"messageId\":\"\u003c20200806044148.F35F813B435@mail.brianjthronton.com\u003e\",\"subject\":\"Important Updated Numbers from the Center for Disease Control\",\"fileHash\":\"eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\"}", + "created": "2021-10-14T18:54:32+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20200806044148.F35F813B435@mail.brianjthronton.com\u003e", + "attachments": { + "hash": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3", + "file": { + "name": "numbers.pdf", + "mime_type": "application/pdf", + "extension": "pdf" + } + }, + "to": { + "address": "johndoe@example.com" + }, + "subject": "Important Updated Numbers from the Center for Disease Control", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "result": "safe", + "details": "Safe \r\nTime taken: 0 hrs, 0 min, 4 sec" + } + }, + { + "@timestamp": "2021-10-14T11:24:23.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hash": [ + "2fb26be55ac710e4d9f80677ba24ae212dbb36bd934a0569fe521839e9f5d16e" + ] + }, + "rule": { + "name": "Inbound - Safe file with On-Demand Sandbox" + }, + "event": { + "action": "user_release_none", + "ingested": "2021-11-25T11:34:14.425311400Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"fileName\":\"Titus-Test Doc - Classification - InternalUseOnly.docx\",\"fileType\":\"application\\/vnd.openxmlformats-officedocument.wordprocessingml.document\",\"result\":\"safe\",\"actionTriggered\":\"user release, none\",\"date\":\"2021-10-14T11:24:23+0000\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 5 sec\",\"route\":\"inbound\",\"messageId\":\"\u003cDB8P194MB0824EE4C8D360CCE3DEB0243A1B89@DB8P194MB0824.EURP194.PROD.OUTLOOK.COM\u003e\",\"subject\":\"FW: Titus classification work\",\"fileHash\":\"2fb26be55ac710e4d9f80677ba24ae212dbb36bd934a0569fe521839e9f5d16e\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\"}", + "created": "2021-10-14T11:24:23+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003cDB8P194MB0824EE4C8D360CCE3DEB0243A1B89@DB8P194MB0824.EURP194.PROD.OUTLOOK.COM\u003e", + "attachments": { + "hash": "2fb26be55ac710e4d9f80677ba24ae212dbb36bd934a0569fe521839e9f5d16e", + "file": { + "name": "Titus-Test Doc - Classification - InternalUseOnly.docx", + "mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "extension": "docx" + } + }, + "to": { + "address": "johndoe@example.com" + }, + "subject": "FW: Titus classification work", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "result": "safe", + "details": "Safe \r\nTime taken: 0 hrs, 0 min, 5 sec" + } + }, + { + "@timestamp": "2021-10-14T11:24:23.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hash": [ + "111b86e1244ce6389efb60cddc001d594d334c540e85f9976be467a4ce472973" + ] + }, + "rule": { + "name": "Inbound - Safe file with On-Demand Sandbox" + }, + "event": { + "action": "user_release_none", + "ingested": "2021-11-25T11:34:14.425312500Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"fileName\":\"Titus classification v0.3.pptx\",\"fileType\":\"application\\/vnd.openxmlformats-officedocument.presentationml\",\"result\":\"safe\",\"actionTriggered\":\"user release, none\",\"date\":\"2021-10-14T11:24:23+0000\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 5 sec\",\"route\":\"inbound\",\"messageId\":\"\u003cDB8P194MB0824EE4C8D360CCE3DEB0243A1B89@DB8P194MB0824.EURP194.PROD.OUTLOOK.COM\u003e\",\"subject\":\"FW: Titus classification work\",\"fileHash\":\"111b86e1244ce6389efb60cddc001d594d334c540e85f9976be467a4ce472973\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\"}", + "created": "2021-10-14T11:24:23+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003cDB8P194MB0824EE4C8D360CCE3DEB0243A1B89@DB8P194MB0824.EURP194.PROD.OUTLOOK.COM\u003e", + "attachments": { + "hash": "111b86e1244ce6389efb60cddc001d594d334c540e85f9976be467a4ce472973", + "file": { + "name": "Titus classification v0.3.pptx", + "mime_type": "application/vnd.openxmlformats-officedocument.presentationml", + "extension": "pptx" + } + }, + "to": { + "address": "johndoe@example.com" + }, + "subject": "FW: Titus classification work", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "result": "safe", + "details": "Safe \r\nTime taken: 0 hrs, 0 min, 5 sec" + } + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..cd7b70bbfc9 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/ttp/attachment/get-logs diff --git a/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..26ffd8cdc39 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/httpjson.yml.hbs @@ -0,0 +1,50 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: +- set: + target: body.data + value: '[{"oldestFirst": false, "route": "all", "result":"all","to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[.cursor.next_date]]"}]' + default: '[{"oldestFirst": false, "route": "all", "result":"all","to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json +- set: + target: header.x-mc-app-id + value: {{app_id}} +- set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' +- set: + target: header.x-mc-req-id + value: "[[uuid]]" +- set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/ttp/attachment/get-logs:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + target: body.data + split: + target: body.attachmentLogs +response.pagination: +- set: + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + fail_on_template_error: true +cursor: +next_date: + value: '[[.first_event.date]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..4a1650557c1 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,124 @@ +--- +description: Pipeline for processing sample logs +processors: + # Generic event/ecs fields we always want to populated + - set: + field: event.ingested + value: "{{ _ingest.timestamp }}" + - set: + field: ecs.version + value: "1.12.0" + - rename: + field: message + target_field: event.original + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.date == null + - date: + description: Use 'mimecast.date' as the '@timestamp' + field: mimecast.date + timezone: UTC + formats: + - yyyy-MM-dd'T'HH:mm:ssZ + + - rename: + field: mimecast.senderAddress + target_field: email.from.address + ignore_missing: true + if: 'ctx?.mimecast?.senderAddress !=null' + - rename: + field: mimecast.recipientAddress + target_field: email.to.address + ignore_missing: true + if: 'ctx?.mimecast?.recipientAddress !=null' + - gsub: + field: mimecast.actionTriggered + pattern: "," + replacement: "" + ignore_missing: true + - gsub: + field: mimecast.actionTriggered + pattern: " " + replacement: "_" + ignore_missing: true + - rename: + field: mimecast.actionTriggered + target_field: event.action + ignore_missing: true + if: 'ctx?.mimecast?.actionTriggered !=null' + - rename: + field: mimecast.subject + target_field: email.subject + ignore_missing: true + if: 'ctx?.mimecast?.subject !=null' + - rename: + field: mimecast.messageId + target_field: email.message_id + ignore_missing: true + if: 'ctx?.mimecast?.messageId !=null' + - rename: + field: mimecast.route + target_field: email.direction + ignore_missing: true + if: 'ctx?.mimecast?.route !=null' + - rename: + field: mimecast.fileName + target_field: email.attachments.file.name + ignore_missing: true + if: 'ctx?.mimecast?.fileName !=null' + - rename: + field: mimecast.definition + target_field: rule.name + ignore_missing: true + if: 'ctx?.mimecast?.definition !=null' + - rename: + field: mimecast.fileHash + target_field: email.attachments.hash + ignore_missing: true + if: 'ctx?.mimecast?.fileHash !=null' + - rename: + field: mimecast.fileType + target_field: email.attachments.file.mime_type + ignore_missing: true + if: 'ctx?.mimecast?.fileType !=null' + - set: + field: event.created + value: "{{mimecast.date}}" + if: 'ctx?.mimecast?.date != null' + - split: + field: email.attachments.file.name + separator: "\\." + target_field: file.parts + if: 'ctx?.email?.attachments?.file?.name != null' + - script: + lang: painless + source: | + ctx.email.attachments.file.extension = ctx.file.parts[ctx.file.parts.length-1]; + if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' + - append: + field: related.hash + value: "{{email.attachments.hash}}" + allow_duplicates: false + if: 'ctx?.email?.attachments?.hash !=null' + ### Cleanup + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast.date + - file.parts + - file + ignore_missing: true + - remove: + description: Remove 'event.original' if 'preserve_original_event' is not set. + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + +# Error handling +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/base-fields.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/base-fields.yml new file mode 100644 index 00000000000..d63e6b34a0e --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.ttp_ap_logs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml new file mode 100644 index 00000000000..d5cf859eb65 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml @@ -0,0 +1,55 @@ +- external: ecs + name: event.original +- external: ecs + name: event.action +- external: ecs + name: tags +- external: ecs + name: ecs.version +- external: ecs + name: event.action +- description: Direction of the message based on the sending and receiving domains + type: keyword + name: email.direction +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. + type: wildcard + name: email.message_id + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: The email address(es) of the message recipient(s) + type: keyword + name: email.to.address +- description: Stores the from email address from the RFC5322 From - header field. + type: keyword + name: email.from.address +- description: A brief summary of the topic of the message + type: keyword + name: email.subject + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: Name of the attachment file including the extension. + type: keyword + name: email.attachments.file.name +- description: MIME type of the attachment file. + type: keyword + name: email.attachments.file.mime_type +- external: ecs + name: rule.name +- description: File hash. + type: keyword + name: email.attachments.hash +- description: Attachment file extension, excluding the leading dot. + type: keyword + name: email.attachments.file.extension +- external: ecs + name: related.hash +- description: MIME type of the attachment file. + type: keyword + name: email.attachments.file.mime_type diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/field.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/field.yml new file mode 100644 index 00000000000..5d10a6e6638 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/field.yml @@ -0,0 +1,39 @@ +- name: mimecast + type: group + fields: + - name: result + type: keyword + description: The result of the attachment analysis - clean, malicious, unknown, or timeout. + - name: senderAddress + type: keyword + description: The sender of the attachment. + - name: fileName + type: keyword + description: The file name of the original attachment. + - name: actionTriggered + type: keyword + description: The action triggered for the attachment. + - name: route + type: keyword + description: The route of the original email containing the attachment, either - inbound, outbound, internal, or external. + - name: details + type: keyword + description: Detailed output of the attachment sandbox processing. + - name: recipientAddress + type: keyword + description: The address of the user that received the attachment. + - name: fileType + type: keyword + description: The file type of the attachment. + - name: subject + type: keyword + description: The subject of the email. + - name: fileHash + type: keyword + description: The hash of the attachment. + - name: messageId + type: keyword + description: The internet message id of the email. + - name: definition + type: keyword + description: The definition. diff --git a/packages/mimecast/data_stream/ttp_ap_logs/manifest.yml b/packages/mimecast/data_stream/ttp_ap_logs/manifest.yml new file mode 100644 index 00000000000..d2edd46208a --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/manifest.yml @@ -0,0 +1,77 @@ +title: "TTP Attachment Logs" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: TTP Attachment Logs + description: Collect TTP Attachment Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: text + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/ttp/attachment/get-logs + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-ttp-ap + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json b/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json new file mode 100644 index 00000000000..78bdf6beb1a --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ap_logs/sample_event.json @@ -0,0 +1,44 @@ +{ + "@timestamp": "2021-10-14T18:54:32.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hash": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3" + }, + "rule": { + "name": "Inbound - Safe file with On-Demand Sandbox" + }, + "event": { + "action": "user_release_none", + "ingested": "2021-11-19T14:40:07.263592900Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"fileName\":\"numbers.pdf\",\"fileType\":\"application\\/pdf\",\"result\":\"safe\",\"actionTriggered\":\"user release, none\",\"date\":\"2021-10-14T18:54:32+0000\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 4 sec\",\"route\":\"inbound\",\"messageId\":\"\u003c20200806044148.F35F813B435@mail.brianjthronton.com\u003e\",\"subject\":\"Important Updated Numbers from the Center for Disease Control\",\"fileHash\":\"eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\"}", + "created": "2021-10-14T18:54:32+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20200806044148.F35F813B435@mail.brianjthronton.com\u003e", + "attachments": { + "hash": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3", + "file": { + "name": "numbers.pdf", + "mime_type": "application/pdf", + "extension": "pdf" + } + }, + "to": { + "address": "johndoe@example.com" + }, + "subject": "Important Updated Numbers from the Center for Disease Control", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "result": "safe", + "details": "Safe \r\nTime taken: 0 hrs, 0 min, 4 sec" + } +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log new file mode 100644 index 00000000000..293dfe60f67 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log @@ -0,0 +1,3 @@ +{"id":"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG","senderAddress":"smtp@example.com","recipientAddress":"johndoe@example.com","subject":"Requested File","definition":"IP - 1 hit (Tag email)","hits":1,"identifiers":["internal_user_name"],"action":"none","taggedExternal":false,"taggedMalicious":true,"senderIpAddress":"8.8.8.8","eventTime":"2021-10-15T17:10:46+0000","impersonationResults":[{"impersonationDomainSource":"internal_user_name","similarDomain":"John Doe Jr ","stringSimilarToDomain":"John Doe Jr","checkerResult":"hit"}],"messageId":""} +{"id":"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzszAx0VEqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGShEhs","senderAddress":"johndoe@gmail.com","recipientAddress":"johndoe@example.com","subject":"Fwd: Here ya go","definition":"IP - 1 hit (Tag email)","hits":1,"identifiers":["internal_user_name"],"action":"none","taggedExternal":false,"taggedMalicious":true,"senderIpAddress":"8.8.8.8","eventTime":"2021-10-15T06:16:34+0000","impersonationResults":[{"impersonationDomainSource":"internal_user_name","similarDomain":"John Doe ","stringSimilarToDomain":"John Doe","checkerResult":"hit"}],"messageId":""} +{"id":"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzMDMz01EqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGQhEhc","senderAddress":"johndoe@mimecast.com","recipientAddress":"johndoe@example.com","subject":"RE: MSP Sales of Managed E2E","definition":"IP - 2 hits (Hold for Review \/ User Hold)","hits":2,"identifiers":["targeted_threat_dictionary","internal_user_name"],"action":"hold","taggedExternal":false,"taggedMalicious":true,"senderIpAddress":"8.8.8.8","eventTime":"2021-10-13T16:12:07+0000","impersonationResults":[{"impersonationDomainSource":"internal_user_name","similarDomain":"Emily Doe ","stringSimilarToDomain":"Emily Doe","checkerResult":"hit"},{"impersonationDomainSource":"targeted_threat_dictionary","stringSimilarToDomain":"who"}],"messageId":""} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json new file mode 100644 index 00000000000..9358a91a0f2 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json @@ -0,0 +1,168 @@ +{ + "expected": [ + { + "@timestamp": "2021-10-15T17:10:46.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "8.8.8.8" + ] + }, + "rule": { + "name": "IP - 1 hit (Tag email)" + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "action": "none", + "ingested": "2021-11-25T11:34:15.002442200Z", + "original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG\",\"senderAddress\":\"smtp@example.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Requested File\",\"definition\":\"IP - 1 hit (Tag email)\",\"hits\":1,\"identifiers\":[\"internal_user_name\"],\"action\":\"none\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"8.8.8.8\",\"eventTime\":\"2021-10-15T17:10:46+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe Jr \u003cjohndoejr@example.com\u003e\",\"stringSimilarToDomain\":\"John Doe Jr\",\"checkerResult\":\"hit\"}],\"messageId\":\"\u003cEE7E97EA-1926-4A90-9399-D049A98893F4@emailsec.ninja\u003e\"}", + "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG", + "created": "2021-10-15T17:10:46+0000" + }, + "email": { + "from": { + "address": "smtp@example.com" + }, + "message_id": "\u003cEE7E97EA-1926-4A90-9399-D049A98893F4@emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Requested File" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "hits": 1, + "impersonationResults": [ + { + "checkerResult": "hit", + "impersonationDomainSource": "internal_user_name", + "stringSimilarToDomain": "John Doe Jr", + "similarDomain": "John Doe Jr \u003cjohndoejr@example.com\u003e" + } + ], + "taggedMalicious": true, + "taggedExternal": false, + "identifiers": [ + "internal_user_name" + ] + } + }, + { + "@timestamp": "2021-10-15T06:16:34.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "8.8.8.8" + ] + }, + "rule": { + "name": "IP - 1 hit (Tag email)" + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "action": "none", + "ingested": "2021-11-25T11:34:15.002445300Z", + "original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzszAx0VEqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGShEhs\",\"senderAddress\":\"johndoe@gmail.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Fwd: Here ya go\",\"definition\":\"IP - 1 hit (Tag email)\",\"hits\":1,\"identifiers\":[\"internal_user_name\"],\"action\":\"none\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"8.8.8.8\",\"eventTime\":\"2021-10-15T06:16:34+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe \u003cjohndoe@example.com\u003e\",\"stringSimilarToDomain\":\"John Doe\",\"checkerResult\":\"hit\"}],\"messageId\":\"\u003cCAOsCE-eP_fM6j=OL7Mwufic_s8t8VgNaCWdWM+sHYvWAFxiDig@mail.gmail.com\u003e\"}", + "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzszAx0VEqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGShEhs", + "created": "2021-10-15T06:16:34+0000" + }, + "email": { + "from": { + "address": "johndoe@gmail.com" + }, + "message_id": "\u003cCAOsCE-eP_fM6j=OL7Mwufic_s8t8VgNaCWdWM+sHYvWAFxiDig@mail.gmail.com\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Fwd: Here ya go" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "hits": 1, + "impersonationResults": [ + { + "checkerResult": "hit", + "impersonationDomainSource": "internal_user_name", + "stringSimilarToDomain": "John Doe", + "similarDomain": "John Doe \u003cjohndoe@example.com\u003e" + } + ], + "taggedMalicious": true, + "taggedExternal": false, + "identifiers": [ + "internal_user_name" + ] + } + }, + { + "@timestamp": "2021-10-13T16:12:07.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "8.8.8.8" + ] + }, + "rule": { + "name": "IP - 2 hits (Hold for Review / User Hold)" + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "action": "hold", + "ingested": "2021-11-25T11:34:15.002446500Z", + "original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzMDMz01EqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGQhEhc\",\"senderAddress\":\"johndoe@mimecast.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"RE: MSP Sales of Managed E2E\",\"definition\":\"IP - 2 hits (Hold for Review \\/ User Hold)\",\"hits\":2,\"identifiers\":[\"targeted_threat_dictionary\",\"internal_user_name\"],\"action\":\"hold\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"8.8.8.8\",\"eventTime\":\"2021-10-13T16:12:07+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"Emily Doe \u003cemilydoe@example.com\u003e\",\"stringSimilarToDomain\":\"Emily Doe\",\"checkerResult\":\"hit\"},{\"impersonationDomainSource\":\"targeted_threat_dictionary\",\"stringSimilarToDomain\":\"who\"}],\"messageId\":\"\u003cPR3P194MB06183A3BE81F0831A8402B47D3B79@PR3P194MB0618.EURP194.PROD.OUTLOOK.COM\u003e\"}", + "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzMDMz01EqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGQhEhc", + "created": "2021-10-13T16:12:07+0000" + }, + "email": { + "from": { + "address": "johndoe@mimecast.com" + }, + "message_id": "\u003cPR3P194MB06183A3BE81F0831A8402B47D3B79@PR3P194MB0618.EURP194.PROD.OUTLOOK.COM\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "RE: MSP Sales of Managed E2E" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "hits": 2, + "impersonationResults": [ + { + "checkerResult": "hit", + "impersonationDomainSource": "internal_user_name", + "stringSimilarToDomain": "Emily Doe", + "similarDomain": "Emily Doe \u003cemilydoe@example.com\u003e" + }, + { + "impersonationDomainSource": "targeted_threat_dictionary", + "stringSimilarToDomain": "who" + } + ], + "taggedMalicious": true, + "taggedExternal": false, + "identifiers": [ + "targeted_threat_dictionary", + "internal_user_name" + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..6cc61933a72 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/ttp/impersonation/get-logs diff --git a/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..b540773d61d --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/httpjson.yml.hbs @@ -0,0 +1,50 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: +- set: + target: body.data + value: '[{"oldestFirst": false,"to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[.cursor.next_date]]"}]' + default: '[{"oldestFirst": false,"to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json +- set: + target: header.x-mc-app-id + value: {{app_id}} +- set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' +- set: + target: header.x-mc-req-id + value: "[[uuid]]" +- set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/ttp/impersonation/get-logs:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + target: body.data + split: + target: body.impersonationLogs +response.pagination: +- set: + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + fail_on_template_error: true +cursor: +next_date: + value: '[[.first_event.eventTime]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..8b3e430570f --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,89 @@ +--- +description: Pipeline for processing sample logs +processors: + # Generic event/ecs fields we always want to populated + - set: + field: event.ingested + value: "{{ _ingest.timestamp }}" + - set: + field: ecs.version + value: "1.12.0" + - rename: + field: message + target_field: event.original + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.eventTime == null + - date: + description: Use 'mimecast.eventTime' as the '@timestamp' + field: mimecast.eventTime + timezone: UTC + formats: + - yyyy-MM-dd'T'HH:mm:ssZ + ### + - rename: + field: mimecast.senderIpAddress + target_field: source.ip + ignore_missing: true + - rename: + field: mimecast.senderAddress + target_field: email.from.address + ignore_missing: true + - rename: + field: mimecast.subject + target_field: email.subject + ignore_missing: true + - rename: + field: mimecast.action + target_field: event.action + ignore_missing: true + - rename: + field: mimecast.definition + target_field: rule.name + ignore_missing: true + - rename: + field: mimecast.id + target_field: event.id + ignore_missing: true + - rename: + field: mimecast.recipientAddress + target_field: email.to.address + ignore_missing: true + - rename: + field: mimecast.messageId + target_field: email.message_id + ignore_missing: true + - rename: + field: mimecast.similarDomain + target_field: source.domain + ignore_missing: true + - set: + field: event.created + value: "{{mimecast.eventTime}}" + if: 'ctx?.mimecast?.eventTime != null' + - append: + field: related.ip + value: "{{source.ip}}" + allow_duplicates: false + if: 'ctx?.source?.ip != null' + + ### Cleanup + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast.eventTime + ignore_missing: true + - remove: + description: Remove 'event.original' if 'preserve_original_event' is not set. + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + +# Error handling +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/base-fields.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/base-fields.yml new file mode 100644 index 00000000000..96b197da78b --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.ttp_ip_logs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml new file mode 100644 index 00000000000..bee1ef94972 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml @@ -0,0 +1,43 @@ +- external: ecs + name: event.original +- external: ecs + name: event.action +- external: ecs + name: tags +- external: ecs + name: ecs.version +- external: ecs + name: event.action +- external: ecs + name: source.ip +- description: Stores the from email address from the RFC5322 From - header field. + type: keyword + name: email.from.address +- description: A brief summary of the topic of the message + type: keyword + name: email.subject + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false +- external: ecs + name: rule.name +- external: ecs + name: event.id +- description: The email address(es) of the message recipient(s) + type: keyword + name: email.to.address +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. + type: wildcard + name: email.message_id + multi_fields: + - name: text + type: text + norms: false + default_field: false +- external: ecs + name: source.domain +- external: ecs + name: related.ip diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/field.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/field.yml new file mode 100644 index 00000000000..d283a051317 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/field.yml @@ -0,0 +1,54 @@ +- name: mimecast + type: group + fields: + - name: hits + type: long + description: The number of identifiers that the message triggered. + - name: taggedMalicious + type: boolean + description: Whether the message was tagged as malicious. + - name: senderIpAddress + type: keyword + description: The source IP address of the message. + - name: senderAddress + type: keyword + description: The email address of the sender of the message. + - name: subject + type: keyword + description: The subject of the email. + - name: identifiers + type: keyword + description: The properties of the message that triggered the action - similar_internal_domain, newly_observed_domain, internal_user_name, reply_address_mismatch, and/or targeted_threat_dictionary. + - name: action + type: keyword + description: The action triggered by the email. + - name: definition + type: keyword + description: The name of the policy definition that triggered the log. + - name: id + type: keyword + description: A token that can be used to retrieve this log again. + - name: recipientAddress + type: keyword + description: The email address of the recipient of the email. + - name: taggedExternal + type: boolean + description: Whether the message was tagged as coming from an external address. + - name: messageId + type: keyword + description: The message-id of the identified message. + - name: impersonationResults + type: group + fields: + - name: impersonationDomainSource + type: keyword + description: Impersonation domain source. + - name: stringSimilarToDomain + type: keyword + description: The string that is suspiciously similar to a known value within the Mimecast configuration. Multiple triggers will be comma-separated. + - name: similarDomain + type: keyword + description: Similar domain. + - name: checkerResult + type: keyword + description: Result checker. diff --git a/packages/mimecast/data_stream/ttp_ip_logs/manifest.yml b/packages/mimecast/data_stream/ttp_ip_logs/manifest.yml new file mode 100644 index 00000000000..4f415213bda --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/manifest.yml @@ -0,0 +1,77 @@ +title: "TTP Impersonation Mimecast Logs" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: TTP Impersonation events + description: Collect TTP Impersonation Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: text + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/ttp/impersonation/get-logs + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-ttp-ip + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/ttp_ip_logs/sample_event.json b/packages/mimecast/data_stream/ttp_ip_logs/sample_event.json new file mode 100644 index 00000000000..1ebe748244a --- /dev/null +++ b/packages/mimecast/data_stream/ttp_ip_logs/sample_event.json @@ -0,0 +1,51 @@ +{ + "@timestamp": "2021-10-15T17:10:46.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": "8.8.8.8" + }, + "rule": { + "name": "IP - 1 hit (Tag email)" + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "action": "none", + "ingested": "2021-11-19T14:42:59.823940200Z", + "original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG\",\"senderAddress\":\"smtp@example.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Requested File\",\"definition\":\"IP - 1 hit (Tag email)\",\"hits\":1,\"identifiers\":[\"internal_user_name\"],\"action\":\"none\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"8.8.8.8\",\"eventTime\":\"2021-10-15T17:10:46+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe Jr \u003cjohndoejr@example.com\u003e\",\"stringSimilarToDomain\":\"John Doe Jr\",\"checkerResult\":\"hit\"}],\"messageId\":\"\u003cEE7E97EA-1926-4A90-9399-D049A98893F4@emailsec.ninja\u003e\"}", + "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG", + "created": "2021-10-15T17:10:46+0000" + }, + "email": { + "from": { + "address": "smtp@example.com" + }, + "message_id": "\u003cEE7E97EA-1926-4A90-9399-D049A98893F4@emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Requested File" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "hits": 1, + "impersonationResults": [ + { + "checkerResult": "hit", + "impersonationDomainSource": "internal_user_name", + "stringSimilarToDomain": "John Doe Jr", + "similarDomain": "John Doe Jr \u003cjohndoejr@example.com\u003e" + } + ], + "taggedMalicious": true, + "taggedExternal": false, + "identifiers": [ + "internal_user_name" + ] + } +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log new file mode 100644 index 00000000000..796ea428c70 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log @@ -0,0 +1,3 @@ +{"userEmailAddress": "johndoe@example.com", "fromUserEmailAddress": "bestbuyinfo@emailinfo.bestbuy.com", "url": "https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d", "ttpDefinition": "Inbound URL 'Aggressive'", "subject": "Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.", "action": "allow", "adminOverride": "N/A", "userOverride": "None", "scanResult": "clean", "category": "Business", "sendingIp": "8.8.8.8", "userAwarenessAction": "Continue", "date": "2021-10-16T14:45:34+0000", "actions": "Allow", "route": "inbound", "creationMethod": "User Click", "emailPartsDescription": [ "Body" ], "messageId": "<31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local>" } +{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"noreply@r.livingsocial.com","url":"https:\/\/www.livingsocial.com\/browse\/?locale=en_US&topCategory=all-deals&p=14&utm_source=newsletter_im&utm_medium=email&t_division=boston&date=20211016&uu=1bea09ca-8a29-11e9-b7f7-0242ac120002&CID=US&tx=0&s=body&c=banner&d=dynamic-banner-4&utm_campaign=194d1bb8-dc74-4bed-b470-0154e934bfb3_0_20211016_treatment0","ttpDefinition":"Inbound URL 'Aggressive'","subject":"Jump Pass + Mega Sale","action":"allow","adminOverride":"N\/A","userOverride":"None","scanResult":"clean","category":"Business","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-10-16T14:07:38+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<803962655.28921622.1634393221485.JavaMail.rocketman@push-dispatcher65.sac1>"} +{"userEmailAddress":"johndoe@example.com","fromUserEmailAddress":"nflshop.com@eml.nflshop.com","url":"https:\/\/www.nflshop.com\/how-can-i-contact-customer-service\/ch-2244","ttpDefinition":"Inbound URL 'Aggressive'","subject":"25% Off Tees to Give During Early Gifting Sale","action":"allow","adminOverride":"N\/A","userOverride":"None","scanResult":"clean","category":"Fashion & Beauty","sendingIp":"8.8.8.8","userAwarenessAction":"Continue","date":"2021-10-16T13:31:56+0000","actions":"Allow","route":"inbound","creationMethod":"User Click","emailPartsDescription":["Body"],"messageId":"<28ad4be3-2d3a-491d-9aa7-a5a907123da1@ind1s01mta1115.xt.local>"} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json new file mode 100644 index 00000000000..baa890b2ad9 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json @@ -0,0 +1,178 @@ +{ + "expected": [ + { + "rule": { + "name": "Inbound URL 'Aggressive'" + }, + "source": { + "ip": "8.8.8.8" + }, + "url": { + "original": "https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-10-16T14:45:34.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "event": { + "action": "Continue", + "ingested": "2021-11-25T11:34:15.770244300Z", + "original": "{\"userEmailAddress\": \"johndoe@example.com\", \"fromUserEmailAddress\": \"bestbuyinfo@emailinfo.bestbuy.com\", \"url\": \"https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d\", \"ttpDefinition\": \"Inbound URL 'Aggressive'\", \"subject\": \"Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.\", \"action\": \"allow\", \"adminOverride\": \"N/A\", \"userOverride\": \"None\", \"scanResult\": \"clean\", \"category\": \"Business\", \"sendingIp\": \"8.8.8.8\", \"userAwarenessAction\": \"Continue\", \"date\": \"2021-10-16T14:45:34+0000\", \"actions\": \"Allow\", \"route\": \"inbound\", \"creationMethod\": \"User Click\", \"emailPartsDescription\": [ \"Body\" ], \"messageId\": \"\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e\" }", + "created": "2021-10-16T14:45:34+0000" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "email": { + "message_id": "\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e", + "from": { + "address": "bestbuyinfo@emailinfo.bestbuy.com" + }, + "subject": "Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.", + "direction": "inbound" + }, + "mimecast": { + "userOverride": "None", + "action": "allow", + "adminOverride": "N/A", + "scanResult": "clean", + "category": "Business", + "actions": "Allow", + "creationMethod": "User Click", + "emailPartsDescription": [ + "Body" + ] + } + }, + { + "rule": { + "name": "Inbound URL 'Aggressive'" + }, + "source": { + "ip": "8.8.8.8" + }, + "url": { + "original": "https://www.livingsocial.com/browse/?locale=en_US\u0026topCategory=all-deals\u0026p=14\u0026utm_source=newsletter_im\u0026utm_medium=email\u0026t_division=boston\u0026date=20211016\u0026uu=1bea09ca-8a29-11e9-b7f7-0242ac120002\u0026CID=US\u0026tx=0\u0026s=body\u0026c=banner\u0026d=dynamic-banner-4\u0026utm_campaign=194d1bb8-dc74-4bed-b470-0154e934bfb3_0_20211016_treatment0" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-10-16T14:07:38.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "event": { + "action": "Continue", + "ingested": "2021-11-25T11:34:15.770246800Z", + "original": "{\"userEmailAddress\":\"johndoe@example.com\",\"fromUserEmailAddress\":\"noreply@r.livingsocial.com\",\"url\":\"https:\\/\\/www.livingsocial.com\\/browse\\/?locale=en_US\u0026topCategory=all-deals\u0026p=14\u0026utm_source=newsletter_im\u0026utm_medium=email\u0026t_division=boston\u0026date=20211016\u0026uu=1bea09ca-8a29-11e9-b7f7-0242ac120002\u0026CID=US\u0026tx=0\u0026s=body\u0026c=banner\u0026d=dynamic-banner-4\u0026utm_campaign=194d1bb8-dc74-4bed-b470-0154e934bfb3_0_20211016_treatment0\",\"ttpDefinition\":\"Inbound URL 'Aggressive'\",\"subject\":\"Jump Pass + Mega Sale\",\"action\":\"allow\",\"adminOverride\":\"N\\/A\",\"userOverride\":\"None\",\"scanResult\":\"clean\",\"category\":\"Business\",\"sendingIp\":\"8.8.8.8\",\"userAwarenessAction\":\"Continue\",\"date\":\"2021-10-16T14:07:38+0000\",\"actions\":\"Allow\",\"route\":\"inbound\",\"creationMethod\":\"User Click\",\"emailPartsDescription\":[\"Body\"],\"messageId\":\"\u003c803962655.28921622.1634393221485.JavaMail.rocketman@push-dispatcher65.sac1\u003e\"}", + "created": "2021-10-16T14:07:38+0000" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "email": { + "message_id": "\u003c803962655.28921622.1634393221485.JavaMail.rocketman@push-dispatcher65.sac1\u003e", + "from": { + "address": "noreply@r.livingsocial.com" + }, + "subject": "Jump Pass + Mega Sale", + "direction": "inbound" + }, + "mimecast": { + "userOverride": "None", + "action": "allow", + "adminOverride": "N/A", + "scanResult": "clean", + "category": "Business", + "actions": "Allow", + "creationMethod": "User Click", + "emailPartsDescription": [ + "Body" + ] + } + }, + { + "rule": { + "name": "Inbound URL 'Aggressive'" + }, + "source": { + "ip": "8.8.8.8" + }, + "url": { + "original": "https://www.nflshop.com/how-can-i-contact-customer-service/ch-2244" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-10-16T13:31:56.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "event": { + "action": "Continue", + "ingested": "2021-11-25T11:34:15.770247900Z", + "original": "{\"userEmailAddress\":\"johndoe@example.com\",\"fromUserEmailAddress\":\"nflshop.com@eml.nflshop.com\",\"url\":\"https:\\/\\/www.nflshop.com\\/how-can-i-contact-customer-service\\/ch-2244\",\"ttpDefinition\":\"Inbound URL 'Aggressive'\",\"subject\":\"25% Off Tees to Give During Early Gifting Sale\",\"action\":\"allow\",\"adminOverride\":\"N\\/A\",\"userOverride\":\"None\",\"scanResult\":\"clean\",\"category\":\"Fashion \u0026 Beauty\",\"sendingIp\":\"8.8.8.8\",\"userAwarenessAction\":\"Continue\",\"date\":\"2021-10-16T13:31:56+0000\",\"actions\":\"Allow\",\"route\":\"inbound\",\"creationMethod\":\"User Click\",\"emailPartsDescription\":[\"Body\"],\"messageId\":\"\u003c28ad4be3-2d3a-491d-9aa7-a5a907123da1@ind1s01mta1115.xt.local\u003e\"}", + "created": "2021-10-16T13:31:56+0000" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "email": { + "message_id": "\u003c28ad4be3-2d3a-491d-9aa7-a5a907123da1@ind1s01mta1115.xt.local\u003e", + "from": { + "address": "nflshop.com@eml.nflshop.com" + }, + "subject": "25% Off Tees to Give During Early Gifting Sale", + "direction": "inbound" + }, + "mimecast": { + "userOverride": "None", + "action": "allow", + "adminOverride": "N/A", + "scanResult": "clean", + "category": "Fashion \u0026 Beauty", + "actions": "Allow", + "creationMethod": "User Click", + "emailPartsDescription": [ + "Body" + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..a1d25258f18 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/system/test-default-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: mimecast +vars: ~ +request.method: "POST" +data_stream: + vars: + preserve_original_event: true + api_key: test + api_url: http://{{Hostname}}:{{Port}}/api/ttp/url/get-logs diff --git a/packages/mimecast/data_stream/ttp_url_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/ttp_url_logs/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..f3648885d6d --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/agent/stream/httpjson.yml.hbs @@ -0,0 +1,50 @@ +config_version: "2" +interval: {{interval}} +request.url: {{api_url}} +request.method: "POST" +request.transforms: +- set: + target: body.data + value: '[{"oldestFirst": false,"scanResult": "all","route":"all","to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[.cursor.next_date]]"}]' + default: '[{"oldestFirst": false,"scanResult": "all","route":"all","to": "[[formatDate (now) "2006-01-02T15:04:05+0700"]]", "from":"[[formatDate (now (parseDuration "-{{interval}}")) "2006-01-02T15:04:05+0700"]]"}]' + value_type: json +- set: + target: header.x-mc-app-id + value: {{app_id}} +- set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' +- set: + target: header.x-mc-req-id + value: "[[uuid]]" +- set: + target: header.Authorization + value: 'MC {{access_key}}:[[hmacBase64 "sha1" (base64Decode "{{secret_key}}") (sprintf "%s:%s:/api/ttp/url/get-logs:{{app_key}}" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]]' + fail_on_template_error: true +response.decode_as: application/json +response.split: + target: body.data + split: + target: body.clickLogs +response.pagination: +- set: + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + fail_on_template_error: true +cursor: + next_date: + value: '[[.first_event.date]]' +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..2d8a3e4dde0 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,118 @@ +--- +description: Pipeline for processing sample logs +processors: + # Generic event/ecs fields we always want to populated + - set: + field: event.ingested + value: "{{ _ingest.timestamp }}" + - set: + field: ecs.version + value: "1.12.0" + - rename: + field: message + target_field: event.original + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.date == null + - date: + description: Use 'mimecast.date' as the '@timestamp' + field: mimecast.date + timezone: UTC + formats: + - yyyy-MM-dd'T'HH:mm:ssZ + + ### + + # Convert 'mimecast.auditType' to a bone-cased event action. + # ie: User Log On -> user-log-on + - rename: + field: mimecast.userEmailAddress + target_field: user.email + ignore_missing: true + - rename: + field: mimecast.url + target_field: url.original + ignore_missing: true + - rename: + field: mimecast.userAwarenessAction + target_field: event.action + ignore_missing: true + - rename: + field: mimecast.route + target_field: email.direction + ignore_missing: true + - rename: + field: mimecast.ttpDefinition + target_field: rule.name + ignore_missing: true + - rename: + field: mimecast.subject + target_field: email.subject + ignore_missing: true + - rename: + field: mimecast.messageId + target_field: email.message_id + ignore_missing: true + - rename: + field: mimecast.fromUserEmailAddress + target_field: email.from.address + ignore_missing: true + - rename: + field: mimecast.sendingIp + target_field: source.ip + ignore_missing: true + - set: + field: event.created + value: "{{mimecast.date}}" + if: 'ctx?.mimecast?.date != null' + - split: + field: user.email + separator: "@" + target_field: user.parts + if: 'ctx?.user?.email != null' + - set: + field: user.name + copy_from: user.parts.0 + if: 'ctx?.user?.parts !=null && ctx?.user?.parts.length > 1' + - set: + field: user.domain + copy_from: user.parts.1 + if: 'ctx?.user?.parts !=null && ctx?.user?.parts.length > 1' + - append: + field: related.ip + value: "{{source.ip}}" + allow_duplicates: false + if: 'ctx?.source?.ip !=null' + - append: + field: related.user + value: "{{user.name}}" + allow_duplicates: false + if: 'ctx?.user?.name !=null' + - append: + field: related.user + value: "{{user.email}}" + allow_duplicates: false + if: 'ctx?.user?.email !=null' + ### Cleanup + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast.date + - user.parts + - user.parts.0 + - user.parts.1 + ignore_missing: true + - remove: + description: Remove 'event.original' if 'preserve_original_event' is not set. + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + +# Error handling +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/base-fields.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/base-fields.yml new file mode 100644 index 00000000000..29bf273c1b1 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.ttp_url_logs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml new file mode 100644 index 00000000000..81fa9adc25a --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml @@ -0,0 +1,49 @@ +- external: ecs + name: event.original +- external: ecs + name: event.action +- external: ecs + name: tags +- external: ecs + name: ecs.version +- external: ecs + name: event.action +- external: ecs + name: source.ip +- description: Stores the from email address from the RFC5322 From - header field. + type: keyword + name: email.from.address +- description: A brief summary of the topic of the message + type: keyword + name: email.subject + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. + type: wildcard + name: email.message_id + multi_fields: + - name: text + type: text + norms: false + default_field: false +- description: Direction of the message based on the sending and receiving domains + type: keyword + name: email.direction +- external: ecs + name: rule.name +- external: ecs + name: user.email +- external: ecs + name: url.original +- external: ecs + name: user.name +- external: ecs + name: user.domain +- external: ecs + name: related.ip +- external: ecs + name: related.user diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/field.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/field.yml new file mode 100644 index 00000000000..b2cd2e06f60 --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/field.yml @@ -0,0 +1,54 @@ +- name: mimecast + type: group + fields: + - name: category + type: keyword + description: The category of the URL clicked. + - name: userEmailAddress + type: keyword + description: The email address of the user who clicked the link. + - name: url + type: keyword + description: The url clicked. + - name: userAwarenessAction + type: keyword + description: The action taken by the user if user awareness was applied. + - name: route + type: keyword + description: The route of the email that contained the link. + - name: adminOverride + type: keyword + description: The action defined by the administrator for the URL. + - name: scanResult + type: keyword + description: The result of the URL scan. + - name: action + type: keyword + description: The action that was taken for the click. + - name: ttpDefinition + type: keyword + description: The description of the definition that triggered the URL to be rewritten by Mimecast. + - name: userOverride + type: keyword + description: The action requested by the user. + - name: emailPartsDescription + type: keyword + description: An array of components of the messge where the URL was found. + - name: subject + type: keyword + description: The subject of the email. + - name: messageId + type: keyword + description: The message-id value of the message. + - name: actions + type: keyword + description: The actions that were taken. + - name: creationMethod + type: keyword + description: The description how event occurred. + - name: fromUserEmailAddress + type: keyword + description: The email of user who triggers the event. + - name: sendingIP + type: keyword + description: The IP of user who triggers the event. diff --git a/packages/mimecast/data_stream/ttp_url_logs/manifest.yml b/packages/mimecast/data_stream/ttp_url_logs/manifest.yml new file mode 100644 index 00000000000..4de8b587cde --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/manifest.yml @@ -0,0 +1,77 @@ +title: "TTP URL Logs" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: TTP URL Logs + description: Collect TTP URL Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. + multi: false + required: true + show_user: false + default: 5m + - name: api_url + type: text + title: API URL + description: API Url. + multi: false + required: true + show_user: false + default: https://eu-api.mimecast.com/api/ttp/url/get-logs + - name: app_key + type: password + title: Application Key + description: Specifies application key for user. + multi: false + required: true + show_user: true + - name: app_id + type: password + title: Application ID + description: Set the Application Id. + multi: false + required: true + show_user: true + - name: access_key + type: password + title: Access Key + description: Set Access Key. + multi: false + required: true + show_user: true + - name: secret_key + type: password + title: Secret Key + description: Set Secret Key. + multi: false + required: true + show_user: true + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-ttp-url + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/ttp_url_logs/sample_event.json b/packages/mimecast/data_stream/ttp_url_logs/sample_event.json new file mode 100644 index 00000000000..caff8ea714c --- /dev/null +++ b/packages/mimecast/data_stream/ttp_url_logs/sample_event.json @@ -0,0 +1,58 @@ +{ + "rule": { + "name": "Inbound URL 'Aggressive'" + }, + "source": { + "ip": "8.8.8.8" + }, + "url": { + "original": "https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-10-16T14:45:34.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "event": { + "action": "Continue", + "ingested": "2021-11-24T14:39:10.084705200Z", + "original": "{\"userEmailAddress\": \"johndoe@example.com\", \"fromUserEmailAddress\": \"bestbuyinfo@emailinfo.bestbuy.com\", \"url\": \"https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d\", \"ttpDefinition\": \"Inbound URL 'Aggressive'\", \"subject\": \"Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.\", \"action\": \"allow\", \"adminOverride\": \"N/A\", \"userOverride\": \"None\", \"scanResult\": \"clean\", \"category\": \"Business\", \"sendingIp\": \"8.8.8.8\", \"userAwarenessAction\": \"Continue\", \"date\": \"2021-10-16T14:45:34+0000\", \"actions\": \"Allow\", \"route\": \"inbound\", \"creationMethod\": \"User Click\", \"emailPartsDescription\": [ \"Body\" ], \"messageId\": \"\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e\" }", + "created": "2021-10-16T14:45:34+0000" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "email": { + "message_id": "\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e", + "from": { + "address": "bestbuyinfo@emailinfo.bestbuy.com" + }, + "subject": "Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.", + "direction": "inbound" + }, + "mimecast": { + "userOverride": "None", + "action": "allow", + "adminOverride": "N/A", + "scanResult": "clean", + "category": "Business", + "actions": "Allow", + "creationMethod": "User Click", + "emailPartsDescription": [ + "Body" + ] + } +} \ No newline at end of file diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md new file mode 100644 index 00000000000..e63251ac9b3 --- /dev/null +++ b/packages/mimecast/docs/README.md @@ -0,0 +1,1086 @@ +# Mimecast Integration + +The Mimecast integration collects events from the Mimecast API. + +## Logs + +### AUDIT EVENTS + +This is the `mimecast.audit_events` dataset. + +An example event for `audit_events` looks as following: + +```json +{ + "@timestamp": "2021-11-16T12:01:37.000Z", + "agent": { + "ephemeral_id": "57841034-22ed-4fcd-bcfd-0a9518249e2d", + "hostname": "docker-fleet-agent", + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.16.0" + }, + "client": { + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "geo": { + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 37.751, + "lon": -97.822 + } + }, + "ip": "8.8.8.8" + }, + "data_stream": { + "dataset": "mimecast.audit_events", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "snapshot": true, + "version": "7.16.0" + }, + "event": { + "action": "case-action", + "agent_id_status": "verified", + "created": "2021-11-16T12:01:37.000Z", + "dataset": "mimecast.audit_events", + "id": "eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI", + "ingested": "2021-11-24T15:39:11Z", + "original": "{\"auditType\":\"Case Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI\",\"user\":\"johndoe@example.com\"}" + }, + "input": { + "type": "httpjson" + }, + "mimecast": { + "application": "mimecast-case-review", + "category": "case_review_logs", + "eventInfo": "Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review" + }, + "related": { + "ip": [ + "8.8.8.8" + ], + "user": [ + "johndoe", + "johndoe@example.com" + ] + }, + "tags": [ + "preserve_original_event", + "forwarded", + "mimecast-audit-events" + ], + "user": { + "domain": "example.com", + "email": "johndoe@example.com", + "name": "johndoe" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| client.as.asn | Client ASN number. | long | +| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| client.as.organization.name | Organization name. | keyword | +| client.as.organization_name | | keyword | +| client.geo.city_name | City name. | keyword | +| client.geo.continent_name | Name of the continent. | keyword | +| client.geo.country_iso_code | Country ISO code. | keyword | +| client.geo.country_name | Country name. | keyword | +| client.geo.location | Longitude and latitude. | geo_point | +| client.geo.region_iso_code | Region ISO code. | keyword | +| client.geo.region_name | Region name. | keyword | +| client.ip | IP address of the client (IPv4 or IPv6). | ip | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | +| email.origination_timestamp | The date and time the email message was composed. Many email clients will fill this in automatically when the message is sent by a user. | date | +| email.subject | A brief summary of the topic of the message | keyword | +| email.to.address | The email address(es) of the message recipient(s) | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.id | Unique ID to describe the event. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| file.name | Name of the file including the extension, without the directory. | keyword | +| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.application | The Mimecast unique id of the event. | keyword | +| mimecast.category | The category of the event. | keyword | +| mimecast.email.address | Email address from event info. | keyword | +| mimecast.email.metadata | The email meta data from audit info. | keyword | +| mimecast.eventInfo | The detailed event information. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| related.user | All the user names or other user identifiers seen on the event. | keyword | +| tags | List of keywords used to tag each event. | keyword | +| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | +| user.email | User email address. | keyword | +| user.name | Short name or login of the user. | keyword | + + +### DLP LOGS + +This is the `mimecast.dlp_logs` dataset. + +An example event for `dlp` looks as following: + +```json +{ + "@timestamp": "2021-11-18T21:41:18.000Z", + "agent": { + "ephemeral_id": "1aef981f-3448-4d12-bd5a-723ac1cdcc81", + "hostname": "docker-fleet-agent", + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.16.0" + }, + "data_stream": { + "dataset": "mimecast.dlp_logs", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab", + "snapshot": true, + "version": "7.16.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20211118214115.B346F10021D@mail.emailsec.ninja\u003e", + "subject": "Undelivered Mail Returned to Sender", + "to": { + "address": "johndoe@example.com" + } + }, + "event": { + "action": "notification", + "agent_id_status": "verified", + "created": "2021-11-18T21:41:18+0000", + "dataset": "mimecast.dlp_logs", + "ingested": "2021-11-24T15:39:49Z", + "original": "{\"action\":\"notification\",\"eventTime\":\"2021-11-18T21:41:18+0000\",\"messageId\":\"\\u003c20211118214115.B346F10021D@mail.emailsec.ninja\\u003e\",\"policy\":\"Content Inspection - Watermark\",\"recipientAddress\":\"johndoe@example.com\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Undelivered Mail Returned to Sender\"}" + }, + "input": { + "type": "httpjson" + }, + "rule": { + "name": "Content Inspection - Watermark" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "mimecast-dlp-logs" + ] +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.direction | Direction of the message based on the sending and receiving domains | keyword | +| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | +| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | +| email.subject | A brief summary of the topic of the message | keyword | +| email.to.address | The email address(es) of the message recipient(s) | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.action | The action taken against the message. | keyword | +| mimecast.messageId | The message-id value of the message. | keyword | +| mimecast.policy | The name of a DLP or Content Examination configuration that triggered the message. | keyword | +| mimecast.recipientAddress | Email address of the recipient. | keyword | +| mimecast.route | The message direction. Possible values are inbound, outbound or internal. | keyword | +| mimecast.senderAddress | Email address of the sender. | keyword | +| mimecast.subject | The message subject. | keyword | +| rule.name | The name of the rule or signature generating the event. | keyword | +| tags | List of keywords used to tag each event. | keyword | + + +### SIEM LOGS + +This is the `mimecast.siem_logs` dataset. + +An example event for `siem` looks as following: + +```json +{ + "@timestamp": "2021-10-18T08:02:43.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "reason": "Spm", + "action": "Hld", + "ingested": "2021-11-25T11:34:11.459620200Z", + "original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":null,\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Hld\":\"Spm\",\"MsgId\":\"\\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\\u003e\",\"MsgSize\":157436,\"Sender\":\"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu\",\"Subject\":\"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!\",\"aCode\":\"HhuwRf_AOcuJZINE2ZgcKw\",\"acc\":\"ABC123\",\"datetime\":\"2021-10-18T09:02:43+0100\"}", + "created": "2021-10-18T09:02:43+0100", + "outcome": "unknown" + }, + "email": { + "message_id": "\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\u003e", + "from": { + "address": "bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu" + }, + "attachments": { + "file": { + "size": 0 + } + }, + "local_id": "HhuwRf_AOcuJZINE2ZgcKw", + "subject": "Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!", + "message_size": 157436 + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "acc": "ABC123", + "log_type": "process", + "AttCnt": 0 + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | +| email.attachments.file.mime_type | MIME type of the attachment file. | keyword | +| email.attachments.file.name | Name of the attachment file including the extension. | keyword | +| email.attachments.file.size | Attachment file size in bytes. | long | +| email.attachments.hash.md5 | MD5 hash of the file attachment. | keyword | +| email.attachments.hash.sha1 | SHA-1 hash of the file attachment. | keyword | +| email.attachments.hash.sha256 | SHA-256 hash of the file attachment. | keyword | +| email.direction | Direction of the message based on the sending and receiving domains. | keyword | +| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | +| email.header_from | The sender address found in the from header of the email. | keyword | +| email.local_id | Unique identifier given to the email by the source (MTA, gateway, etc.) that created the event and is not persistent across hops (for example, the X-MS-Exchange-Organization-Network-Message-Id id). | keyword | +| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | +| email.message_size | The total size of the email.The total size of the email. | long | +| email.subject | A brief summary of the topic of the message | keyword | +| email.to.address | The email address(es) of the message recipient(s). | keyword | +| error.code | Error code describing the error. | keyword | +| error.message | Error message. | match_only_text | +| error.type | The type of the error, for example the class name of the exception. | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.id | Unique ID to describe the event. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | +| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.AttCnt | The number of attachments on the email. | long | +| mimecast.AttNames | The filenames of all attachments on the email. | keyword | +| mimecast.Attempt | The count of attempts that the Mimecast MTA has made to deliver the email. | long | +| mimecast.CustomName | The message has matched a custom name. | keyword | +| mimecast.CustomThreatDictionary | The content of the email was detected to contain words in a custom threat dictionary. | keyword | +| mimecast.CustomerIP | The source IP is one of the accounts authorised IPs or one of the authorised IPs belonging to an Umbrella Account, if the Account uses an Umbrella Account. | keyword | +| mimecast.Hits | Number of items flagged for the message. | keyword | +| mimecast.IPInternalName | For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detected to be from an internal user name. | keyword | +| mimecast.IPNewDomain | For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detected to be from a new domain. | keyword | +| mimecast.IPReplyMismatch | For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detetced to have a mismatch in the reply to address. | keyword | +| mimecast.IPSimilarDomain | For emails subject to Targeted Threat Protection - Impersonation Protect, if the email was detetced to be from a similar domain to any domain you have registered as an Internal Domain. | keyword | +| mimecast.IPThreadDict | For emails subject to Targeted Threat Protection - Impersonation Protect, if the content of the email was detected to contain words in the Mimecast threat dictionary. | keyword | +| mimecast.InternalName | The email was detected to be from an internal user name. | keyword | +| mimecast.Latency | The time in milliseconds that the delivery attempt took. | long | +| mimecast.MimecastIP | The source IP is one of the Mimecast' IPs e.g. Mimecast Personal Portal. | keyword | +| mimecast.MsgId | The internet message id of the email. | keyword | +| mimecast.MsgSize | The total size of the email. | long | +| mimecast.RcptActType | Action after reception. | keyword | +| mimecast.RcptHdrType | Type of the receipt header. | keyword | +| mimecast.ReceiptAck | The receipt acknowledgment message received by Mimecast from the receiving mail server. | keyword | +| mimecast.ReplyMismatch | The reply address does not correspond to the senders address. | keyword | +| mimecast.ScanResultInfo | The reason that the click was blocked. | keyword | +| mimecast.SenderDomainInternal | The sender domain is a registered internal domain. | keyword | +| mimecast.SimilarCustomExternalDomain | The senders domain is similar to a custom external domain list. | keyword | +| mimecast.SimilarInternalDomain | The senders domain is similar to a registered internal domain. | keyword | +| mimecast.SimilarMimecastExternalDomain | The senders domain is similar to a Mimecast managed list of domains. | keyword | +| mimecast.Snt | The amount of data in bytes that were delivered. | long | +| mimecast.SpamInfo | Information from Mimecast Spam scanners for messages found to be Spam. | keyword | +| mimecast.SpamLimit | The Spam limit defined for the given sender and recipient. | long | +| mimecast.SpamProcessingDetail | The Spam processing details for DKIM, SPF, DMARC. | keyword | +| mimecast.SpamScore | The Spam score the email was given. | long | +| mimecast.Subject | The subject of the email, limited to 150 characters. | keyword | +| mimecast.TaggedExternal | The message has been tagged as originating from a external source. | keyword | +| mimecast.TaggedMalicious | The message has been tagged as malicious. | keyword | +| mimecast.ThreatDictionary | The content of the email was detected to contain words in the Mimecast threat dictionary. | keyword | +| mimecast.UrlCategory | The category of the URL that was clicked. | keyword | +| mimecast.Virus | The name of the virus found on the email, if applicable. | keyword | +| mimecast.acc | The Mimecast account code for your account. | keyword | +| mimecast.credentialTheft | The info about credential theft. | keyword | +| mimecast.log_type | String to get type of SIEM log. | keyword | +| mimecast.urlCategory | The category of the URL that was clicked. | keyword | +| rule.name | The name of the rule or signature generating the event. | keyword | +| source.domain | Source domain. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | +| tls.cipher | String indicating the cipher used during the current connection. | keyword | +| tls.established | Boolean flag indicating if the TLS negotiation was successful and transitioned to an encrypted tunnel. | boolean | +| tls.version | Numeric part of the version parsed from the original string. | keyword | +| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | +| user.email | User email address. | keyword | + + +### TTP IMPERSONATION LOGS + +This is the `mimecast.ttp_ip_logs` dataset. + +An example event for `ttp_ip` looks as following: + +```json +{ + "@timestamp": "2021-10-15T17:10:46.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": "8.8.8.8" + }, + "rule": { + "name": "IP - 1 hit (Tag email)" + }, + "source": { + "ip": "8.8.8.8" + }, + "event": { + "action": "none", + "ingested": "2021-11-19T14:42:59.823940200Z", + "original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG\",\"senderAddress\":\"smtp@example.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Requested File\",\"definition\":\"IP - 1 hit (Tag email)\",\"hits\":1,\"identifiers\":[\"internal_user_name\"],\"action\":\"none\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"8.8.8.8\",\"eventTime\":\"2021-10-15T17:10:46+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe Jr \u003cjohndoejr@example.com\u003e\",\"stringSimilarToDomain\":\"John Doe Jr\",\"checkerResult\":\"hit\"}],\"messageId\":\"\u003cEE7E97EA-1926-4A90-9399-D049A98893F4@emailsec.ninja\u003e\"}", + "id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG", + "created": "2021-10-15T17:10:46+0000" + }, + "email": { + "from": { + "address": "smtp@example.com" + }, + "message_id": "\u003cEE7E97EA-1926-4A90-9399-D049A98893F4@emailsec.ninja\u003e", + "to": { + "address": "johndoe@example.com" + }, + "subject": "Requested File" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "hits": 1, + "impersonationResults": [ + { + "checkerResult": "hit", + "impersonationDomainSource": "internal_user_name", + "stringSimilarToDomain": "John Doe Jr", + "similarDomain": "John Doe Jr \u003cjohndoejr@example.com\u003e" + } + ], + "taggedMalicious": true, + "taggedExternal": false, + "identifiers": [ + "internal_user_name" + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | +| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | +| email.subject | A brief summary of the topic of the message | keyword | +| email.to.address | The email address(es) of the message recipient(s) | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.id | Unique ID to describe the event. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.action | The action triggered by the email. | keyword | +| mimecast.definition | The name of the policy definition that triggered the log. | keyword | +| mimecast.hits | The number of identifiers that the message triggered. | long | +| mimecast.id | A token that can be used to retrieve this log again. | keyword | +| mimecast.identifiers | The properties of the message that triggered the action - similar_internal_domain, newly_observed_domain, internal_user_name, reply_address_mismatch, and/or targeted_threat_dictionary. | keyword | +| mimecast.impersonationResults.checkerResult | Result checker. | keyword | +| mimecast.impersonationResults.impersonationDomainSource | Impersonation domain source. | keyword | +| mimecast.impersonationResults.similarDomain | Similar domain. | keyword | +| mimecast.impersonationResults.stringSimilarToDomain | The string that is suspiciously similar to a known value within the Mimecast configuration. Multiple triggers will be comma-separated. | keyword | +| mimecast.messageId | The message-id of the identified message. | keyword | +| mimecast.recipientAddress | The email address of the recipient of the email. | keyword | +| mimecast.senderAddress | The email address of the sender of the message. | keyword | +| mimecast.senderIpAddress | The source IP address of the message. | keyword | +| mimecast.subject | The subject of the email. | keyword | +| mimecast.taggedExternal | Whether the message was tagged as coming from an external address. | boolean | +| mimecast.taggedMalicious | Whether the message was tagged as malicious. | boolean | +| related.ip | All of the IPs seen on your event. | ip | +| rule.name | The name of the rule or signature generating the event. | keyword | +| source.domain | Source domain. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | + + +### TTP ATTACHMENT LOGS + +This is the `mimecast.ttp_ap_logs` dataset. + +An example event for `ttp_ap` looks as following: + +```json +{ + "@timestamp": "2021-10-14T18:54:32.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hash": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3" + }, + "rule": { + "name": "Inbound - Safe file with On-Demand Sandbox" + }, + "event": { + "action": "user_release_none", + "ingested": "2021-11-19T14:40:07.263592900Z", + "original": "{\"senderAddress\":\"\u003c\u003e\",\"recipientAddress\":\"johndoe@example.com\",\"fileName\":\"numbers.pdf\",\"fileType\":\"application\\/pdf\",\"result\":\"safe\",\"actionTriggered\":\"user release, none\",\"date\":\"2021-10-14T18:54:32+0000\",\"details\":\"Safe \\r\\nTime taken: 0 hrs, 0 min, 4 sec\",\"route\":\"inbound\",\"messageId\":\"\u003c20200806044148.F35F813B435@mail.brianjthronton.com\u003e\",\"subject\":\"Important Updated Numbers from the Center for Disease Control\",\"fileHash\":\"eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3\",\"definition\":\"Inbound - Safe file with On-Demand Sandbox\"}", + "created": "2021-10-14T18:54:32+0000" + }, + "email": { + "from": { + "address": "\u003c\u003e" + }, + "message_id": "\u003c20200806044148.F35F813B435@mail.brianjthronton.com\u003e", + "attachments": { + "hash": "eaeef09b60a59b913e9bfc0a4373e25d6182beff388957473fba517cc09345e3", + "file": { + "name": "numbers.pdf", + "mime_type": "application/pdf", + "extension": "pdf" + } + }, + "to": { + "address": "johndoe@example.com" + }, + "subject": "Important Updated Numbers from the Center for Disease Control", + "direction": "inbound" + }, + "tags": [ + "preserve_original_event" + ], + "mimecast": { + "result": "safe", + "details": "Safe \r\nTime taken: 0 hrs, 0 min, 4 sec" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | +| email.attachments.file.mime_type | MIME type of the attachment file. | keyword | +| email.attachments.file.name | Name of the attachment file including the extension. | keyword | +| email.attachments.hash | File hash. | keyword | +| email.direction | Direction of the message based on the sending and receiving domains | keyword | +| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | +| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | +| email.subject | A brief summary of the topic of the message | keyword | +| email.to.address | The email address(es) of the message recipient(s) | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.actionTriggered | The action triggered for the attachment. | keyword | +| mimecast.definition | The definition. | keyword | +| mimecast.details | Detailed output of the attachment sandbox processing. | keyword | +| mimecast.fileHash | The hash of the attachment. | keyword | +| mimecast.fileName | The file name of the original attachment. | keyword | +| mimecast.fileType | The file type of the attachment. | keyword | +| mimecast.messageId | The internet message id of the email. | keyword | +| mimecast.recipientAddress | The address of the user that received the attachment. | keyword | +| mimecast.result | The result of the attachment analysis - clean, malicious, unknown, or timeout. | keyword | +| mimecast.route | The route of the original email containing the attachment, either - inbound, outbound, internal, or external. | keyword | +| mimecast.senderAddress | The sender of the attachment. | keyword | +| mimecast.subject | The subject of the email. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| rule.name | The name of the rule or signature generating the event. | keyword | +| tags | List of keywords used to tag each event. | keyword | + + +### TTP URL LOGS + +This is the `mimecast.ttp_url_logs` dataset. + +An example event for `ttp_url` looks as following: + +```json +{ + "rule": { + "name": "Inbound URL 'Aggressive'" + }, + "source": { + "ip": "8.8.8.8" + }, + "url": { + "original": "https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2021-10-16T14:45:34.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "johndoe", + "johndoe@example.com" + ], + "ip": [ + "8.8.8.8" + ] + }, + "event": { + "action": "Continue", + "ingested": "2021-11-24T14:39:10.084705200Z", + "original": "{\"userEmailAddress\": \"johndoe@example.com\", \"fromUserEmailAddress\": \"bestbuyinfo@emailinfo.bestbuy.com\", \"url\": \"https://click.emailinfo2.bestbuy.com/?qs=5c47c91aeb44fac857370c26ddf09c3f484431e1ccfa636fc64e26e40dd87efdb43d4deeeab8c2e727ebfa079e8cf1404c095c511152e4b09e7d00bf8377f32d\", \"ttpDefinition\": \"Inbound URL 'Aggressive'\", \"subject\": \"Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.\", \"action\": \"allow\", \"adminOverride\": \"N/A\", \"userOverride\": \"None\", \"scanResult\": \"clean\", \"category\": \"Business\", \"sendingIp\": \"8.8.8.8\", \"userAwarenessAction\": \"Continue\", \"date\": \"2021-10-16T14:45:34+0000\", \"actions\": \"Allow\", \"route\": \"inbound\", \"creationMethod\": \"User Click\", \"emailPartsDescription\": [ \"Body\" ], \"messageId\": \"\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e\" }", + "created": "2021-10-16T14:45:34+0000" + }, + "user": { + "name": "johndoe", + "email": "johndoe@example.com", + "domain": "example.com" + }, + "email": { + "message_id": "\u003c31b43097-94f9-4f64-8e37-8ad23650c692@ind1s01mta1292.xt.local\u003e", + "from": { + "address": "bestbuyinfo@emailinfo.bestbuy.com" + }, + "subject": "Today only: Save $100 on Tineco Pure One S12 smart cordless stick vacuum, plus more.", + "direction": "inbound" + }, + "mimecast": { + "userOverride": "None", + "action": "allow", + "adminOverride": "N/A", + "scanResult": "clean", + "category": "Business", + "actions": "Allow", + "creationMethod": "User Click", + "emailPartsDescription": [ + "Body" + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.direction | Direction of the message based on the sending and receiving domains | keyword | +| email.from.address | Stores the from email address from the RFC5322 From - header field. | keyword | +| email.message_id | Identifier from the RFC5322 Message-ID - header field that refers to a particular version of a particular message. | wildcard | +| email.subject | A brief summary of the topic of the message | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.action | The action that was taken for the click. | keyword | +| mimecast.actions | The actions that were taken. | keyword | +| mimecast.adminOverride | The action defined by the administrator for the URL. | keyword | +| mimecast.category | The category of the URL clicked. | keyword | +| mimecast.creationMethod | The description how event occurred. | keyword | +| mimecast.emailPartsDescription | An array of components of the messge where the URL was found. | keyword | +| mimecast.fromUserEmailAddress | The email of user who triggers the event. | keyword | +| mimecast.messageId | The message-id value of the message. | keyword | +| mimecast.route | The route of the email that contained the link. | keyword | +| mimecast.scanResult | The result of the URL scan. | keyword | +| mimecast.sendingIP | The IP of user who triggers the event. | keyword | +| mimecast.subject | The subject of the email. | keyword | +| mimecast.ttpDefinition | The description of the definition that triggered the URL to be rewritten by Mimecast. | keyword | +| mimecast.url | The url clicked. | keyword | +| mimecast.userAwarenessAction | The action taken by the user if user awareness was applied. | keyword | +| mimecast.userEmailAddress | The email address of the user who clicked the link. | keyword | +| mimecast.userOverride | The action requested by the user. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| related.user | All the user names or other user identifiers seen on the event. | keyword | +| rule.name | The name of the rule or signature generating the event. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | +| user.email | User email address. | keyword | +| user.name | Short name or login of the user. | keyword | + + +### THREAT INTEL FEED MALWARE CUSTOMER + +This is the `mimecast.threat_intel_malware_customer` dataset. + +An example event for `threat_intel_malware_customer` looks as following: + +```json +{ + "@timestamp": "2021-10-29T15:07:26.653Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:26.653Z", + "file": { + "hash": { + "sha256": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + } + }, + "modified_at": "2021-10-29T15:07:26.653Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-17T13:42:34.324885300Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_customer_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--18c62174-0d31-4653-afe6-d104c57b6b2c\", \"created\": \"2021-10-29T15:07:26.653Z\", \"modified\": \"2021-10-29T15:07:26.653Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']\", \"valid_from\": \"2021-10-29T15:07:26.653Z\" }", + "category": "threat", + "type": "indicator", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", + "log_type": "malware_customer", + "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| mimecast.created | When the indicator was last created. | date | +| mimecast.hashtype | The hash type. | keyword | +| mimecast.id | The ID of the indicator. | keyword | +| mimecast.labels | The labels related to the indicator. | keyword | +| mimecast.log_type | String to get type of Threat intel feed. | keyword | +| mimecast.modified | When the indicator was last modified. | date | +| mimecast.name | Name of the file. | keyword | +| mimecast.pattern | The pattern. | keyword | +| mimecast.relationship_type | Type of the relationship. | keyword | +| mimecast.source_ref | Source of the reference. | keyword | +| mimecast.target_ref | Reference target. | keyword | +| mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | +| mimecast.valid_from | The valid from date. | date | +| mimecast.value | The value of the indicator. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| tags | List of keywords used to tag each event. | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | + + +### THREAT INTEL FEED MALWARE GRID + +This is the `mimecast.threat_intel_malware_grid` dataset. + +An example event for `threat_intel_malware_grid` looks as following: + +```json +{ + "@timestamp": "2021-10-29T15:07:26.653Z", + "ecs": { + "version": "1.12" + }, + "related": { + "hash": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + }, + "threat": { + "indicator": { + "first_seen": "2021-10-29T15:07:26.653Z", + "file": { + "hash": { + "sha256": "c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de" + } + }, + "modified_at": "2021-10-29T15:07:26.653Z", + "type": "file" + } + }, + "event": { + "ingested": "2021-11-17T13:42:35.248902200Z", + "original": "{ \"Content-Disposition\":\"attachment; filename=\\\"malware_grid_stix_20211028161801144.stix\\\"\",\"type\": \"indicator\", \"id\": \"indicator--18c62174-0d31-4653-afe6-d104c57b6b2c\", \"created\": \"2021-10-29T15:07:26.653Z\", \"modified\": \"2021-10-29T15:07:26.653Z\", \"labels\": [ \"malicious-activity\" ], \"pattern\": \"[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']\", \"valid_from\": \"2021-10-29T15:07:26.653Z\" }", + "category": "threat", + "kind": "enrichment" + }, + "tags": [ + "preserve_original_event", + "malicious-activity" + ], + "mimecast": { + "pattern": "[file:hashes.'SHA-256' = 'c20d551424f2df6312f7fa700ed97cd199c3d5c8a0f4dfd683627f18913096de']", + "log_type": "malware_grid", + "id": "indicator--18c62174-0d31-4653-afe6-d104c57b6b2c", + "type": "indicator", + "labels": [ + "malicious-activity" + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| mimecast.created | When the indicator was last created. | date | +| mimecast.hashtype | The hash type. | keyword | +| mimecast.id | The ID of the indicator. | keyword | +| mimecast.labels | The labels related to the indicator. | keyword | +| mimecast.log_type | String to get type of Threat intel feed. | keyword | +| mimecast.modified | When the indicator was last modified. | date | +| mimecast.name | Name of the file. | keyword | +| mimecast.pattern | The pattern. | keyword | +| mimecast.relationship_type | Type of the relationship. | keyword | +| mimecast.source_ref | Source of the reference. | keyword | +| mimecast.target_ref | Reference target. | keyword | +| mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | +| mimecast.valid_from | The valid from date. | date | +| mimecast.value | The value of the indicator. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| tags | List of keywords used to tag each event. | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | diff --git a/packages/mimecast/img/sample-logo.svg b/packages/mimecast/img/sample-logo.svg new file mode 100644 index 00000000000..6268dd88f3b --- /dev/null +++ b/packages/mimecast/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/mimecast/img/sample-screenshot.png b/packages/mimecast/img/sample-screenshot.png new file mode 100644 index 00000000000..d7a56a3ecc0 Binary files /dev/null and b/packages/mimecast/img/sample-screenshot.png differ diff --git a/packages/mimecast/kibana/dashboard/mimecast-061eb320-3e4a-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/dashboard/mimecast-061eb320-3e4a-11ec-80fa-4dfb04910642.json new file mode 100644 index 00000000000..43ac605831d --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-061eb320-3e4a-11ec-80fa-4dfb04910642.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-55f1e965-a3d5-4941-820e-46277d3f3cba", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "55f1e965-a3d5-4941-820e-46277d3f3cba": { + "columnOrder": [ + "2984698c-20fb-4eca-975b-a42fcb4136a4", + "839e65a6-2bfb-4b3a-aa86-044a081338bf" + ], + "columns": { + "2984698c-20fb-4eca-975b-a42fcb4136a4": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of email.from", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "839e65a6-2bfb-4b3a-aa86-044a081338bf", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "email.from" + }, + "839e65a6-2bfb-4b3a-aa86-044a081338bf": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ap_logs\" and mimecast.result : \"malicious\" " + }, + "visualization": { + "columns": [ + { + "columnId": "2984698c-20fb-4eca-975b-a42fcb4136a4" + }, + { + "columnId": "839e65a6-2bfb-4b3a-aa86-044a081338bf" + } + ], + "layerId": "55f1e965-a3d5-4941-820e-46277d3f3cba", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "48b9eff5-a32e-457a-a741-55e072d516c7", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "48b9eff5-a32e-457a-a741-55e072d516c7", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Attachment threats detected by sender", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-061eb320-3e4a-11ec-80fa-4dfb04910642", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "48b9eff5-a32e-457a-a741-55e072d516c7:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "48b9eff5-a32e-457a-a741-55e072d516c7:indexpattern-datasource-layer-55f1e965-a3d5-4941-820e-46277d3f3cba", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-13366440-3e45-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/dashboard/mimecast-13366440-3e45-11ec-80fa-4dfb04910642.json new file mode 100644 index 00000000000..afdaed8d2ba --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-13366440-3e45-11ec-80fa-4dfb04910642.json @@ -0,0 +1,142 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-cc0ca8f3-6cdf-46d7-a3a8-88a1818b2340", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "cc0ca8f3-6cdf-46d7-a3a8-88a1818b2340": { + "columnOrder": [ + "ff48f1ba-4593-40a2-88f0-a317519f65a0", + "379f2d4d-5cdb-495b-866b-a67eb523bd86" + ], + "columns": { + "379f2d4d-5cdb-495b-866b-a67eb523bd86": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "ff48f1ba-4593-40a2-88f0-a317519f65a0": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Top potencial malicious senders", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "379f2d4d-5cdb-495b-866b-a67eb523bd86", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "email.from" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ip_logs\" and mimecast.taggedMalicious : true " + }, + "visualization": { + "columns": [ + { + "columnId": "ff48f1ba-4593-40a2-88f0-a317519f65a0" + }, + { + "columnId": "379f2d4d-5cdb-495b-866b-a67eb523bd86" + } + ], + "layerId": "cc0ca8f3-6cdf-46d7-a3a8-88a1818b2340", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "efad42a4-2002-470a-9cf0-19cc930a633b", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "efad42a4-2002-470a-9cf0-19cc930a633b", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "TopMaliciousSenders", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-13366440-3e45-11ec-80fa-4dfb04910642", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "efad42a4-2002-470a-9cf0-19cc930a633b:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "efad42a4-2002-470a-9cf0-19cc930a633b:indexpattern-datasource-layer-cc0ca8f3-6cdf-46d7-a3a8-88a1818b2340", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-1a4e8460-3fd5-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-1a4e8460-3fd5-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..65ef8be840f --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-1a4e8460-3fd5-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,116 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-3732d54a-b698-4a66-baef-5d0674eff6c9", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "3732d54a-b698-4a66-baef-5d0674eff6c9": { + "columnOrder": [ + "eaf6d751-71b7-431a-b597-6f58857c0ea9" + ], + "columns": { + "eaf6d751-71b7-431a-b597-6f58857c0ea9": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "users logged on", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.audit_events\" and event.action : \"user-logged-on\" " + }, + "visualization": { + "accessor": "eaf6d751-71b7-431a-b597-6f58857c0ea9", + "layerId": "3732d54a-b698-4a66-baef-5d0674eff6c9", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "3012e9b4-838e-4129-b472-711cdfd19cfd", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "3012e9b4-838e-4129-b472-711cdfd19cfd", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Count of successful logins", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-1a4e8460-3fd5-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "3012e9b4-838e-4129-b472-711cdfd19cfd:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "3012e9b4-838e-4129-b472-711cdfd19cfd:indexpattern-datasource-layer-3732d54a-b698-4a66-baef-5d0674eff6c9", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-1d771e30-3f1d-11ec-9edf-13c963822dec.json b/packages/mimecast/kibana/dashboard/mimecast-1d771e30-3f1d-11ec-9edf-13c963822dec.json new file mode 100644 index 00000000000..21396e89264 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-1d771e30-3f1d-11ec-9edf-13c963822dec.json @@ -0,0 +1,142 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-0fff056b-7794-4070-8170-3657002b9253", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "0fff056b-7794-4070-8170-3657002b9253": { + "columnOrder": [ + "e4eb146d-7546-4a24-ae35-eb2824b345a2", + "c9c6ab54-8f0d-49b4-bf62-33f88decd52c" + ], + "columns": { + "c9c6ab54-8f0d-49b4-bf62-33f88decd52c": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "e4eb146d-7546-4a24-ae35-eb2824b345a2": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Actions", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "c9c6ab54-8f0d-49b4-bf62-33f88decd52c", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "event.action" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.dlp_logs\"" + }, + "visualization": { + "columns": [ + { + "columnId": "e4eb146d-7546-4a24-ae35-eb2824b345a2" + }, + { + "columnId": "c9c6ab54-8f0d-49b4-bf62-33f88decd52c" + } + ], + "layerId": "0fff056b-7794-4070-8170-3657002b9253", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "a5dfdd0f-a1a5-46f4-b9e3-e28cd76ce371", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "a5dfdd0f-a1a5-46f4-b9e3-e28cd76ce371", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "DLP Logs - Actions taken on message", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-1d771e30-3f1d-11ec-9edf-13c963822dec", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "a5dfdd0f-a1a5-46f4-b9e3-e28cd76ce371:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "a5dfdd0f-a1a5-46f4-b9e3-e28cd76ce371:indexpattern-datasource-layer-0fff056b-7794-4070-8170-3657002b9253", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-222caa50-3e49-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/dashboard/mimecast-222caa50-3e49-11ec-80fa-4dfb04910642.json new file mode 100644 index 00000000000..1e86ea090cb --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-222caa50-3e49-11ec-80fa-4dfb04910642.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-cc987f4b-7570-4117-a216-abb8b85d6a74", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "cc987f4b-7570-4117-a216-abb8b85d6a74": { + "columnOrder": [ + "68fb7687-4b9e-4269-9514-d871fd23acf6", + "accab1cb-cf0c-4e6c-94c6-cc50396d0d58" + ], + "columns": { + "68fb7687-4b9e-4269-9514-d871fd23acf6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Malicious files extensions", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "accab1cb-cf0c-4e6c-94c6-cc50396d0d58", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "email.attachments.file.extension" + }, + "accab1cb-cf0c-4e6c-94c6-cc50396d0d58": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ap_logs\" and mimecast.result : \"malicious\" " + }, + "visualization": { + "columns": [ + { + "columnId": "68fb7687-4b9e-4269-9514-d871fd23acf6", + "isTransposed": false + }, + { + "columnId": "accab1cb-cf0c-4e6c-94c6-cc50396d0d58", + "isTransposed": false + } + ], + "layerId": "cc987f4b-7570-4117-a216-abb8b85d6a74", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "6da6b60b-bb44-42f0-8432-bb993575a078", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "6da6b60b-bb44-42f0-8432-bb993575a078", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Malicious attachments file extensions", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-222caa50-3e49-11ec-80fa-4dfb04910642", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "6da6b60b-bb44-42f0-8432-bb993575a078:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "6da6b60b-bb44-42f0-8432-bb993575a078:indexpattern-datasource-layer-cc987f4b-7570-4117-a216-abb8b85d6a74", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-2c81bc30-3e48-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/dashboard/mimecast-2c81bc30-3e48-11ec-80fa-4dfb04910642.json new file mode 100644 index 00000000000..4627edaa2c2 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-2c81bc30-3e48-11ec-80fa-4dfb04910642.json @@ -0,0 +1,172 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-7fd2fb45-58d3-499c-8b39-a65a1d337c30", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "7fd2fb45-58d3-499c-8b39-a65a1d337c30": { + "columnOrder": [ + "4c2264ac-1102-43db-b405-02295ddba570", + "29a6d63f-6b9e-42f5-a062-026e264b7905" + ], + "columns": { + "29a6d63f-6b9e-42f5-a062-026e264b7905": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "4c2264ac-1102-43db-b405-02295ddba570": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1w" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ap_logs\" and mimecast.result : \"malicious\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "29a6d63f-6b9e-42f5-a062-026e264b7905" + ], + "layerId": "7fd2fb45-58d3-499c-8b39-a65a1d337c30", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "4c2264ac-1102-43db-b405-02295ddba570" + } + ], + "legend": { + "isInside": false, + "isVisible": true, + "position": "right", + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "7845513e-a6db-4a67-b334-1f14a77fe25d", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "7845513e-a6db-4a67-b334-1f14a77fe25d", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Threats tagged malicious over time", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-2c81bc30-3e48-11ec-80fa-4dfb04910642", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "7845513e-a6db-4a67-b334-1f14a77fe25d:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "7845513e-a6db-4a67-b334-1f14a77fe25d:indexpattern-datasource-layer-7fd2fb45-58d3-499c-8b39-a65a1d337c30", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-44778db0-3e34-11ec-a4d8-6b18148fbf3a.json b/packages/mimecast/kibana/dashboard/mimecast-44778db0-3e34-11ec-a4d8-6b18148fbf3a.json new file mode 100644 index 00000000000..36cb36f20a1 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-44778db0-3e34-11ec-a4d8-6b18148fbf3a.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-8a4f8003-e917-44ab-9b50-c46553bacd59", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "8a4f8003-e917-44ab-9b50-c46553bacd59": { + "columnOrder": [ + "aaa283a2-4c24-432c-b7f3-a3304e800b51", + "826ba46a-7476-493d-a256-c717d69e7d2b" + ], + "columns": { + "826ba46a-7476-493d-a256-c717d69e7d2b": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "aaa283a2-4c24-432c-b7f3-a3304e800b51": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Held Reasons", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "826ba46a-7476-493d-a256-c717d69e7d2b", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "event.reason" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.siem_logs\" and mimecast.log_type : \"process\" and event.action : \"Hld\"" + }, + "visualization": { + "columns": [ + { + "columnId": "aaa283a2-4c24-432c-b7f3-a3304e800b51", + "isTransposed": false + }, + { + "columnId": "826ba46a-7476-493d-a256-c717d69e7d2b", + "isTransposed": false + } + ], + "layerId": "8a4f8003-e917-44ab-9b50-c46553bacd59", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "5b323448-f2b1-4612-bd94-8a84757ecd07", + "w": 47, + "x": 0, + "y": 0 + }, + "panelIndex": "5b323448-f2b1-4612-bd94-8a84757ecd07", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Held Reasons", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-44778db0-3e34-11ec-a4d8-6b18148fbf3a", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "5b323448-f2b1-4612-bd94-8a84757ecd07:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "5b323448-f2b1-4612-bd94-8a84757ecd07:indexpattern-datasource-layer-8a4f8003-e917-44ab-9b50-c46553bacd59", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-4f4f23d0-3fe5-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-4f4f23d0-3fe5-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..63813a67b08 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-4f4f23d0-3fe5-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_url_logs\"" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-76a01545-a0d3-4529-9185-e99aa33aa198", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "76a01545-a0d3-4529-9185-e99aa33aa198": { + "columnOrder": [ + "0f3030c5-e2c2-46b0-94d9-9fedf71bbedd", + "1e318351-5ec1-484c-8a9f-dd79a8c26759" + ], + "columns": { + "0f3030c5-e2c2-46b0-94d9-9fedf71bbedd": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "url", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "1e318351-5ec1-484c-8a9f-dd79a8c26759", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "url.original" + }, + "1e318351-5ec1-484c-8a9f-dd79a8c26759": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "count", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_url_logs\" and mimecast.scanResult : \"malicious\" " + }, + "visualization": { + "columns": [ + { + "columnId": "0f3030c5-e2c2-46b0-94d9-9fedf71bbedd", + "isTransposed": false + }, + { + "columnId": "1e318351-5ec1-484c-8a9f-dd79a8c26759", + "isTransposed": false + } + ], + "layerId": "76a01545-a0d3-4529-9185-e99aa33aa198", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "6a6d5d4f-8bf4-47ad-ab00-6752ea17abde", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "6a6d5d4f-8bf4-47ad-ab00-6752ea17abde", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Top malicious URLs", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-4f4f23d0-3fe5-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "6a6d5d4f-8bf4-47ad-ab00-6752ea17abde:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "6a6d5d4f-8bf4-47ad-ab00-6752ea17abde:indexpattern-datasource-layer-76a01545-a0d3-4529-9185-e99aa33aa198", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-50bfa050-4080-11ec-b8da-95c3fba730d0.json b/packages/mimecast/kibana/dashboard/mimecast-50bfa050-4080-11ec-b8da-95c3fba730d0.json new file mode 100644 index 00000000000..dce71b69bdf --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-50bfa050-4080-11ec-b8da-95c3fba730d0.json @@ -0,0 +1,143 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-231039d5-8ca6-4e3d-b6ce-304ff967550c", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "231039d5-8ca6-4e3d-b6ce-304ff967550c": { + "columnOrder": [ + "e751fb41-0eb0-444c-858b-b2ffafe590cf", + "b642290b-f2dd-46a6-8641-ef25b6e6e794" + ], + "columns": { + "b642290b-f2dd-46a6-8641-ef25b6e6e794": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "count", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "e751fb41-0eb0-444c-858b-b2ffafe590cf": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Indicator", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "b642290b-f2dd-46a6-8641-ef25b6e6e794", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.type" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.threat_intel_malware_grid\" " + }, + "visualization": { + "columns": [ + { + "columnId": "e751fb41-0eb0-444c-858b-b2ffafe590cf" + }, + { + "columnId": "b642290b-f2dd-46a6-8641-ef25b6e6e794" + } + ], + "layerId": "231039d5-8ca6-4e3d-b6ce-304ff967550c", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "acbf0708-2dbc-416a-959d-b9bce6c5f48a", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "acbf0708-2dbc-416a-959d-b9bce6c5f48a", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "Threat Intel - Malware Grid by indicator type", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-50bfa050-4080-11ec-b8da-95c3fba730d0", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "acbf0708-2dbc-416a-959d-b9bce6c5f48a:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "acbf0708-2dbc-416a-959d-b9bce6c5f48a:indexpattern-datasource-layer-231039d5-8ca6-4e3d-b6ce-304ff967550c", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-53cd8660-3fed-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-53cd8660-3fed-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..b692ff834ff --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-53cd8660-3fed-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,143 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-2a0ae18b-3b74-4c61-8a14-3f87a634e8ba", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "2a0ae18b-3b74-4c61-8a14-3f87a634e8ba": { + "columnOrder": [ + "2b26e9ef-78d9-4173-97fa-ec7526af0773", + "2782be47-0178-4935-ac5b-05c8a15a61f2" + ], + "columns": { + "2782be47-0178-4935-ac5b-05c8a15a61f2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "count", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "2b26e9ef-78d9-4173-97fa-ec7526af0773": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "category", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "2782be47-0178-4935-ac5b-05c8a15a61f2", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "mimecast.category" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_url_logs\" " + }, + "visualization": { + "columns": [ + { + "columnId": "2b26e9ef-78d9-4173-97fa-ec7526af0773" + }, + { + "columnId": "2782be47-0178-4935-ac5b-05c8a15a61f2" + } + ], + "layerId": "2a0ae18b-3b74-4c61-8a14-3f87a634e8ba", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "d5155a39-77f0-42ea-9051-8901b8c43a7f", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "d5155a39-77f0-42ea-9051-8901b8c43a7f", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "TTP URL top categories", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-53cd8660-3fed-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "d5155a39-77f0-42ea-9051-8901b8c43a7f:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "d5155a39-77f0-42ea-9051-8901b8c43a7f:indexpattern-datasource-layer-2a0ae18b-3b74-4c61-8a14-3f87a634e8ba", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-57b10a70-3e32-11ec-a4d8-6b18148fbf3a.json b/packages/mimecast/kibana/dashboard/mimecast-57b10a70-3e32-11ec-a4d8-6b18148fbf3a.json new file mode 100644 index 00000000000..ffb99e26d8c --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-57b10a70-3e32-11ec-a4d8-6b18148fbf3a.json @@ -0,0 +1,190 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.siem_logs\" " + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-1faf17aa-0298-4830-a031-00f1b48435b6", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "1faf17aa-0298-4830-a031-00f1b48435b6": { + "columnOrder": [ + "95cdbe62-23e4-43ee-9bab-123bfc4a3e68", + "c9f7cf64-8a98-4e3c-b12c-a22d26ca20be", + "2611cbf0-c905-44cc-a98e-25fbdcd5dbee" + ], + "columns": { + "2611cbf0-c905-44cc-a98e-25fbdcd5dbee": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "95cdbe62-23e4-43ee-9bab-123bfc4a3e68": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1M" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "c9f7cf64-8a98-4e3c-b12c-a22d26ca20be": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of email.direction", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "2611cbf0-c905-44cc-a98e-25fbdcd5dbee", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 4 + }, + "scale": "ordinal", + "sourceField": "email.direction" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.siem_logs\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "2611cbf0-c905-44cc-a98e-25fbdcd5dbee" + ], + "layerId": "1faf17aa-0298-4830-a031-00f1b48435b6", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "c9f7cf64-8a98-4e3c-b12c-a22d26ca20be", + "xAccessor": "95cdbe62-23e4-43ee-9bab-123bfc4a3e68" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "0ea3ece9-36fc-40d8-974b-19207c25d614", + "w": 46, + "x": 0, + "y": 0 + }, + "panelIndex": "0ea3ece9-36fc-40d8-974b-19207c25d614", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Email Activity Summary", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-57b10a70-3e32-11ec-a4d8-6b18148fbf3a", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "0ea3ece9-36fc-40d8-974b-19207c25d614:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "0ea3ece9-36fc-40d8-974b-19207c25d614:indexpattern-datasource-layer-1faf17aa-0298-4830-a031-00f1b48435b6", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-68b50ce0-3f1d-11ec-9edf-13c963822dec.json b/packages/mimecast/kibana/dashboard/mimecast-68b50ce0-3f1d-11ec-9edf-13c963822dec.json new file mode 100644 index 00000000000..e202b74224a --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-68b50ce0-3f1d-11ec-9edf-13c963822dec.json @@ -0,0 +1,142 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-854e5002-cd2e-466a-ba28-04e926663f66", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "854e5002-cd2e-466a-ba28-04e926663f66": { + "columnOrder": [ + "5745adf7-04d2-4886-8dad-897d57705772", + "b9e528af-178d-488b-8997-fbaf60f2e4aa" + ], + "columns": { + "5745adf7-04d2-4886-8dad-897d57705772": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Policies", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "b9e528af-178d-488b-8997-fbaf60f2e4aa", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "rule.name" + }, + "b9e528af-178d-488b-8997-fbaf60f2e4aa": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.dlp_logs\"" + }, + "visualization": { + "columns": [ + { + "columnId": "5745adf7-04d2-4886-8dad-897d57705772" + }, + { + "columnId": "b9e528af-178d-488b-8997-fbaf60f2e4aa" + } + ], + "layerId": "854e5002-cd2e-466a-ba28-04e926663f66", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "00417ad0-0944-408d-9646-ebcf1a43e3ff", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "00417ad0-0944-408d-9646-ebcf1a43e3ff", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "DLP Logs - Policies Triggered", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-68b50ce0-3f1d-11ec-9edf-13c963822dec", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "00417ad0-0944-408d-9646-ebcf1a43e3ff:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "00417ad0-0944-408d-9646-ebcf1a43e3ff:indexpattern-datasource-layer-854e5002-cd2e-466a-ba28-04e926663f66", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-68d3cfa0-3e45-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/dashboard/mimecast-68d3cfa0-3e45-11ec-80fa-4dfb04910642.json new file mode 100644 index 00000000000..a8ef85d5640 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-68d3cfa0-3e45-11ec-80fa-4dfb04910642.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ip_logs\" and mimecast.taggedMalicious : true " + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-ab543c4a-7b11-40f3-bca3-74ea65af48f4", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "ab543c4a-7b11-40f3-bca3-74ea65af48f4": { + "columnOrder": [ + "e4e885a4-eebd-48b5-bf7a-1c8acf4553fa", + "c09ef631-df6f-4df9-b8c2-9fa883d711e8" + ], + "columns": { + "c09ef631-df6f-4df9-b8c2-9fa883d711e8": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "e4e885a4-eebd-48b5-bf7a-1c8acf4553fa": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Top potencial malicious recipients", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "c09ef631-df6f-4df9-b8c2-9fa883d711e8", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "email.to" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ip_logs\" and mimecast.taggedMalicious : true " + }, + "visualization": { + "columns": [ + { + "columnId": "e4e885a4-eebd-48b5-bf7a-1c8acf4553fa", + "isTransposed": false + }, + { + "columnId": "c09ef631-df6f-4df9-b8c2-9fa883d711e8", + "isTransposed": false + } + ], + "layerId": "ab543c4a-7b11-40f3-bca3-74ea65af48f4", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "c120fe23-ca11-4ec3-b0f4-b69db5bb84f2", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "c120fe23-ca11-4ec3-b0f4-b69db5bb84f2", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "TopPotencialMaliciousRecipients", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-68d3cfa0-3e45-11ec-80fa-4dfb04910642", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "c120fe23-ca11-4ec3-b0f4-b69db5bb84f2:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c120fe23-ca11-4ec3-b0f4-b69db5bb84f2:indexpattern-datasource-layer-ab543c4a-7b11-40f3-bca3-74ea65af48f4", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-88cbbc00-3fd5-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-88cbbc00-3fd5-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..c98381a813b --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-88cbbc00-3fd5-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,116 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-d1772930-cd84-4843-ad0d-64b5bf4d1e9c", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "d1772930-cd84-4843-ad0d-64b5bf4d1e9c": { + "columnOrder": [ + "4abe2c7c-88ea-4177-8ea9-aaa8f34bc902" + ], + "columns": { + "4abe2c7c-88ea-4177-8ea9-aaa8f34bc902": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "login failed attempts", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.audit_events\" and event.action : \"logon-authentication-failed\" " + }, + "visualization": { + "accessor": "4abe2c7c-88ea-4177-8ea9-aaa8f34bc902", + "layerId": "d1772930-cd84-4843-ad0d-64b5bf4d1e9c", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "69856c61-642f-4d2e-b404-05e3260eebbd", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "69856c61-642f-4d2e-b404-05e3260eebbd", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Count of failed login attemots", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-88cbbc00-3fd5-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "69856c61-642f-4d2e-b404-05e3260eebbd:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "69856c61-642f-4d2e-b404-05e3260eebbd:indexpattern-datasource-layer-d1772930-cd84-4843-ad0d-64b5bf4d1e9c", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-a2f56a50-3e49-11ec-80fa-4dfb04910642.json b/packages/mimecast/kibana/dashboard/mimecast-a2f56a50-3e49-11ec-80fa-4dfb04910642.json new file mode 100644 index 00000000000..dfcc2e23279 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-a2f56a50-3e49-11ec-80fa-4dfb04910642.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-675873f9-5e65-4f7d-a731-1e5170a98700", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "675873f9-5e65-4f7d-a731-1e5170a98700": { + "columnOrder": [ + "a413b181-ad13-4316-97ad-f563a54dd33d", + "757fdc1e-7a28-470c-a730-e3b9a67ec253" + ], + "columns": { + "757fdc1e-7a28-470c-a730-e3b9a67ec253": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "a413b181-ad13-4316-97ad-f563a54dd33d": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Threats detected by recipients", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "757fdc1e-7a28-470c-a730-e3b9a67ec253", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "email.to" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_ap_logs\" and mimecast.result : \"malicious\" " + }, + "visualization": { + "columns": [ + { + "columnId": "a413b181-ad13-4316-97ad-f563a54dd33d", + "isTransposed": false + }, + { + "columnId": "757fdc1e-7a28-470c-a730-e3b9a67ec253", + "isTransposed": false + } + ], + "layerId": "675873f9-5e65-4f7d-a731-1e5170a98700", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "452363c8-218d-4c2a-ab84-6de4e244016f", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "452363c8-218d-4c2a-ab84-6de4e244016f", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Attachemnt threats detected by recipients", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-a2f56a50-3e49-11ec-80fa-4dfb04910642", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "452363c8-218d-4c2a-ab84-6de4e244016f:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "452363c8-218d-4c2a-ab84-6de4e244016f:indexpattern-datasource-layer-675873f9-5e65-4f7d-a731-1e5170a98700", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-a36ff3a0-3f1c-11ec-9edf-13c963822dec.json b/packages/mimecast/kibana/dashboard/mimecast-a36ff3a0-3f1c-11ec-9edf-13c963822dec.json new file mode 100644 index 00000000000..ff469ad99c3 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-a36ff3a0-3f1c-11ec-9edf-13c963822dec.json @@ -0,0 +1,172 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-47e0f438-1420-40d4-a779-1845993eb7ea", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "47e0f438-1420-40d4-a779-1845993eb7ea": { + "columnOrder": [ + "031fd53e-b3ed-422e-b50a-6da93afe2752", + "6fb9dc4a-1056-4e74-a4e4-a469941b6efa" + ], + "columns": { + "031fd53e-b3ed-422e-b50a-6da93afe2752": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "6fb9dc4a-1056-4e74-a4e4-a469941b6efa": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.dlp_logs\"" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "6fb9dc4a-1056-4e74-a4e4-a469941b6efa" + ], + "layerId": "47e0f438-1420-40d4-a779-1845993eb7ea", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "031fd53e-b3ed-422e-b50a-6da93afe2752" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "42cd9bd0-a9a4-41ae-8177-19e72bce8942", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "42cd9bd0-a9a4-41ae-8177-19e72bce8942", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "DLP Logs Count Over Time", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-a36ff3a0-3f1c-11ec-9edf-13c963822dec", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "42cd9bd0-a9a4-41ae-8177-19e72bce8942:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "42cd9bd0-a9a4-41ae-8177-19e72bce8942:indexpattern-datasource-layer-47e0f438-1420-40d4-a779-1845993eb7ea", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-adefc320-3fd8-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-adefc320-3fd8-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..c32fc5b8aae --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-adefc320-3fd8-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,80 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.audit_events\" and event.action : \"logon-authentication-failed\" " + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"6d200d4d-9645-457c-82ee-84bfb2da30ca\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\"},\"includeInFitToBounds\":true,\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"indexPatternId\":\"logs-*\",\"geoField\":\"client.geo.location\",\"filterByMapBounds\":true,\"scalingType\":\"CLUSTERS\",\"id\":\"d0374776-f76c-46ed-a656-a0a35583a2ba\",\"type\":\"ES_SEARCH\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"topHitsSplitField\":\"\",\"topHitsSize\":1},\"id\":\"84b4eec1-9626-4236-8164-b59027952799\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.5,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"BLENDED_VECTOR\",\"joins\":[]}]", + "mapStateJSON": "{\"zoom\":0.83,\"center\":{\"lon\":4.00755,\"lat\":45.66276},\"timeFilters\":{\"from\":\"now-1y/d\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":0},\"query\":{\"query\":\"data_stream.dataset : \\\"mimecast.audit_events\\\"\",\"language\":\"kuery\"},\"filters\":[],\"settings\":{\"autoFitToDataBounds\":false,\"backgroundColor\":\"#ffffff\",\"disableInteractive\":false,\"disableTooltipControl\":false,\"hideToolbarOverlay\":false,\"hideLayerControl\":false,\"hideViewControl\":false,\"initialLocation\":\"LAST_SAVED_LOCATION\",\"fixedLocation\":{\"lat\":0,\"lon\":0,\"zoom\":2},\"browserLocation\":{\"zoom\":2},\"maxZoom\":24,\"minZoom\":0,\"showScaleControl\":false,\"showSpatialFilters\":true,\"showTimesliderToggleButton\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}", + "title": "", + "uiStateJSON": "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}" + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 85.05113, + "maxLon": 180, + "minLat": -85.05113, + "minLon": -180 + }, + "mapCenter": { + "lat": 45.66276, + "lon": 4.00755, + "zoom": 0.83 + }, + "openTOCDetails": [] + }, + "gridData": { + "h": 15, + "i": "c0ed40fd-9fb1-45ac-9d50-5544abcdac2c", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "c0ed40fd-9fb1-45ac-9d50-5544abcdac2c", + "type": "map", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Failed authentication attempts by country", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-adefc320-3fd8-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "c0ed40fd-9fb1-45ac-9d50-5544abcdac2c:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-b8e38930-3fdb-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-b8e38930-3fdb-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..bc94dfc272d --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-b8e38930-3fdb-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,193 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.audit_events\" and event.action : \"logon-authentication-failed\" " + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-e10fb6fc-8079-4a60-9ea5-f54da0eff2f6", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e10fb6fc-8079-4a60-9ea5-f54da0eff2f6": { + "columnOrder": [ + "13c9775c-4b14-4314-a394-e97ffc0e1499", + "a7feab8c-0abd-49eb-96cb-f7a351fa44d3", + "07a0c304-5e0b-4fc7-9b79-e81ddcbe766e", + "01f5144f-929b-4f88-8a0e-995d804e0037" + ], + "columns": { + "01f5144f-929b-4f88-8a0e-995d804e0037": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "count", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "07a0c304-5e0b-4fc7-9b79-e81ddcbe766e": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "src", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "01f5144f-929b-4f88-8a0e-995d804e0037", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 3 + }, + "scale": "ordinal", + "sourceField": "client.ip" + }, + "13c9775c-4b14-4314-a394-e97ffc0e1499": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "user", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "01f5144f-929b-4f88-8a0e-995d804e0037", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "user.email" + }, + "a7feab8c-0abd-49eb-96cb-f7a351fa44d3": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "app", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "01f5144f-929b-4f88-8a0e-995d804e0037", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 3 + }, + "scale": "ordinal", + "sourceField": "mimecast.application" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.audit_events\" " + }, + "visualization": { + "columns": [ + { + "columnId": "13c9775c-4b14-4314-a394-e97ffc0e1499", + "isTransposed": false + }, + { + "columnId": "a7feab8c-0abd-49eb-96cb-f7a351fa44d3", + "isTransposed": false + }, + { + "columnId": "07a0c304-5e0b-4fc7-9b79-e81ddcbe766e", + "isTransposed": false + }, + { + "columnId": "01f5144f-929b-4f88-8a0e-995d804e0037", + "isTransposed": false + } + ], + "layerId": "e10fb6fc-8079-4a60-9ea5-f54da0eff2f6", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "e06e3757-9319-497a-a8ca-9ba059bbcac1", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "e06e3757-9319-497a-a8ca-9ba059bbcac1", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Failed authentication by user, app and src", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-b8e38930-3fdb-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "e06e3757-9319-497a-a8ca-9ba059bbcac1:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e06e3757-9319-497a-a8ca-9ba059bbcac1:indexpattern-datasource-layer-e10fb6fc-8079-4a60-9ea5-f54da0eff2f6", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-be142f50-3fe4-11ec-8ace-9fcc35bfe253.json b/packages/mimecast/kibana/dashboard/mimecast-be142f50-3fe4-11ec-8ace-9fcc35bfe253.json new file mode 100644 index 00000000000..29e712dc9a7 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-be142f50-3fe4-11ec-8ace-9fcc35bfe253.json @@ -0,0 +1,193 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-7a34769f-5338-4cf1-8611-76ee68762548", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "7a34769f-5338-4cf1-8611-76ee68762548": { + "columnOrder": [ + "93e854a1-a782-4a03-97b8-b4f8a98b931e", + "a116654e-42ef-4dbf-9c3f-07dc0ab0eb15", + "73bd76e9-d764-4c7c-bfb0-71205b4f7df5" + ], + "columns": { + "73bd76e9-d764-4c7c-bfb0-71205b4f7df5": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "93e854a1-a782-4a03-97b8-b4f8a98b931e": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of mimecast.scanResult", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "73bd76e9-d764-4c7c-bfb0-71205b4f7df5", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "mimecast.scanResult" + }, + "a116654e-42ef-4dbf-9c3f-07dc0ab0eb15": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1m" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.ttp_url_logs\"" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "73bd76e9-d764-4c7c-bfb0-71205b4f7df5" + ], + "layerId": "7a34769f-5338-4cf1-8611-76ee68762548", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "93e854a1-a782-4a03-97b8-b4f8a98b931e", + "xAccessor": "a116654e-42ef-4dbf-9c3f-07dc0ab0eb15" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "92d24487-e592-4782-96b3-b3ea803de9f3", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "92d24487-e592-4782-96b3-b3ea803de9f3", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h/h", + "timeRestore": true, + "timeTo": "now", + "title": "TTP URL - Clean vs malicious over time", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-be142f50-3fe4-11ec-8ace-9fcc35bfe253", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "92d24487-e592-4782-96b3-b3ea803de9f3:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "92d24487-e592-4782-96b3-b3ea803de9f3:indexpattern-datasource-layer-7a34769f-5338-4cf1-8611-76ee68762548", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-c06de870-4081-11ec-b8da-95c3fba730d0.json b/packages/mimecast/kibana/dashboard/mimecast-c06de870-4081-11ec-b8da-95c3fba730d0.json new file mode 100644 index 00000000000..d0bda8c9215 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-c06de870-4081-11ec-b8da-95c3fba730d0.json @@ -0,0 +1,171 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-399531fb-a3b2-4881-aa91-9b3f9e7d34e7", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "399531fb-a3b2-4881-aa91-9b3f9e7d34e7": { + "columnOrder": [ + "d17db96e-f800-4bb6-ad48-2f10d7c1fc34", + "9ba4c455-c64a-4ce6-8d0e-a17e79390bd3" + ], + "columns": { + "9ba4c455-c64a-4ce6-8d0e-a17e79390bd3": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "d17db96e-f800-4bb6-ad48-2f10d7c1fc34": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.threat_intel_malware_customer\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "9ba4c455-c64a-4ce6-8d0e-a17e79390bd3" + ], + "layerId": "399531fb-a3b2-4881-aa91-9b3f9e7d34e7", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "d17db96e-f800-4bb6-ad48-2f10d7c1fc34" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "224296e8-ab37-4224-a65b-c929646f14fa", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "224296e8-ab37-4224-a65b-c929646f14fa", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "Threat Intel Malware Customer - over time", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-c06de870-4081-11ec-b8da-95c3fba730d0", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "224296e8-ab37-4224-a65b-c929646f14fa:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "224296e8-ab37-4224-a65b-c929646f14fa:indexpattern-datasource-layer-399531fb-a3b2-4881-aa91-9b3f9e7d34e7", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-c9649870-3e33-11ec-a4d8-6b18148fbf3a.json b/packages/mimecast/kibana/dashboard/mimecast-c9649870-3e33-11ec-a4d8-6b18148fbf3a.json new file mode 100644 index 00000000000..b6283d834a9 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-c9649870-3e33-11ec-a4d8-6b18148fbf3a.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-87e37d53-70f7-4337-86ed-832fcb7f9383", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "87e37d53-70f7-4337-86ed-832fcb7f9383": { + "columnOrder": [ + "482922c8-4843-45af-9b42-01c50685bfbe", + "9643e088-9c36-476d-a969-244e0d2ecc23" + ], + "columns": { + "482922c8-4843-45af-9b42-01c50685bfbe": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Delivery Failures", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "9643e088-9c36-476d-a969-244e0d2ecc23", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "error.type" + }, + "9643e088-9c36-476d-a969-244e0d2ecc23": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.siem_logs\" and mimecast.log_type : \"delivery\" and event.outcome : \"false\"" + }, + "visualization": { + "columns": [ + { + "columnId": "482922c8-4843-45af-9b42-01c50685bfbe", + "isTransposed": false + }, + { + "columnId": "9643e088-9c36-476d-a969-244e0d2ecc23", + "isTransposed": false + } + ], + "layerId": "87e37d53-70f7-4337-86ed-832fcb7f9383", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "6e9ef8dd-bd73-4145-89ca-d0467eb8c6d8", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "6e9ef8dd-bd73-4145-89ca-d0467eb8c6d8", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Delivery Failures", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-c9649870-3e33-11ec-a4d8-6b18148fbf3a", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "6e9ef8dd-bd73-4145-89ca-d0467eb8c6d8:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "6e9ef8dd-bd73-4145-89ca-d0467eb8c6d8:indexpattern-datasource-layer-87e37d53-70f7-4337-86ed-832fcb7f9383", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-cd32b740-407f-11ec-b8da-95c3fba730d0.json b/packages/mimecast/kibana/dashboard/mimecast-cd32b740-407f-11ec-b8da-95c3fba730d0.json new file mode 100644 index 00000000000..d1de839ba2d --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-cd32b740-407f-11ec-b8da-95c3fba730d0.json @@ -0,0 +1,170 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-482f4c89-6ca6-4520-826e-876c0256ae1b", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "482f4c89-6ca6-4520-826e-876c0256ae1b": { + "columnOrder": [ + "6035b29a-145b-48c5-9faf-0d33060bfda0", + "26106801-2a8f-464c-9a0e-439bb734b16b" + ], + "columns": { + "26106801-2a8f-464c-9a0e-439bb734b16b": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "6035b29a-145b-48c5-9faf-0d33060bfda0": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.threat_intel_malware_grid\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "26106801-2a8f-464c-9a0e-439bb734b16b" + ], + "layerId": "482f4c89-6ca6-4520-826e-876c0256ae1b", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "6035b29a-145b-48c5-9faf-0d33060bfda0" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "19e621d8-66a9-4c26-8970-f56a25c1fd98", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "19e621d8-66a9-4c26-8970-f56a25c1fd98", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "Threat Intel Malware Grid - logs over time", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-cd32b740-407f-11ec-b8da-95c3fba730d0", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "19e621d8-66a9-4c26-8970-f56a25c1fd98:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "19e621d8-66a9-4c26-8970-f56a25c1fd98:indexpattern-datasource-layer-482f4c89-6ca6-4520-826e-876c0256ae1b", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-ced41610-3f1d-11ec-9edf-13c963822dec.json b/packages/mimecast/kibana/dashboard/mimecast-ced41610-3f1d-11ec-9edf-13c963822dec.json new file mode 100644 index 00000000000..1deb14fe725 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-ced41610-3f1d-11ec-9edf-13c963822dec.json @@ -0,0 +1,148 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-0f5b8670-33ce-47e6-ac1f-b29f55afaf24", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "0f5b8670-33ce-47e6-ac1f-b29f55afaf24": { + "columnOrder": [ + "7f11f183-c159-43db-8b95-cbb8fd2d8fd7", + "0033ecfa-a5f3-4828-9fd8-ae82caf7c8f1" + ], + "columns": { + "0033ecfa-a5f3-4828-9fd8-ae82caf7c8f1": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "7f11f183-c159-43db-8b95-cbb8fd2d8fd7": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Senders", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "0033ecfa-a5f3-4828-9fd8-ae82caf7c8f1", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "email.from" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.dlp_logs\"" + }, + "visualization": { + "columns": [ + { + "columnId": "7f11f183-c159-43db-8b95-cbb8fd2d8fd7", + "isTransposed": false + }, + { + "columnId": "0033ecfa-a5f3-4828-9fd8-ae82caf7c8f1", + "isTransposed": false + } + ], + "layerId": "0f5b8670-33ce-47e6-ac1f-b29f55afaf24", + "layerType": "data", + "sorting": { + "columnId": "0033ecfa-a5f3-4828-9fd8-ae82caf7c8f1", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "da26e53f-9dd1-4f9a-80a2-3bbe0a30a54b", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "da26e53f-9dd1-4f9a-80a2-3bbe0a30a54b", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "DLP Logs Senders that triggered policies", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-ced41610-3f1d-11ec-9edf-13c963822dec", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "da26e53f-9dd1-4f9a-80a2-3bbe0a30a54b:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "da26e53f-9dd1-4f9a-80a2-3bbe0a30a54b:indexpattern-datasource-layer-0f5b8670-33ce-47e6-ac1f-b29f55afaf24", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-f5a92540-3e33-11ec-a4d8-6b18148fbf3a.json b/packages/mimecast/kibana/dashboard/mimecast-f5a92540-3e33-11ec-a4d8-6b18148fbf3a.json new file mode 100644 index 00000000000..f96a0ac7cde --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-f5a92540-3e33-11ec-a4d8-6b18148fbf3a.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-e55c6dff-df9b-4c78-96e4-af36202efbde", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e55c6dff-df9b-4c78-96e4-af36202efbde": { + "columnOrder": [ + "f8efadab-8604-4947-8ef2-7f0d38db76f4", + "7f83a56b-b863-482d-962d-78a2e36940d5" + ], + "columns": { + "7f83a56b-b863-482d-962d-78a2e36940d5": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "f8efadab-8604-4947-8ef2-7f0d38db76f4": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of error.type", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "7f83a56b-b863-482d-962d-78a2e36940d5", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "error.type" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.siem_logs\" and mimecast.log_type : \"receipt\" and event.action : \"Rej\"" + }, + "visualization": { + "columns": [ + { + "columnId": "f8efadab-8604-4947-8ef2-7f0d38db76f4" + }, + { + "columnId": "7f83a56b-b863-482d-962d-78a2e36940d5" + } + ], + "layerId": "e55c6dff-df9b-4c78-96e4-af36202efbde", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 13, + "i": "cc80e96c-c8ca-420c-8f12-495157f0154f", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "cc80e96c-c8ca-420c-8f12-495157f0154f", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1y/d", + "timeRestore": true, + "timeTo": "now", + "title": "Rejections Reasons", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-f5a92540-3e33-11ec-a4d8-6b18148fbf3a", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "cc80e96c-c8ca-420c-8f12-495157f0154f:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "cc80e96c-c8ca-420c-8f12-495157f0154f:indexpattern-datasource-layer-e55c6dff-df9b-4c78-96e4-af36202efbde", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/kibana/dashboard/mimecast-f7faa440-4081-11ec-b8da-95c3fba730d0.json b/packages/mimecast/kibana/dashboard/mimecast-f7faa440-4081-11ec-b8da-95c3fba730d0.json new file mode 100644 index 00000000000..3ca15c86d92 --- /dev/null +++ b/packages/mimecast/kibana/dashboard/mimecast-f7faa440-4081-11ec-b8da-95c3fba730d0.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-662c8260-62a4-4b11-8942-e7900c2fb1bb", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "662c8260-62a4-4b11-8942-e7900c2fb1bb": { + "columnOrder": [ + "c9e207f1-1b64-4b4a-b6cb-ddc770733a8b", + "7c2cbcee-2579-4971-a811-12bbb4815d9e" + ], + "columns": { + "7c2cbcee-2579-4971-a811-12bbb4815d9e": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "c9e207f1-1b64-4b4a-b6cb-ddc770733a8b": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of threat.indicator.type", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "7c2cbcee-2579-4971-a811-12bbb4815d9e", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.type" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset : \"mimecast.threat_intel_malware_customer\" " + }, + "visualization": { + "columns": [ + { + "columnId": "c9e207f1-1b64-4b4a-b6cb-ddc770733a8b" + }, + { + "columnId": "7c2cbcee-2579-4971-a811-12bbb4815d9e" + } + ], + "layerId": "662c8260-62a4-4b11-8942-e7900c2fb1bb", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "24de1393-e9af-4d34-829e-bc19b169f89b", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "24de1393-e9af-4d34-829e-bc19b169f89b", + "type": "lens", + "version": "7.16.0-SNAPSHOT" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-7d/d", + "timeRestore": true, + "timeTo": "now", + "title": "Thret Intel Malware Customer by indicator type", + "version": 1 + }, + "coreMigrationVersion": "7.16.0", + "id": "mimecast-f7faa440-4081-11ec-b8da-95c3fba730d0", + "migrationVersion": { + "dashboard": "7.16.0" + }, + "references": [ + { + "id": "logs-*", + "name": "24de1393-e9af-4d34-829e-bc19b169f89b:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "24de1393-e9af-4d34-829e-bc19b169f89b:indexpattern-datasource-layer-662c8260-62a4-4b11-8942-e7900c2fb1bb", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml new file mode 100644 index 00000000000..9b75381f4f8 --- /dev/null +++ b/packages/mimecast/manifest.yml @@ -0,0 +1,32 @@ +format_version: 1.0.0 +name: mimecast +title: "Mimecast/Elastic Integration" +version: 0.0.1 +license: basic +description: "Fetching logs from Mimecast API and ingest into Elasticsearch" +type: integration +categories: + - custom +release: experimental +conditions: + kibana.version: "^7.16.0" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: mimecast + title: Mimecast + description: Mimecast Integration + inputs: + - type: httpjson + title: Mimecast API + description: Collect logs from Mimecast API +owner: + github: elastic/external-security-integrations