diff --git a/packages/proofpoint_tap/changelog.yml b/packages/proofpoint_tap/changelog.yml index 7bf7281b19b..98655ce13d6 100644 --- a/packages/proofpoint_tap/changelog.yml +++ b/packages/proofpoint_tap/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.2" + changes: + - description: Ensure that query endpoints have been published to the stored cursor state. + type: bugfix + link: https://github.com/elastic/integrations/pull/11475 - version: "1.24.1" changes: - description: Ensure that queries satisfy API restrictions. diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log b/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log index 7a63004ccfb..649a10ea527 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log @@ -3,3 +3,4 @@ {"url":"https://www.example.com/url?q=httpabc12345","classification":"spam","clickTime":"2022-03-30T07:10:19.000Z","threatTime":"2022-03-29T09:27:21.000Z","userAgent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"85219a90-1234-1234-1234-axx5xx4xxxfxxxx","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"b81458bb9f757994e79a9287b8447622@example.com","senderIP":"81.2.69.143","GUID":"JXXXXaXehXHXzX-XxXhXyXXXXX7","threatID":"eaxxxxxxxxxxxx6376xxxxxxxxxxx1cba65xxx9x7xxxxxxxxxxfbbxx4x0","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/eaxxxxxa6597fd3xxxxxxxxx92e4xxxxxxxxxx27c98052fxxxxxxxxxx1234","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"} {"url":"https://www.example.org/abcdabcd123?query=0","classification":"malware","clickTime":"2022-03-30T10:11:12.000Z","threatTime":"2022-03-21T14:40:31.000Z","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/199.0.427504638 Mobile/15E148 Safari/604.1","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"a5c9f8bb-1234-1234-1234-dxx9xcxxxx8xxxc","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"9c52aa64228824247c48df69b066e5a7@example.com","senderIP":"81.2.69.143","GUID":"XXcXXxXDXVXXXXXXXXXXXX4XXXXX","threatID":"502bxxxxxxxxxxx70513b6cxxxxxxxxxxxxebc7fc699xxxxxxxxxxxxxxxxd5f","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/502xxxxxxxxxcebxxxxxxxxxxa04277xxxxx5dxc6xxxxxxxxx5f","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"} {"url":"https://www.example.org","classification":"spam","clickTime":"2022-03-30T10:01:01.000Z","threatTime":"2022-03-14T05:59:12.000Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"d35cc5fc-1234-1234-1234-2xxx0xaxbxcxx","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"xyz@example.com","senderIP":"81.2.69.143","GUID":"uHXXXJXTXlXDXmXgXTX3XOXLNXVXNX3XXXHX","threatID":"47580xdx0x2x5x2xfx8x3x3x7x7xxxxcx6x7x4x4x1xexcx5cx9x3xfxfxxx1","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/4xxxxd02xxxxxxxxxxxxcacf9da3xxxxxxxxxxx9a947xxxxxxxxxx1","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"} +{"queryEndTime":"2024-10-11T14:34:53Z","clicksBlocked":[]} diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log-expected.json b/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log-expected.json index 5749dd1677b..d74041f341b 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log-expected.json +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log-expected.json @@ -496,6 +496,7 @@ }, "version": "99.0.4844.82" } - } + }, + null ] } \ No newline at end of file diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs index 19aa7e39da1..c8e6955eb8a 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs @@ -92,7 +92,7 @@ cursor: value: '[[.last_response.body.queryEndTime]]' response.split: target: body.clicksBlocked - ignore_empty_value: true + ignore_empty_value: false tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml index ea2aae1d416..ec916265c71 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,8 @@ processors: field: event.original target_field: json ignore_failure: true + - drop: + if: ctx.json?.clicksBlocked instanceof List && ctx.json.clicksBlocked.length == 0 - fingerprint: fields: - event.original diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log b/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log index c74362e0cb1..1de6a58f482 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log @@ -2,3 +2,4 @@ {"url":"https://example.com/collab/?id=x4x3x6xsx1xxxx8xEdxexnxxxaxX","classification":"phish","clickTime":"2022-03-21T20:39:37.000Z","threatTime":"2022-03-30T10:05:57.000Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"de7eef56-1234-1234-1234-54xxxxx123","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"abc@example.com","senderIP":"81.2.69.143","GUID":"cXXTXpX7jXXXXHXxXBXXkXXXwXXX","threatID":"92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"} {"url":"http://example.com/ixxxx464xxx6x6xxd_cXxxxT_kxxTuQx_xIhxlx2qxxnxvxPxn","classification":"spam","clickTime":"2022-03-30T10:51:53.000Z","threatTime":"2022-02-26T00:36:25.000Z","userAgent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"90dd54bc-1234-1234-1234-cxxxxxxxxx4","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"exxxxxxx8x2xxxx2x6x6xxxxx6xxxx5@example.com","senderIP":"81.2.69.143","GUID":"QUWXXxXXJHlYXRXXXXVXUXXk","threatID":"xxxxxxbx1cxcxx0xcx5xxxxdx5xex8xbx7xxxeexxxxxxxx9","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/xxxxxxbx1cxcxx0xcx5xxxxdx5xex8xbx7xxxeexxxxxxxx9","threatStatus":"cleared","messageID":"12345678912345.12345.mail@example.com"} {"url":"https://xyz123456789.support.com#xyz@example.com","classification":"phish","clickTime":"2022-03-30T00:56:14.000Z","threatTime":"2022-03-30T00:53:43.000Z","userAgent":"Mozilla/5.0 (Linux; Android 12; SM-N976U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.88 Mobile Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"4b4ae949-1234-1234-1234-6axxxxx9xxxxx3","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"f3xxxx0x2xcx3xaxbxcx2xaxxxcxxxx2@example.com","senderIP":"81.2.69.143","GUID":"VXXhXiXyXBXlXdXXfXXXXXWXLXXX","threatID":"xxxdxxdx6x7x6xxxxx5xxx837ex4x4xcx8xcxxxexxx2xxxxxx5","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/xxxdxxdx6x7x6xxxxx5xxx837ex4x4xcx8xcxxxexxx2xxxxxx5","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"} +{"queryEndTime":"2024-10-11T14:34:53Z","clicksPermitted":[]} diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log-expected.json b/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log-expected.json index 0a1072095da..20209d37719 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log-expected.json +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log-expected.json @@ -396,6 +396,7 @@ }, "version": "99.0.4844.88" } - } + }, + null ] } \ No newline at end of file diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs index 5d503f17559..2777905d6f2 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs @@ -92,7 +92,7 @@ cursor: value: '[[.last_response.body.queryEndTime]]' response.split: target: body.clicksPermitted - ignore_empty_value: true + ignore_empty_value: false tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml index e6cc5707162..c48310f0320 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,8 @@ processors: field: event.original target_field: json ignore_failure: true + - drop: + if: ctx.json?.clicksPermitted instanceof List && ctx.json.clicksPermitted.length == 0 - fingerprint: fields: - event.original diff --git a/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log b/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log index d32cfca8381..8bc0d41c546 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log +++ b/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log @@ -4,3 +4,4 @@ {"spamScore":100,"phishScore":100,"threatsInfoMap":[{"threatID":"cfdhgondhgonvjdsdefghjikhlonvjdsvsbnvjd56546ghjikhlonvjdsvsbnvjd","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/adhgon-vjdsdef-ghjikhlonv-abcdefghij/threat/email/7921af132d1aa6a88fdbdadkhlonvj1a8xxxxxxxxxxxxxxxxxxxxxdkhlonvj1","threatTime":"2022-01-01T05:02:48.832Z","threat":"https://example.com/","campaignID":null,"threatType":"url"},{"threatID":"124563bcdefghijkabcdefghi201256abcdefghijk201256aswe20abc","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/abcdefgh-1234-1234-1234-1234-abcdefgh/threat/email/85738a8x9x7x1x04x5329xaadc9x425925abdf84089wcwe3x022xx4x19x123","threatTime":"2022-01-01T00:00:00.000Z","threat":"example.com","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T00:25:20.010Z","impostorScore":0.0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":"Statement From (Trinity Groundwater)","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound","allow_relay"],"modulesRun":["av","zerohour","dkimv","spf","spam","dmarc"],"messageSize":111091,"headerFrom":"Laura Schumacher ","headerReplyTo":null,"fromAddress":["abc@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["mail@example.com","abc@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"attached","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"image001.png","sandboxStatus":null,"oContentType":"image/png","contentType":"image/png"}],"completelyRewritten":false,"id":"8f12300-f387-1234-xxxx-a4abcd12347","QID":"0XX0XXXXaX3XXX-X1","GUID":"_pxxxxOxQxxXxx4wxjxtx2xxxTxxxYxxx","sender":"abc@example.com","recipient":["mailer-daemon@example.com"],"senderIP":"175.16.199.1","messageID":"<77F0EA74-7D6F-453A-AB7F-31B192481AE8@example.com>"} {"spamScore":100,"phishScore":100,"threatsInfoMap":[{"threatID":"9dhgabcdefghijkhgonvjdsdefghjikhlonvjdsvsbnvjdvjdsdefghjikhlonv","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/adhgon-vjdsdefghj-ikhlonvj-abcdefghij/threat/email/97921af132d1aa6a88fdbdadkhlonvjbc9fxxxxxxxxxxxxxxxxxxxxxbdadkhlonvjd","threatTime":"2022-01-01T03:02:25.092Z","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T00:00:00.000Z","impostorScore":0.0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":"(1) VOICE MAIL MESSSAGE","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound"],"modulesRun":["av","zerohour","dkimv","spf","spam","dmarc","pdr"],"messageSize":5776,"headerFrom":"VOICE MAIL","headerReplyTo":null,"fromAddress":["man.web@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["mailer-daemon@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"}],"completelyRewritten":false,"id":"ee212323-1234-1234-1234-0f0abcd123456","QID":"3XXXf1XaXX-X1XX","GUID":"gxxxxxgxx3xcx-MxZxixxoxxxxxAxxx2","sender":"man.web@example.com","recipient":["mailer-daemon@example.com"],"senderIP":"175.16.199.1","messageID":"<20220327194933.12463F24B8AC1B73@example.com>"} {"spamScore":100,"phishScore":100,"threatsInfoMap":[{"threatID":"abcdefghijkabcdefghijkabcdefghijkefghjikhlonvjdsvsbnvjd","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/adhgonvj-dsdefgh-jikhlon-abcdefghij/threat/email/7921af132xxxxxxxxxxxxxxxxxxviuerhvuie35abcdefghabcdefghijk","threatTime":"2022-01-01T00:00:00.000Z","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T05:00:02.010Z","impostorScore":0.0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":"(1) VOICE MAIL MESSSAGE","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound"],"modulesRun":["av","zerohour","dkimv","spf","spam","dmarc","pdr"],"messageSize":5776,"headerFrom":"VOICE MAIL","headerReplyTo":null,"fromAddress":["man.web@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["mailer-daemon@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"}],"completelyRewritten":false,"id":"ee212323-1234-1234-1234-0f0abcd123456","QID":"3XXfXabXcXXXX1","GUID":"gxxxxgx3xcx-xMx7xPxxZxxxxoxAx2xxxxx","sender":"man.web@example.com","recipient":["mailer-daemon@example.com"],"senderIP":"89.160.20.112","messageID":"<20220327194933.12463F24B8AC1B73@example.com>"} +{"queryEndTime":"2024-10-11T14:34:53Z","messagesBlocked":[]} diff --git a/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log-expected.json b/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log-expected.json index ee21be49e1e..1169887d5a3 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log-expected.json +++ b/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log-expected.json @@ -845,6 +845,7 @@ "tags": [ "preserve_original_event" ] - } + }, + null ] } \ No newline at end of file diff --git a/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs index 93619e6223f..e2c87b4f274 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs @@ -92,7 +92,7 @@ cursor: value: '[[.last_response.body.queryEndTime]]' response.split: target: body.messagesBlocked - ignore_empty_value: true + ignore_empty_value: false tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml index 5ce7c094da9..fd080fb4bd2 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,8 @@ processors: field: event.original target_field: json ignore_failure: true + - drop: + if: ctx.json?.messagesBlocked instanceof List && ctx.json.messagesBlocked.length == 0 - date: field: json.messageTime if: ctx.json?.messageTime != null && ctx.json.messageTime != '' diff --git a/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log b/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log index e579c6b8ac9..b8c211dc62d 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log +++ b/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log @@ -6,3 +6,4 @@ {"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"6exxxxxxxxxxx123456xxxxxxxxxxx12345643cedfbbe1xxxxxxxxxxx123456b","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/3183a23b-d9c3-1234-1234-2babcd123408/threat/email/6e2eefdxxxxxxxxxxxxxxxxb3f43ceaafxxxxxxxxxxe5c91axxxbb","threatTime":"2022-04-01T23:14:30.450Z","threat":"https://example.com/view/8yxxxxvjxxxx5","campaignID":null,"threatType":"url"}],"messageTime":"2021-09-28T16:28:59.490Z","impostorScore":0,"malwareScore":0,"cluster":"example_hosted","subject":"RSVP today to Join Transpose Platform’s Ecosystem Days Summit","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound"],"modulesRun":["av","zerohour","spf","dkimv","spam","pdr","urldefense"],"messageSize":2657297,"headerFrom":"abc.xyz@example.com","headerReplyTo":null,"fromAddress":["abc.xyz@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["abc.xyz@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"attached","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"image.png","sandboxStatus":null,"oContentType":"image/png","contentType":"image/png"}],"completelyRewritten":true,"id":"fbxxxxxx1-xxxxx123-xxxxx-xxxxx1234","QID":"2XX2XXOXFXXGX8X9X","GUID":"pxxxxvxxxxPxTxxxixxxxFxxxUxx2xxxxx","sender":"abc.xyz@example.com","recipient":["abc.xyz@example.com"],"senderIP":"175.16.199.1","messageID":""} {"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"xxxxxxxxxxx12345678914xxxxxxxxxxx123456e9ff24a9xxxxxxxxxxx123456","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/3183a23b-d9c3-1234-1234-2babcd123408/threat/email/9f2dbcaa9xxxxxxxxxxe810d280xxxxxxxxxxxe48f6e69xxxxxxf","threatTime":"2022-04-01T12:48:03.852Z","threat":"https://example.com/view/xp45xxxxxxir9y","campaignID":null,"threatType":"url"}],"messageTime":"2022-08-17T18:00:22.060Z","impostorScore":0,"malwareScore":0,"cluster":"example_hosted","subject":"Speakers Announced | Ecosystem Days Summit","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["bypass_maxsize","default_inbound"],"modulesRun":["av","zerohour","spf","dkimv","spam","pdr","urldefense"],"messageSize":68353,"headerFrom":"Trang, Alex & Transpose Platform Team ","headerReplyTo":"Trang, Alex & Transpose Platform Team ","fromAddress":["client.services@example.com"],"ccAddresses":[],"replyToAddress":["client.services@example.com"],"toAddresses":["abc.xyz@example.com"],"xmailer":"Mailchimp Mailer - **CIDxxxxxxxxxx1234**","messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"}],"completelyRewritten":true,"id":"fxxxdxxa-xxxxx123-xxxxx-xxxxx1234","QID":"X2XXX0XXX2XX4","GUID":"wxxAxxxx8x8x5xxxxxJxPxxax7xxxxx","sender":"xyz-abc.us1_152023242.12345678-6xxxx123456789@example.com","recipient":["abc.xyz@example.com"],"senderIP":"175.16.199.1","messageID":"<200cxyz1234xyz1234bcb96f3.6xyz12345.202204125625899.736a993333.x12345678e@example.com>"} {"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"xxxxxxxxxxx123456xxxxxxxxxx1234xxxxxxxxxxx123456bbe1xxxxxx123456","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/3183a23b-d9c3-1234-1234-2babcd123408/threat/email/6e2eefd8cxxxxxxxxxeef270d0a1b3f43cexxxxxxxxx34abe5c91axxxcb","threatTime":"2022-04-01T20:56:13.000Z","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","campaignID":null,"threatType":"url"}],"messageTime":"2022-03-24T13:24:57.000Z","impostorScore":0,"malwareScore":0,"cluster":"example_hosted","subject":"RSVP today to Join Transpose Platform’s Ecosystem Days Summit","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["bypass_maxsize","default_inbound"],"modulesRun":["av","zerohour","spf","dkimv","spam","pdr","urldefense"],"messageSize":2642117,"headerFrom":"abc.xyz@example.com","headerReplyTo":null,"fromAddress":["abc.xyz@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["abc.xyz@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"attached","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"image.png","sandboxStatus":null,"oContentType":"image/png","contentType":"image/png"}],"completelyRewritten":true,"id":"cxxxxbxxxb-xxxxx123-xxxxx-xxxxx1234","QID":"2XXX2X5XX5XX7","GUID":"gpxxx5xx2xHxxxJx7xxxxmx5xcxxxxxZ","sender":"abc.xyz@example.com","recipient":["abc.xyz@example.com"],"senderIP":"175.16.199.1","messageID":""} +{"queryEndTime":"2024-10-11T14:34:53Z","messagesDelivered":[]} diff --git a/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log-expected.json b/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log-expected.json index ac9780a48b6..a21e13ce2df 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log-expected.json +++ b/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log-expected.json @@ -916,6 +916,7 @@ "tags": [ "preserve_original_event" ] - } + }, + null ] } \ No newline at end of file diff --git a/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs index a183bb103e1..ddfd7f681d2 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs @@ -92,7 +92,7 @@ cursor: value: '[[.last_response.body.queryEndTime]]' response.split: target: body.messagesDelivered - ignore_empty_value: true + ignore_empty_value: false tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml index 80268f467c8..e2a343c94ad 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,8 @@ processors: field: event.original target_field: json ignore_failure: true + - drop: + if: ctx.json?.messagesDelivered instanceof List && ctx.json.messagesDelivered.length == 0 - date: field: json.messageTime if: ctx.json?.messageTime != null && ctx.json.messageTime != '' diff --git a/packages/proofpoint_tap/manifest.yml b/packages/proofpoint_tap/manifest.yml index 4ec06182fea..aad5c0d643a 100644 --- a/packages/proofpoint_tap/manifest.yml +++ b/packages/proofpoint_tap/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: proofpoint_tap title: Proofpoint TAP -version: "1.24.1" +version: "1.24.2" description: Collect logs from Proofpoint TAP with Elastic Agent. type: integration categories: