proofpoint_tap: ensure cursor state is published when a query has been made

packages/proofpoint_tap/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.24.2"
+  changes:
+    - description: Ensure that query endpoints have been published to the stored cursor state.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11475
 - version: "1.24.1"
   changes:
     - description: Ensure that queries satisfy API restrictions.

packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log

@@ -3,3 +3,4 @@
 {"url":"https://www.example.com/url?q=httpabc12345","classification":"spam","clickTime":"2022-03-30T07:10:19.000Z","threatTime":"2022-03-29T09:27:21.000Z","userAgent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"85219a90-1234-1234-1234-axx5xx4xxxfxxxx","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"b81458bb9f757994e79a9287b8447622@example.com","senderIP":"81.2.69.143","GUID":"JXXXXaXehXHXzX-XxXhXyXXXXX7","threatID":"eaxxxxxxxxxxxx6376xxxxxxxxxxx1cba65xxx9x7xxxxxxxxxxfbbxx4x0","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/eaxxxxxa6597fd3xxxxxxxxx92e4xxxxxxxxxx27c98052fxxxxxxxxxx1234","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}
 {"url":"https://www.example.org/abcdabcd123?query=0","classification":"malware","clickTime":"2022-03-30T10:11:12.000Z","threatTime":"2022-03-21T14:40:31.000Z","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/199.0.427504638 Mobile/15E148 Safari/604.1","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"a5c9f8bb-1234-1234-1234-dxx9xcxxxx8xxxc","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"9c52aa64228824247c48df69b066e5a7@example.com","senderIP":"81.2.69.143","GUID":"XXcXXxXDXVXXXXXXXXXXXX4XXXXX","threatID":"502bxxxxxxxxxxx70513b6cxxxxxxxxxxxxebc7fc699xxxxxxxxxxxxxxxxd5f","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/502xxxxxxxxxcebxxxxxxxxxxa04277xxxxx5dxc6xxxxxxxxx5f","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}
 {"url":"https://www.example.org","classification":"spam","clickTime":"2022-03-30T10:01:01.000Z","threatTime":"2022-03-14T05:59:12.000Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"d35cc5fc-1234-1234-1234-2xxx0xaxbxcxx","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"xyz@example.com","senderIP":"81.2.69.143","GUID":"uHXXXJXTXlXDXmXgXTX3XOXLNXVXNX3XXXHX","threatID":"47580xdx0x2x5x2xfx8x3x3x7x7xxxxcx6x7x4x4x1xexcx5cx9x3xfxfxxx1","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/4xxxxd02xxxxxxxxxxxxcacf9da3xxxxxxxxxxx9a947xxxxxxxxxx1","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}
+{"queryEndTime":"2024-10-11T14:34:53Z","clicksBlocked":[]}

packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/pipeline/test-clicks-blocked.log-expected.json

@@ -496,6 +496,7 @@
                 },
                 "version": "99.0.4844.82"
             }
-        }
+        },
+        null
     ]
 }

packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs

@@ -92,7 +92,7 @@ cursor:
     value: '[[.last_response.body.queryEndTime]]'
 response.split:
   target: body.clicksBlocked
-  ignore_empty_value: true
+  ignore_empty_value: false
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml

@@ -13,6 +13,8 @@ processors:
       field: event.original
       target_field: json
       ignore_failure: true
+  - drop:
+      if: ctx.json?.clicksBlocked instanceof List && ctx.json.clicksBlocked.length == 0
   - fingerprint:
       fields:
         - event.original

packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log

@@ -2,3 +2,4 @@
 {"url":"https://example.com/collab/?id=x4x3x6xsx1xxxx8xEdxexnxxxaxX","classification":"phish","clickTime":"2022-03-21T20:39:37.000Z","threatTime":"2022-03-30T10:05:57.000Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"de7eef56-1234-1234-1234-54xxxxx123","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"abc@example.com","senderIP":"81.2.69.143","GUID":"cXXTXpX7jXXXXHXxXBXXkXXXwXXX","threatID":"92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}
 {"url":"http://example.com/ixxxx464xxx6x6xxd_cXxxxT_kxxTuQx_xIhxlx2qxxnxvxPxn","classification":"spam","clickTime":"2022-03-30T10:51:53.000Z","threatTime":"2022-02-26T00:36:25.000Z","userAgent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"90dd54bc-1234-1234-1234-cxxxxxxxxx4","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"exxxxxxx8x2xxxx2x6x6xxxxx6xxxx5@example.com","senderIP":"81.2.69.143","GUID":"QUWXXxXXJHlYXRXXXXVXUXXk","threatID":"xxxxxxbx1cxcxx0xcx5xxxxdx5xex8xbx7xxxeexxxxxxxx9","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/xxxxxxbx1cxcxx0xcx5xxxxdx5xex8xbx7xxxeexxxxxxxx9","threatStatus":"cleared","messageID":"12345678912345.12345.mail@example.com"}
 {"url":"https://xyz123456789.support.com#xyz@example.com","classification":"phish","clickTime":"2022-03-30T00:56:14.000Z","threatTime":"2022-03-30T00:53:43.000Z","userAgent":"Mozilla/5.0 (Linux; Android 12; SM-N976U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.88 Mobile Safari/537.36","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"4b4ae949-1234-1234-1234-6axxxxx9xxxxx3","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"f3xxxx0x2xcx3xaxbxcx2xaxxxcxxxx2@example.com","senderIP":"81.2.69.143","GUID":"VXXhXiXyXBXlXdXXfXXXXXWXLXXX","threatID":"xxxdxxdx6x7x6xxxxx5xxx837ex4x4xcx8xcxxxexxx2xxxxxx5","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/xxxdxxdx6x7x6xxxxx5xxx837ex4x4xcx8xcxxxexxx2xxxxxx5","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}
+{"queryEndTime":"2024-10-11T14:34:53Z","clicksPermitted":[]}

packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/pipeline/test-clicks-permitted.log-expected.json

@@ -396,6 +396,7 @@
                 },
                 "version": "99.0.4844.88"
             }
-        }
+        },
+        null
     ]
 }

packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs

@@ -92,7 +92,7 @@ cursor:
     value: '[[.last_response.body.queryEndTime]]'
 response.split:
   target: body.clicksPermitted
-  ignore_empty_value: true
+  ignore_empty_value: false
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml

@@ -13,6 +13,8 @@ processors:
       field: event.original
       target_field: json
       ignore_failure: true
+  - drop:
+      if: ctx.json?.clicksPermitted instanceof List && ctx.json.clicksPermitted.length == 0
   - fingerprint:
       fields:
         - event.original

packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log

@@ -4,3 +4,4 @@
 {"spamScore":100,"phishScore":100,"threatsInfoMap":[{"threatID":"cfdhgondhgonvjdsdefghjikhlonvjdsvsbnvjd56546ghjikhlonvjdsvsbnvjd","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/adhgon-vjdsdef-ghjikhlonv-abcdefghij/threat/email/7921af132d1aa6a88fdbdadkhlonvj1a8xxxxxxxxxxxxxxxxxxxxxdkhlonvj1","threatTime":"2022-01-01T05:02:48.832Z","threat":"https://example.com/","campaignID":null,"threatType":"url"},{"threatID":"124563bcdefghijkabcdefghi201256abcdefghijk201256aswe20abc","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/abcdefgh-1234-1234-1234-1234-abcdefgh/threat/email/85738a8x9x7x1x04x5329xaadc9x425925abdf84089wcwe3x022xx4x19x123","threatTime":"2022-01-01T00:00:00.000Z","threat":"example.com","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T00:25:20.010Z","impostorScore":0.0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":"Statement From (Trinity Groundwater)","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound","allow_relay"],"modulesRun":["av","zerohour","dkimv","spf","spam","dmarc"],"messageSize":111091,"headerFrom":"Laura Schumacher <abc@example.com>","headerReplyTo":null,"fromAddress":["abc@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["mail@example.com","abc@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"attached","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"image001.png","sandboxStatus":null,"oContentType":"image/png","contentType":"image/png"}],"completelyRewritten":false,"id":"8f12300-f387-1234-xxxx-a4abcd12347","QID":"0XX0XXXXaX3XXX-X1","GUID":"_pxxxxOxQxxXxx4wxjxtx2xxxTxxxYxxx","sender":"abc@example.com","recipient":["mailer-daemon@example.com"],"senderIP":"175.16.199.1","messageID":"<77F0EA74-7D6F-453A-AB7F-31B192481AE8@example.com>"}
 {"spamScore":100,"phishScore":100,"threatsInfoMap":[{"threatID":"9dhgabcdefghijkhgonvjdsdefghjikhlonvjdsvsbnvjdvjdsdefghjikhlonv","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/adhgon-vjdsdefghj-ikhlonvj-abcdefghij/threat/email/97921af132d1aa6a88fdbdadkhlonvjbc9fxxxxxxxxxxxxxxxxxxxxxbdadkhlonvjd","threatTime":"2022-01-01T03:02:25.092Z","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T00:00:00.000Z","impostorScore":0.0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":"(1) VOICE MAIL MESSSAGE","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound"],"modulesRun":["av","zerohour","dkimv","spf","spam","dmarc","pdr"],"messageSize":5776,"headerFrom":"VOICE MAIL<man.web@example.com>","headerReplyTo":null,"fromAddress":["man.web@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["mailer-daemon@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"}],"completelyRewritten":false,"id":"ee212323-1234-1234-1234-0f0abcd123456","QID":"3XXXf1XaXX-X1XX","GUID":"gxxxxxgxx3xcx-MxZxixxoxxxxxAxxx2","sender":"man.web@example.com","recipient":["mailer-daemon@example.com"],"senderIP":"175.16.199.1","messageID":"<20220327194933.12463F24B8AC1B73@example.com>"}
 {"spamScore":100,"phishScore":100,"threatsInfoMap":[{"threatID":"abcdefghijkabcdefghijkabcdefghijkefghjikhlonvjdsvsbnvjd","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/adhgonvj-dsdefgh-jikhlon-abcdefghij/threat/email/7921af132xxxxxxxxxxxxxxxxxxviuerhvuie35abcdefghabcdefghijk","threatTime":"2022-01-01T00:00:00.000Z","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T05:00:02.010Z","impostorScore":0.0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":"(1) VOICE MAIL MESSSAGE","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound"],"modulesRun":["av","zerohour","dkimv","spf","spam","dmarc","pdr"],"messageSize":5776,"headerFrom":"VOICE MAIL<man.web@example.com>","headerReplyTo":null,"fromAddress":["man.web@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["mailer-daemon@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"}],"completelyRewritten":false,"id":"ee212323-1234-1234-1234-0f0abcd123456","QID":"3XXfXabXcXXXX1","GUID":"gxxxxgx3xcx-xMx7xPxxZxxxxoxAx2xxxxx","sender":"man.web@example.com","recipient":["mailer-daemon@example.com"],"senderIP":"89.160.20.112","messageID":"<20220327194933.12463F24B8AC1B73@example.com>"}
+{"queryEndTime":"2024-10-11T14:34:53Z","messagesBlocked":[]}

packages/proofpoint_tap/data_stream/message_blocked/_dev/test/pipeline/test-message-blocked.log-expected.json

@@ -845,6 +845,7 @@
             "tags": [
                 "preserve_original_event"
             ]
-        }
+        },
+        null
     ]
 }

packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs

@@ -92,7 +92,7 @@ cursor:
     value: '[[.last_response.body.queryEndTime]]'
 response.split:
   target: body.messagesBlocked
-  ignore_empty_value: true
+  ignore_empty_value: false
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml

@@ -13,6 +13,8 @@ processors:
       field: event.original
       target_field: json
       ignore_failure: true
+  - drop:
+      if: ctx.json?.messagesBlocked instanceof List && ctx.json.messagesBlocked.length == 0
   - date:
       field: json.messageTime
       if: ctx.json?.messageTime != null && ctx.json.messageTime != ''

packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log

@@ -6,3 +6,4 @@
 {"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"6exxxxxxxxxxx123456xxxxxxxxxxx12345643cedfbbe1xxxxxxxxxxx123456b","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/3183a23b-d9c3-1234-1234-2babcd123408/threat/email/6e2eefdxxxxxxxxxxxxxxxxb3f43ceaafxxxxxxxxxxe5c91axxxbb","threatTime":"2022-04-01T23:14:30.450Z","threat":"https://example.com/view/8yxxxxvjxxxx5","campaignID":null,"threatType":"url"}],"messageTime":"2021-09-28T16:28:59.490Z","impostorScore":0,"malwareScore":0,"cluster":"example_hosted","subject":"RSVP today to Join Transpose Platform’s Ecosystem Days Summit","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["default_inbound"],"modulesRun":["av","zerohour","spf","dkimv","spam","pdr","urldefense"],"messageSize":2657297,"headerFrom":"abc.xyz@example.com","headerReplyTo":null,"fromAddress":["abc.xyz@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["abc.xyz@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"attached","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"image.png","sandboxStatus":null,"oContentType":"image/png","contentType":"image/png"}],"completelyRewritten":true,"id":"fbxxxxxx1-xxxxx123-xxxxx-xxxxx1234","QID":"2XX2XXOXFXXGX8X9X","GUID":"pxxxxvxxxxPxTxxxixxxxFxxxUxx2xxxxx","sender":"abc.xyz@example.com","recipient":["abc.xyz@example.com"],"senderIP":"175.16.199.1","messageID":"<CAXYZXYZxyz123_83rxxxXxXxXXk-+_TT_XxXxXxxQ@example.com>"}
 {"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"xxxxxxxxxxx12345678914xxxxxxxxxxx123456e9ff24a9xxxxxxxxxxx123456","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/3183a23b-d9c3-1234-1234-2babcd123408/threat/email/9f2dbcaa9xxxxxxxxxxe810d280xxxxxxxxxxxe48f6e69xxxxxxf","threatTime":"2022-04-01T12:48:03.852Z","threat":"https://example.com/view/xp45xxxxxxir9y","campaignID":null,"threatType":"url"}],"messageTime":"2022-08-17T18:00:22.060Z","impostorScore":0,"malwareScore":0,"cluster":"example_hosted","subject":"Speakers Announced | Ecosystem Days Summit","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["bypass_maxsize","default_inbound"],"modulesRun":["av","zerohour","spf","dkimv","spam","pdr","urldefense"],"messageSize":68353,"headerFrom":"Trang, Alex & Transpose Platform Team <client.services@example.com>","headerReplyTo":"Trang, Alex & Transpose Platform Team <client.services@example.com>","fromAddress":["client.services@example.com"],"ccAddresses":[],"replyToAddress":["client.services@example.com"],"toAddresses":["abc.xyz@example.com"],"xmailer":"Mailchimp Mailer - **CIDxxxxxxxxxx1234**","messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"}],"completelyRewritten":true,"id":"fxxxdxxa-xxxxx123-xxxxx-xxxxx1234","QID":"X2XXX0XXX2XX4","GUID":"wxxAxxxx8x8x5xxxxxJxPxxax7xxxxx","sender":"xyz-abc.us1_152023242.12345678-6xxxx123456789@example.com","recipient":["abc.xyz@example.com"],"senderIP":"175.16.199.1","messageID":"<200cxyz1234xyz1234bcb96f3.6xyz12345.202204125625899.736a993333.x12345678e@example.com>"}
 {"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"xxxxxxxxxxx123456xxxxxxxxxx1234xxxxxxxxxxx123456bbe1xxxxxx123456","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/3183a23b-d9c3-1234-1234-2babcd123408/threat/email/6e2eefd8cxxxxxxxxxeef270d0a1b3f43cexxxxxxxxx34abe5c91axxxcb","threatTime":"2022-04-01T20:56:13.000Z","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","campaignID":null,"threatType":"url"}],"messageTime":"2022-03-24T13:24:57.000Z","impostorScore":0,"malwareScore":0,"cluster":"example_hosted","subject":"RSVP today to Join Transpose Platform’s Ecosystem Days Summit","quarantineFolder":null,"quarantineRule":null,"policyRoutes":["bypass_maxsize","default_inbound"],"modulesRun":["av","zerohour","spf","dkimv","spam","pdr","urldefense"],"messageSize":2642117,"headerFrom":"abc.xyz@example.com","headerReplyTo":null,"fromAddress":["abc.xyz@example.com"],"ccAddresses":[],"replyToAddress":[],"toAddresses":["abc.xyz@example.com"],"xmailer":null,"messageParts":[{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.html","sandboxStatus":null,"oContentType":"text/html","contentType":"text/html"},{"disposition":"inline","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"text.txt","sandboxStatus":null,"oContentType":"text/plain","contentType":"text/plain"},{"disposition":"attached","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","md5":"b10a8db164e0754105b7a99be72e3fe5","filename":"image.png","sandboxStatus":null,"oContentType":"image/png","contentType":"image/png"}],"completelyRewritten":true,"id":"cxxxxbxxxb-xxxxx123-xxxxx-xxxxx1234","QID":"2XXX2X5XX5XX7","GUID":"gpxxx5xx2xHxxxJx7xxxxmx5xcxxxxxZ","sender":"abc.xyz@example.com","recipient":["abc.xyz@example.com"],"senderIP":"175.16.199.1","messageID":"<ABC-Y_xyz1-83rxxxXxXxXXk-N9==P_XxXxXxxQ@example.com>"}
+{"queryEndTime":"2024-10-11T14:34:53Z","messagesDelivered":[]}

packages/proofpoint_tap/data_stream/message_delivered/_dev/test/pipeline/test-message-delivered.log-expected.json

@@ -916,6 +916,7 @@
             "tags": [
                 "preserve_original_event"
             ]
-        }
+        },
+        null
     ]
 }