Org: Collect inactive email addresses daily and indicate delivery fai…

…lures for newsletter recipients

TYPE: Feature
LINK: ogc-1896

src/onegov/newsletter/models.py

@@ -183,6 +183,27 @@ def __table_args__(cls) -> tuple[Index, ...]:
     def subscription(self) -> Subscription:
         return Subscription(self, self.token)
 
+    @property
+    def is_inactive(self) -> bool:
+        """
+        Checks if the recipient's email address is marked as inactive.
+
+        Returns:
+            bool: True if the email address is marked as inactive, False
+            otherwise.
+        """
+        return self.meta.get('inactive', False)
+
+    def mark_inactive(self) -> None:
+        """
+        Marks the recipient's email address as inactive.
+
+        This method sets the 'inactive' flag in the recipient's metadata to
+        True. It is typically used when a bounce event causes the email
+        address to be deactivated by Postmark.
+        """
+        self.meta['inactive'] = True
+
 
 class Subscription:
     """ Adds subscription management to a recipient. """

src/onegov/org/cronjobs.py

@@ -1,20 +1,23 @@
 from __future__ import annotations
-
 from collections import OrderedDict
+
+import requests
 from babel.dates import get_month_names
 from datetime import datetime, timedelta
 from itertools import groupby
+
 from onegov.chat.collections import ChatCollection
 from onegov.chat.models import Chat
 from onegov.core.cache import lru_cache
-from onegov.core.orm import find_models
+from onegov.core.orm import find_models, Base
 from onegov.core.orm.mixins.publication import UTCPublicationMixin
 from onegov.core.templates import render_template
 from onegov.event import Occurrence, Event
 from onegov.file import FileCollection
-from onegov.form import FormSubmission, parse_form
+from onegov.form import FormSubmission, parse_form, Form
 from onegov.org.mail import send_ticket_mail
-from onegov.newsletter import Newsletter, NewsletterCollection
+from onegov.newsletter import (Newsletter, NewsletterCollection,
+                               RecipientCollection)
 from onegov.org import _, OrgApp
 from onegov.org.layout import DefaultMailLayout
 from onegov.org.models import (
@@ -42,9 +45,8 @@
 from typing import Any, TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Iterator
-    from onegov.core.orm import Base
     from onegov.core.types import RenderData
-    from onegov.form import Form
+
     from onegov.org.request import OrgRequest
 
 
@@ -726,3 +728,59 @@ def delete_content_marked_deletable(request: OrgRequest) -> None:
 
     if count:
         print(f'Cron: Deleted {count} expired deletable objects in db')
+
+
+@OrgApp.cronjob(hour=7, minute=0, timezone='Europe/Zurich')
+def update_newsletter_email_bounce_statistics(
+    request: OrgRequest
+) -> None:
+    # I choose hour=7 as the maximum time difference between Eastern Standard
+    # Time (EST) and Central European Summer Time (CEST) is 7 hours. This
+    # occurs when EST is observing standard time (UTC-5) and CEST is observing
+    # daylight saving time (UTC+2).
+    # Postmark uses EST in `fromdate` and `todate`, see
+    # https://postmarkapp.com/developer/api/bounce-api.
+
+    def get_postmark_token() -> str:
+        # read postmark token from the applications configuration
+        mail_config = request.app.mail
+        if mail_config:
+            mailer = mail_config.get('marketing', {}).get('mailer', None)
+            if mailer == 'postmark':
+                return mail_config.get('marketing', {}).get('token', '')
+
+        return ''
+
+    token = get_postmark_token()
+
+    recipients = RecipientCollection(request.session)
+    yesterday = utcnow() - timedelta(days=1)
+
+    try:
+        r = requests.get(
+            'https://api.postmarkapp.com/bounces?count=500&offset=0',
+            f'fromDate={yesterday.date()}&toDate='
+            f'{yesterday.date()}&inactive=true',
+            headers={
+                'Accept': 'application/json',
+                'X-Postmark-Server-Token': token
+        },
+            timeout=30
+        )
+        r.raise_for_status()
+        bounces = r.json().get('Bounces', [])
+    except requests.exceptions.HTTPError as http_err:
+        if r.status_code == 401:
+            raise RuntimeWarning(
+                f'Postmark API token is not set or invalid: {http_err}'
+            ) from None
+        else:
+            raise
+
+    for bounce in bounces:
+        email = bounce.get('Email', '')
+        inactive = bounce.get('Inactive', False)
+        recipient = recipients.by_address(email)
+
+        if recipient and inactive:
+            recipient.mark_inactive()

src/onegov/org/locale/de_CH/LC_MESSAGES/onegov.org.po

@@ -2809,6 +2809,9 @@ msgstr ""
 msgid "Delete content"
 msgstr "Inhalt löschen"
 
+msgid "Photo album. Will be shown at the end of content."
+msgstr "Fotoalbum. Wird am Ende des Inhalts angezeigt."
+
 msgid "Photo album"
 msgstr "Fotoalbum"
 
@@ -6237,6 +6240,9 @@ msgstr "Ein Konto wurde für Sie erstellt"
 msgid "The user was created successfully"
 msgstr "Der Benutzer wurde erfolgreich erstellt"
 
+msgid "This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list."
+msgstr "Dieser Empfänger hat Zustellungsfehler, einschließlich Hard Bounces, ungültige E-Mail-Adresse, Spam-Beschwerde, manuelle Deaktivierung oder wird blockiert. Wir empfehlen, ihn von der Liste abzumelden."
+
 #~ msgid "Describes in detail how this form is to be used"
 #~ msgstr "Beschreibt detailliert wie dieses Formular benutzt werden soll"
 

src/onegov/org/locale/fr_CH/LC_MESSAGES/onegov.org.po

@@ -2808,6 +2808,9 @@ msgstr ""
 msgid "Delete content"
 msgstr "Supprimer le contenu"
 
+msgid "Photo album. Will be shown at the end of content."
+msgstr "Album photo. Sera affiché à la fin du contenu."
+
 msgid "Photo album"
 msgstr "Album photo"
 
@@ -6253,5 +6256,8 @@ msgstr "Un compte a été créé pour vous"
 msgid "The user was created successfully"
 msgstr "L'utilisateur a bien été créé"
 
+msgid "This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list."
+msgstr "Ce destinataire a des échecs de livraison, y compris des rebonds, des adresses e-mail invalides, des plaintes pour spam, des désactivations manuelles, ou est bloqué. Nous recommandons de le désinscrire de la liste."
+
 #~ msgid "Photo album. Will be shown at the end of content."
 #~ msgstr "Album photo. Sera affiché à la fin du contenu."

src/onegov/org/locale/it_CH/LC_MESSAGES/onegov.org.po

@@ -2815,6 +2815,9 @@ msgstr ""
 msgid "Delete content"
 msgstr "Cancellare il contenuto"
 
+msgid "Photo album. Will be shown at the end of content."
+msgstr "Album fotografico. Sarà mostrato alla fine del contenuto."
+
 msgid "Photo album"
 msgstr "Album fotografico"
 
@@ -6241,5 +6244,8 @@ msgstr "È stato creato un account per te"
 msgid "The user was created successfully"
 msgstr "Utente creato correttamente"
 
+msgid "This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list."
+msgstr "Questo destinatario presenta errori di consegna, tra cui rimbalzi, indirizzi e-mail non validi, reclami per spam, disattivazioni manuali o blocco. Si consiglia di cancellarlo dall'elenco."
+
 #~ msgid "Photo album. Will be shown at the end of content."
 #~ msgstr "Album fotografico. Sarà mostrato alla fine del contenuto."

src/onegov/org/templates/recipients.pt

@@ -7,6 +7,10 @@
             <p i18n:translate="">No subscribers yet</p>
         </tal:b>
 
+        <tal:b condition="count">
+            <p i18n:translate>There are currently <tal:b i18n:name="count">${count}</tal:b> recipients registered.</p>
+        </tal:b>
+
         <tal:b tal:repeat="letter by_letter">
             <h2>${letter}</h2>
 
@@ -30,6 +34,9 @@
                        i18n:attributes="data-confirm-extra;data-confirm-yes;data-confirm-no">
                         unsubscribe
                     </a>
+                    <span tal:condition="recipient.is_inactive" class="small-text info-text" title="This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list." i18n:attributes="title">
+                        <i class="fa fa-exclamation-triangle"></i>
+                    </span>
                 </li>
             </ul>
         </tal:b>

src/onegov/org/views/newsletter.py

@@ -399,7 +399,7 @@ def view_subscribers(
         'layout': layout or RecipientLayout(self, request),
         'title': _('Subscribers'),
         'by_letter': by_letter,
-        'count': len(by_letter),
+        'count': recipients.count(),
         'warning': warning,
     }
 

src/onegov/town6/locale/de_CH/LC_MESSAGES/onegov.town6.po

@@ -2188,6 +2188,9 @@ msgstr ""
 msgid "New Note"
 msgstr "Neue Notiz"
 
+msgid "This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list."
+msgstr "Dieser Empfänger hat Zustellungsfehler, einschließlich Hard Bounces, ungültige E-Mail-Adresse, Spam-Beschwerde, manuelle Deaktivierung oder wird blockiert. Wir empfehlen, ihn von der Liste abzumelden."
+
 #~ msgid "Change Request"
 #~ msgstr "Änderung vorschlagen"
 

src/onegov/town6/locale/fr_CH/LC_MESSAGES/onegov.town6.po

@@ -2190,6 +2190,9 @@ msgstr ""
 msgid "New Note"
 msgstr "Nouveau commentaire"
 
+msgid "This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list."
+msgstr "Ce destinataire a des échecs de livraison, y compris des rebonds, des adresses e-mail invalides, des plaintes pour spam, des désactivations manuelles, ou est bloqué. Nous recommandons de le désinscrire de la liste."
+
 #~ msgid "Sign up to our newsletter to always stay up to date:"
 #~ msgstr ""
 #~ "Abonnez-vous à notre lettre d'informations pour rester au courant en "

src/onegov/town6/locale/it_CH/LC_MESSAGES/onegov.town6.po

@@ -2187,6 +2187,9 @@ msgstr ""
 msgid "New Note"
 msgstr "Nuova nota"
 
+msgid "This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list."
+msgstr "Questo destinatario presenta errori di consegna, tra cui rimbalzi, indirizzi e-mail non validi, reclami per spam, disattivazioni manuali o blocco. Si consiglia di cancellarlo dall'elenco."
+
 #~ msgid "Sign up to our newsletter to always stay up to date:"
 #~ msgstr "Iscriviti alla nostra newsletter per rimanere sempre aggiornato:"
 

src/onegov/town6/templates/recipients.pt

@@ -7,6 +7,10 @@
             <p i18n:translate="">No subscribers yet</p>
         </tal:b>
 
+        <tal:b condition="count">
+            <p i18n:translate>There are currently <tal:b i18n:name="count">${count}</tal:b> recipients registered.</p>
+        </tal:b>
+
         <tal:b tal:repeat="letter by_letter">
             <h2>${letter}</h2>
 
@@ -30,6 +34,9 @@
                        i18n:attributes="data-confirm-extra;data-confirm-yes;data-confirm-no">
                         unsubscribe
                     </a>
+                    <span tal:condition="recipient.is_inactive" class="small-text info-text" title="This recipient has delivery failures, including hard bounces, invalid email addresses, spam complaints, manual deactivations, or being blocked. We recommend unsubscribing it from the list." i18n:attributes="title">
+                        <i class="fa fa-exclamation-triangle"></i>
+                    </span>
                 </li>
             </ul>
         </tal:b>

tests/onegov/org/test_cronjobs.py

@@ -1,8 +1,10 @@
 import json
 import os
 from pathlib import Path
+from unittest.mock import patch, Mock
 
 import pytest
+import requests
 import transaction
 from datetime import datetime, timedelta
 from freezegun import freeze_time
@@ -1069,6 +1071,7 @@ def test_delete_content_marked_deletable__directory_entries(org_app, handlers):
         grundeigentumer_in='Berta Bertinio',
         publication_start=datetime(2024, 4, 1, tzinfo=tz),
         publication_end=datetime(2024, 4, 10, tzinfo=tz),
+        # delete_when_expired=True,
     ))
     event.delete_when_expired = True
 
@@ -1442,3 +1445,82 @@ def count_recipients():
         assert message['To'] == john.address
         assert sport_clubs().title in message['Subject']
         assert entry_2.name in message['TextBody']
+
+
+def test_update_newsletter_email_bounce_statistics(org_app, handlers):
+    register_echo_handler(handlers)
+    register_directory_handler(handlers)
+
+    # fake postmark mailer
+    org_app.mail['marketing']['mailer'] = 'postmark'
+
+    client = Client(org_app)
+    job = get_cronjob_by_name(org_app,
+                              'update_newsletter_email_bounce_statistics')
+    job.app = org_app
+    # tz = ensure_timezone('Europe/Zurich')
+
+    transaction.begin()
+
+    # create recipients Franz and Heinz
+    recipients = RecipientCollection(org_app.session())
+    recipients.add('[email protected]', confirmed=True)
+    recipients.add('[email protected]', confirmed=True)
+    recipients.add('[email protected]', confirmed=True)
+
+    transaction.commit()
+    close_all_sessions()
+
+    with patch('requests.get') as mock_get:
+        mock_get.return_value = Bunch(
+            status_code=200,
+            json=lambda: {
+                'TotalCount': 2,
+                'Bounces': [
+                    {'RecordType': 'Bounce', 'ID': 3719297970,
+                     'Inactive': False, 'Email': '[email protected]'},
+                    {'RecordType': 'Bounce', 'ID': 4739297971,
+                     'Inactive': True, 'Email': '[email protected]'}
+                ]
+            },
+            raise_for_status=Mock(return_value=None),
+        )
+
+        # execute cronjob
+        client.get(get_cronjob_url(job))
+
+        # check if the statistics are updated
+        assert mock_get.called
+        assert RecipientCollection(org_app.session()).by_address(
+            '[email protected]').is_inactive is False
+        assert RecipientCollection(org_app.session()).by_address(
+            '[email protected]').is_inactive is True
+        assert RecipientCollection(org_app.session()).by_address(
+            '[email protected]').is_inactive is False
+
+    # test raising runtime warning exception for status code 401
+    with patch('requests.get') as mock_get:
+        mock_get.return_value = Bunch(
+            status_code=401,
+            json=lambda: {},
+            raise_for_status=Mock(
+                side_effect=requests.exceptions.HTTPError('401 Unauthorized')),
+        )
+
+        # execute cronjob
+        with pytest.raises(RuntimeWarning):
+            client.get(get_cronjob_url(job))
+
+    # for other 30x and 40x status codes, the cronjob shall raise an exception
+    for status_code in [301, 302, 303, 400, 402, 403, 404, 405]:
+        with patch('requests.get') as mock_get:
+            mock_get.return_value = Bunch(
+                status_code=status_code,
+                json=lambda: {},
+                raise_for_status=Mock(
+                    side_effect=requests.exceptions.HTTPError()),
+            )
+
+            # execute cronjob
+            with pytest.raises(requests.exceptions.HTTPError):
+                client.get(get_cronjob_url(job))