From 61ea75c476567df553d1912fd390e08b6a40bb4f Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 15:40:09 +0000 Subject: [PATCH 01/12] Move output logic to separate module and class --- src/main.py | 117 ++++++++++--------------------------------------- src/outputs.py | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 93 deletions(-) create mode 100644 src/outputs.py diff --git a/src/main.py b/src/main.py index dacd1a3..42def35 100755 --- a/src/main.py +++ b/src/main.py @@ -1,87 +1,23 @@ #!/usr/bin/env python +from outputs import SendOutputByEmail import pdfkit import json from imap_tools import MailBox, AND, MailMessageFlags import os -import smtplib -from pathlib import Path -from email.mime.multipart import MIMEMultipart -from email.mime.base import MIMEBase -from email.mime.text import MIMEText -from email.utils import formatdate -from email import encoders - - -def send_mail( - send_from, - send_to, - subject, - message, - files=[], - server=None, - port=None, - username=None, - password=None, - use_tls=None, -): - """Compose and send email with provided info and attachments. - - Args: - send_from (str): from name - send_to (str): to name(s) - subject (str): message title - message (str): message body - files (list[str]): list of file paths to be attached to email - server (str): mail server host name - port (int): port number - username (str): server auth username - password (str): server auth password - use_tls (bool): use TLS mode - """ - msg = MIMEMultipart() - msg["From"] = send_from - msg["To"] = send_to - msg["Date"] = formatdate(localtime=True) - msg["Subject"] = subject - - msg.attach(MIMEText(message)) - - for path in files: - part = MIMEBase("application", "octet-stream") - with open(path, "rb") as file: - part.set_payload(file.read()) - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - "attachment", - filename=format(Path(path).name), - ) - msg.attach(part) - - smtp = smtplib.SMTP(server, port) - if use_tls: - smtp.starttls() - smtp.login(username, password) - smtp.sendmail(send_from, send_to, msg.as_string()) - smtp.quit() def process_mail( + output, mark_msg=True, num_emails_limit=50, imap_url=None, imap_username=None, imap_password=None, imap_folder=None, - mail_sender=None, - server_smtp=None, - smtp_tls=None, - smtp_port=None, - mail_destination=None, printfailedmessage=None, pdfkit_options=None, - mail_msg_flag=None, + mail_msg_flag=None ): print("Starting mail processing run", flush=True) if printfailedmessage: @@ -153,18 +89,7 @@ def process_mail( print(f"\nBody/HTML Above") raise e - send_mail( - mail_sender, - mail_destination, - f"{msg.subject}", - f"Converted PDF of email from {msg.from_} on {msg.date_str} wih topic {msg.subject}. Content below.\n\n\n\n{msg.text}", - files=[filename], - server=server_smtp, - username=imap_username, - password=imap_password, - port=smtp_port, - use_tls=smtp_tls, - ) + output.process(msg, [filename]) if mark_msg: flag = None @@ -194,6 +119,8 @@ def process_mail( password = os.environ.get("IMAP_PASSWORD") folder = os.environ.get("IMAP_FOLDER") + output_type = os.getenv('OUTPUT_TYPE', 'mailto') + server_smtp = os.environ.get("SMTP_URL") sender = os.environ.get("MAIL_SENDER") destination = os.environ.get("MAIL_DESTINATION") @@ -204,19 +131,23 @@ def process_mail( pdfkit_options = os.environ.get("WKHTMLTOPDF_OPTIONS") mail_msg_flag = os.environ.get("MAIL_MESSAGE_FLAG") + output=None + if output_type == 'mailto': + output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, username, password, smtp_tls) + + if not output: + raise ValueError("Unknown output type '{output_type}'") + print("Running emails-html-to-pdf") - process_mail( - imap_url=server_imap, - imap_username=username, - imap_password=password, - imap_folder=folder, - mail_sender=sender, - mail_destination=destination, - server_smtp=server_smtp, - printfailedmessage=printfailedmessage, - pdfkit_options=pdfkit_options, - smtp_tls=smtp_tls, - smtp_port=smtp_port, - mail_msg_flag=mail_msg_flag, - ) + with output: + process_mail( + output=output, + imap_url=server_imap, + imap_username=username, + imap_password=password, + imap_folder=folder, + printfailedmessage=printfailedmessage, + pdfkit_options=pdfkit_options, + mail_msg_flag=mail_msg_flag + ) diff --git a/src/outputs.py b/src/outputs.py new file mode 100644 index 0000000..e5711cd --- /dev/null +++ b/src/outputs.py @@ -0,0 +1,85 @@ + + +from abc import abstractmethod +from email import encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formatdate +from pathlib import Path +import smtplib + + +class AbstractOutput: + + @abstractmethod + def process(self, originalMessage, generatedPdfs): + """Process the output of the email conversion + + Args: + originalMessage (???): The original message which was converted + generatedPdfs (list[str]): A list of file paths to the PDFs generated from the email + """ + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, exc_traceback): + pass + +class SendOutputByEmail: + """Sends output to an email address + + Args: + mail_from (str): from name + mail_to (str): to name(s) + server (str): mail server host name + port (int): port number + username (str): server auth username + password (str): server auth password + use_tls (bool): use TLS mode + """ + + def __init__(self, mail_from, mail_to, server=None, port=None, username=None, password=None, use_tls=None): + self.__mail_from = mail_from + self.__mail_to = mail_to + self.__server = server + self.__port = port + self.__username = username + self.__password = password + self.__use_tls = use_tls + + def __enter__(self): + self.__smtp = smtplib.SMTP(self.__server, self.__port) + if self.__use_tls: + self.__smtp.starttls() + self.__smtp.login(self.__username, self.__password) + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.__smtp.quit() + + def process(self, originalMessage, generatedPdfs): + msg = MIMEMultipart() + msg["From"] = self.__mail_from + msg["To"] = self.__mail_to + msg["Date"] = formatdate(localtime=True) + msg["Subject"] = originalMessage.subject + + message = f"Converted PDF of email from {originalMessage.from_} on {originalMessage.date_str} wih topic {originalMessage.subject}. Content below.\n\n\n\n{originalMessage.text}" + msg.attach(MIMEText(message)) + + for path in generatedPdfs: + part = MIMEBase("application", "octet-stream") + with open(path, "rb") as file: + part.set_payload(file.read()) + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + "attachment", + filename=format(Path(path).name), + ) + msg.attach(part) + + self.__smtp.sendmail(self.__mail_from, self.__mail_to, msg.as_string()) From bff5231060fe6797b3b01f8f056257fe5ddd9aca Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 16:33:14 +0000 Subject: [PATCH 02/12] Add support for SSL/TLS encryption (not STARTTLS) --- src/main.py | 4 ++-- src/outputs.py | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/main.py b/src/main.py index 42def35..80d48f6 100755 --- a/src/main.py +++ b/src/main.py @@ -125,7 +125,7 @@ def process_mail( sender = os.environ.get("MAIL_SENDER") destination = os.environ.get("MAIL_DESTINATION") smtp_port = os.getenv("SMTP_PORT", 587) - smtp_tls = os.getenv("SMTP_TLS", True) + smtp_encryption = os.getenv("SMTP_ENCRYPTION", SendOutputByEmail.SMTP_ENCRYPTION_STARTTLS) printfailedmessage = os.getenv("PRINT_FAILED_MSG", "False") == "True" pdfkit_options = os.environ.get("WKHTMLTOPDF_OPTIONS") @@ -133,7 +133,7 @@ def process_mail( output=None if output_type == 'mailto': - output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, username, password, smtp_tls) + output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, username, password, smtp_encryption) if not output: raise ValueError("Unknown output type '{output_type}'") diff --git a/src/outputs.py b/src/outputs.py index e5711cd..421bf59 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -38,22 +38,29 @@ class SendOutputByEmail: port (int): port number username (str): server auth username password (str): server auth password - use_tls (bool): use TLS mode + encryption (str): Type of encryption to use (if any) """ + SMTP_ENCRYPTION_STARTTLS="STARTTLS" + SMTP_ENCRYPTION_SSL="SSL" - def __init__(self, mail_from, mail_to, server=None, port=None, username=None, password=None, use_tls=None): + def __init__(self, mail_from, mail_to, server, port, username, password, encryption): self.__mail_from = mail_from self.__mail_to = mail_to self.__server = server self.__port = port self.__username = username self.__password = password - self.__use_tls = use_tls + self.__encryption = encryption def __enter__(self): - self.__smtp = smtplib.SMTP(self.__server, self.__port) - if self.__use_tls: + if self.__encryption == self.SMTP_ENCRYPTION_SSL: + self.__smtp = smtplib.SMTP_SSL(self.__server, self.__port) + else: + self.__smtp = smtplib.SMTP(self.__server, self.__port) + + if self.__encryption == self.SMTP_ENCRYPTION_STARTTLS: self.__smtp.starttls() + self.__smtp.login(self.__username, self.__password) return self From b9accfc82d381f2ef3be83ef8214619a7f6ad88a Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 16:33:42 +0000 Subject: [PATCH 03/12] Add python defaults to gitignore --- .gitignore | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/.gitignore b/.gitignore index 10ba96e..a91dda2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,147 @@ ***.idea + + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# End of https://www.toptal.com/developers/gitignore/api/python From 311eff02e9e7dfb3549ed74965dc15b0487cea95 Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 16:47:20 +0000 Subject: [PATCH 04/12] Convert logging to standard python library --- src/main.py | 63 +++++++++++++++++++++++++++++++------------------- src/outputs.py | 11 +++++++++ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/main.py b/src/main.py index 80d48f6..4595e4a 100755 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import logging from outputs import SendOutputByEmail import pdfkit import json @@ -19,9 +20,9 @@ def process_mail( pdfkit_options=None, mail_msg_flag=None ): - print("Starting mail processing run", flush=True) + logging.info("Starting mail processing run") if printfailedmessage: - print("*On failure, the Body of the email will be printed*") + logging.warn("*On failure, the Body of the email will be printed*") PDF_CONTENT_ERRORS = [ "ContentNotFoundError", @@ -41,7 +42,7 @@ def process_mail( ) ): if len(msg.attachments) == 0: - print(f"\nNo attachments in: {msg.subject}") + logging.debug(f"\nNo attachments in: {msg.subject}") if not msg.html.strip() == "": # handle text only emails pdftext = ( '' @@ -50,10 +51,10 @@ def process_mail( else: pdftext = msg.text filename = f'{msg.subject.replace(".", "_").replace(" ", "-")[:50]}.pdf' - print(f"\nPDF: {filename}") + logging.debug(f"\nPDF: {filename}") for bad_char in ["/", "*", ":", "<", ">", "|", '"', "’", "–"]: filename = filename.replace(bad_char, "_") - print(f"\nPDF: {filename}") + logging.debug(f"\nPDF: {filename}") options = {} if pdfkit_options is not None: # parse WKHTMLTOPDF Options to dict @@ -61,32 +62,32 @@ def process_mail( try: pdfkit.from_string(pdftext, filename, options=options) except OSError as e: + outputMessage = "" if any([error in str(e) for error in PDF_CONTENT_ERRORS]): # allow pdfs with missing images if file got created if os.path.exists(filename): if printfailedmessage: - print(f"\n{pdftext}\n") - print(f"\n **** HANDLED EXCEPTION ****") - print(f"\n\n{str(e)}\n") - print( - f"\nError with images in file, continuing without them. Email Body/HTML Above" - ) + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n **** HANDLED EXCEPTION ****" + outputMessage += f"\n\n{str(e)}\n" + outputMessage += f"\nError with images in file, continuing without them. Email Body/HTML Above" + logging.warn(outputMessage) else: if printfailedmessage: - print(f"\n{pdftext}\n") - print( - f"\n !!!! UNHANDLED EXCEPTION with PDF Content Errors: {PDF_CONTENT_ERRORS} !!!!" - ) - print(f"\n{str(e)}") - print(f"\nBody/HTML Above") + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n !!!! UNHANDLED EXCEPTION with PDF Content Errors: {PDF_CONTENT_ERRORS} !!!!" + outputMessage += f"\n{str(e)}" + outputMessage += f"\nBody/HTML Above" + logging.error(outputMessage) raise e else: if printfailedmessage: - print(f"\n{pdftext}\n") - print(f"\n !!!! UNHANDLED EXCEPTION !!!!") - print(f"\n{str(e)}") - print(f"\nBody/HTML Above") + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n !!!! UNHANDLED EXCEPTION !!!!" + outputMessage += f"\n{str(e)}" + outputMessage += f"\nBody/HTML Above" + logging.error(outputMessage) raise e output.process(msg, [filename]) @@ -109,11 +110,25 @@ def process_mail( flag = MailMessageFlags.SEEN mailbox.flag(msg.uid, flag, True) os.remove(filename) - print("Completed mail processing run\n\n", flush=True) + logging.info("Completed mail processing run") if __name__ == "__main__": + log_level = os.environ.get("LOG_LEVEL", "INFO") + if log_level == 'DEBUG': + log_level = logging.DEBUG + elif log_level == 'INFO': + log_level = logging.INFO + elif log_level == 'WARN': + log_level = logging.WARN + elif log_level == 'ERROR': + log_level = logging.ERROR + else: + logging.warn(f"Unrecognised logging level '{log_level}'. Defaulting to INFO level.") + log_level = logging.INFO + logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=log_level) + server_imap = os.environ.get("IMAP_URL") username = os.environ.get("IMAP_USERNAME") password = os.environ.get("IMAP_PASSWORD") @@ -136,9 +151,9 @@ def process_mail( output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, username, password, smtp_encryption) if not output: - raise ValueError("Unknown output type '{output_type}'") + raise ValueError(f"Unknown output type '{output_type}'") - print("Running emails-html-to-pdf") + logging.info("Running emails-html-to-pdf") with output: process_mail( diff --git a/src/outputs.py b/src/outputs.py index 421bf59..fd759c1 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -8,6 +8,7 @@ from email.utils import formatdate from pathlib import Path import smtplib +import logging class AbstractOutput: @@ -43,6 +44,8 @@ class SendOutputByEmail: SMTP_ENCRYPTION_STARTTLS="STARTTLS" SMTP_ENCRYPTION_SSL="SSL" + __logger = logging.getLogger(__name__) + def __init__(self, mail_from, mail_to, server, port, username, password, encryption): self.__mail_from = mail_from self.__mail_to = mail_to @@ -53,19 +56,27 @@ def __init__(self, mail_from, mail_to, server, port, username, password, encrypt self.__encryption = encryption def __enter__(self): + self.__logger.info(f"Connecting to SMTP server {self.__server}:{self.__port}...") if self.__encryption == self.SMTP_ENCRYPTION_SSL: + self.__logger.debug(f"Using SSL encryption for SMTP") self.__smtp = smtplib.SMTP_SSL(self.__server, self.__port) else: self.__smtp = smtplib.SMTP(self.__server, self.__port) if self.__encryption == self.SMTP_ENCRYPTION_STARTTLS: + self.__logger.debug(f"Using STARTTLS encryption for SMTP") self.__smtp.starttls() + self.__logger.debug(f"Logging in to SMTP server as {self.__username}...") self.__smtp.login(self.__username, self.__password) + + self.__logger.info("SMTP setup successful") return self def __exit__(self, exc_type, exc_value, exc_traceback): + self.__logger.debug("Closing SMTP server connection...") self.__smtp.quit() + self.__logger.info("SMTP server closed gracefully") def process(self, originalMessage, generatedPdfs): msg = MIMEMultipart() From ea62b3c6ea627abaae8ad2c03e179a945b9a6b1a Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 17:09:17 +0000 Subject: [PATCH 05/12] Improve logging --- src/main.py | 142 ++++++++++++++++++++++++++----------------------- src/outputs.py | 3 ++ 2 files changed, 79 insertions(+), 66 deletions(-) diff --git a/src/main.py b/src/main.py index 4595e4a..9d91da4 100755 --- a/src/main.py +++ b/src/main.py @@ -22,7 +22,7 @@ def process_mail( ): logging.info("Starting mail processing run") if printfailedmessage: - logging.warn("*On failure, the Body of the email will be printed*") + logging.warning("*On failure, the Body of the email will be printed*") PDF_CONTENT_ERRORS = [ "ContentNotFoundError", @@ -41,75 +41,85 @@ def process_mail( mark_seen=False, ) ): - if len(msg.attachments) == 0: - logging.debug(f"\nNo attachments in: {msg.subject}") - if not msg.html.strip() == "": # handle text only emails - pdftext = ( - '' - + msg.html - ) - else: - pdftext = msg.text - filename = f'{msg.subject.replace(".", "_").replace(" ", "-")[:50]}.pdf' - logging.debug(f"\nPDF: {filename}") - for bad_char in ["/", "*", ":", "<", ">", "|", '"', "’", "–"]: - filename = filename.replace(bad_char, "_") - logging.debug(f"\nPDF: {filename}") - options = {} - if pdfkit_options is not None: - # parse WKHTMLTOPDF Options to dict - options = json.loads(pdfkit_options) - try: - pdfkit.from_string(pdftext, filename, options=options) - except OSError as e: - outputMessage = "" - if any([error in str(e) for error in PDF_CONTENT_ERRORS]): - # allow pdfs with missing images if file got created - if os.path.exists(filename): - if printfailedmessage: - outputMessage += f"\n{pdftext}\n" - outputMessage += f"\n **** HANDLED EXCEPTION ****" - outputMessage += f"\n\n{str(e)}\n" - outputMessage += f"\nError with images in file, continuing without them. Email Body/HTML Above" - logging.warn(outputMessage) - - else: - if printfailedmessage: - outputMessage += f"\n{pdftext}\n" - outputMessage += f"\n !!!! UNHANDLED EXCEPTION with PDF Content Errors: {PDF_CONTENT_ERRORS} !!!!" - outputMessage += f"\n{str(e)}" - outputMessage += f"\nBody/HTML Above" - logging.error(outputMessage) - raise e + if len(msg.attachments) != 0: + logging.warning(f"Attachments found in {msg.subject}. Messages with attachments cannot be converted to PDF. Skipping.") + continue + + if not msg.html.strip() == "": # handle text only emails + logging.debug(f"Message '{msg.subject}' is HTML") + pdftext = ( + '' + + msg.html + ) + else: + logging.debug(f"Message '{msg.subject}' is plain text") + pdftext = msg.text + + filename = f'{msg.subject.replace(".", "_").replace(" ", "-")[:50]}.pdf' + for bad_char in ["/", "*", ":", "<", ">", "|", '"', "’", "–"]: + filename = filename.replace(bad_char, "_") + logging.debug(f"Using '{filename}' for PDF filename") + + logging.info(f"Exporting message '{msg.subject}' to PDF") + options = {} + if pdfkit_options is not None: + # parse WKHTMLTOPDF Options to dict + options = json.loads(pdfkit_options) + try: + pdfkit.from_string(pdftext, filename, options=options) + except OSError as e: + outputMessage = "" + if any([error in str(e) for error in PDF_CONTENT_ERRORS]): + # allow pdfs with missing images if file got created + if os.path.exists(filename): + if printfailedmessage: + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n **** HANDLED EXCEPTION ****" + outputMessage += f"\n\n{str(e)}\n" + outputMessage += f"\nOne or more remote resources failed to load, continuing without them." + logging.warning(outputMessage) + else: if printfailedmessage: outputMessage += f"\n{pdftext}\n" - outputMessage += f"\n !!!! UNHANDLED EXCEPTION !!!!" + outputMessage += f"\n !!!! UNHANDLED EXCEPTION with PDF Content Errors: {PDF_CONTENT_ERRORS} !!!!" outputMessage += f"\n{str(e)}" - outputMessage += f"\nBody/HTML Above" logging.error(outputMessage) raise e + else: + if printfailedmessage: + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n !!!! UNHANDLED EXCEPTION !!!!" + outputMessage += f"\n{str(e)}" + logging.error(outputMessage) + raise e + + output.process(msg, [filename]) + + if mark_msg: + flag = None + if mail_msg_flag == "SEEN": + flag = MailMessageFlags.SEEN + elif mail_msg_flag == "ANSWERED": + flag = MailMessageFlags.ANSWERED + elif mail_msg_flag == "FLAGGED": + flag = MailMessageFlags.FLAGGED + elif mail_msg_flag == "DELETED": + flag = MailMessageFlags.DELETED + elif mail_msg_flag == "DRAFT": + flag = MailMessageFlags.DRAFT + elif mail_msg_flag == "RECENT": + flag = MailMessageFlags.RECENT + else: + logging.warning(f"Unrecognised message flag '{mail_msg_flag}'. Using 'SEEN' instead.") + flag = MailMessageFlags.SEEN + logging.info(f"Marking processed message as '{mail_msg_flag}'") + mailbox.flag(msg.uid, flag, True) + + logging.debug(f"Deleting processed PDF '{filename}'...") + os.remove(filename) + logging.info(f"Finished processing of message '{msg.subject}'") - output.process(msg, [filename]) - - if mark_msg: - flag = None - if mail_msg_flag == "SEEN": - flag = MailMessageFlags.SEEN - elif mail_msg_flag == "ANSWERED": - flag = MailMessageFlags.ANSWERED - elif mail_msg_flag == "FLAGGED": - flag = MailMessageFlags.FLAGGED - elif mail_msg_flag == "DELETED": - flag = MailMessageFlags.DELETED - elif mail_msg_flag == "DRAFT": - flag = MailMessageFlags.DRAFT - elif mail_msg_flag == "RECENT": - flag = MailMessageFlags.RECENT - else: - flag = MailMessageFlags.SEEN - mailbox.flag(msg.uid, flag, True) - os.remove(filename) logging.info("Completed mail processing run") @@ -120,12 +130,12 @@ def process_mail( log_level = logging.DEBUG elif log_level == 'INFO': log_level = logging.INFO - elif log_level == 'WARN': - log_level = logging.WARN + elif log_level == 'WARNING': + log_level = logging.WARNING elif log_level == 'ERROR': log_level = logging.ERROR else: - logging.warn(f"Unrecognised logging level '{log_level}'. Defaulting to INFO level.") + logging.warning(f"Unrecognised logging level '{log_level}'. Defaulting to INFO level.") log_level = logging.INFO logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=log_level) diff --git a/src/outputs.py b/src/outputs.py index fd759c1..ea40926 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -79,6 +79,7 @@ def __exit__(self, exc_type, exc_value, exc_traceback): self.__logger.info("SMTP server closed gracefully") def process(self, originalMessage, generatedPdfs): + logging.debug(f"Building output email for '{originalMessage.subject}'...") msg = MIMEMultipart() msg["From"] = self.__mail_from msg["To"] = self.__mail_to @@ -100,4 +101,6 @@ def process(self, originalMessage, generatedPdfs): ) msg.attach(part) + logging.info(f"Sending PDF output for '{originalMessage.subject}' to '{self.__mail_to}...") self.__smtp.sendmail(self.__mail_from, self.__mail_to, msg.as_string()) + logging.info(f"Sent PDF output for '{originalMessage.subject}' to '{self.__mail_to}...") From e713521e74a0c88fedf4b2690e46b4d28ce87f42 Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 17:42:22 +0000 Subject: [PATCH 06/12] Add folder output type --- src/filenameutils.py | 22 ++++++++++++++++++++++ src/main.py | 11 +++++++---- src/outputs.py | 36 ++++++++++++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 src/filenameutils.py diff --git a/src/filenameutils.py b/src/filenameutils.py new file mode 100644 index 0000000..cb1c9a8 --- /dev/null +++ b/src/filenameutils.py @@ -0,0 +1,22 @@ + +BAD_CHARS=["/", "*", ":", "<", ">", "|", '"', "’", "–"] + +def replace_bad_chars(string, replace_char='_'): + """Replaces characters in the given string which are not valid for some filesystems. + + Args: + string (str): The string to perfom the replacement on + replace_char (char): The character to replace the bad characters with + """ + for char in BAD_CHARS: + string = string.replace(char, replace_char) + return string + +def replace_unpleasant_chars(string): + """Replaces characters which are considered unpleasant in filenames (space and period). + + Args: + string (str): The string to perfom the replacement on + """ + return string.replace(".", "_").replace(" ", "-") + diff --git a/src/main.py b/src/main.py index 9d91da4..a113939 100755 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,8 @@ #!/usr/bin/env python import logging -from outputs import SendOutputByEmail +from filenameutils import replace_bad_chars, replace_unpleasant_chars +from outputs import OutputToFolder, SendOutputByEmail import pdfkit import json from imap_tools import MailBox, AND, MailMessageFlags @@ -55,9 +56,8 @@ def process_mail( logging.debug(f"Message '{msg.subject}' is plain text") pdftext = msg.text - filename = f'{msg.subject.replace(".", "_").replace(" ", "-")[:50]}.pdf' - for bad_char in ["/", "*", ":", "<", ">", "|", '"', "’", "–"]: - filename = filename.replace(bad_char, "_") + filename = replace_bad_chars(replace_unpleasant_chars(msg.subject)) + filename = f"{filename[:50]}.pdf" logging.debug(f"Using '{filename}' for PDF filename") logging.info(f"Exporting message '{msg.subject}' to PDF") @@ -159,6 +159,9 @@ def process_mail( output=None if output_type == 'mailto': output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, username, password, smtp_encryption) + elif output_type == 'folder': + output_folder = os.getenv("OUTPUT_FOLDER") + output=OutputToFolder(output_folder) if not output: raise ValueError(f"Unknown output type '{output_type}'") diff --git a/src/outputs.py b/src/outputs.py index ea40926..5ea84a7 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -6,12 +6,18 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate +import os from pathlib import Path import smtplib import logging +import shutil +from filenameutils import replace_bad_chars, replace_unpleasant_chars -class AbstractOutput: + +class OutputProcessor: + + _logger = logging.getLogger(__name__) @abstractmethod def process(self, originalMessage, generatedPdfs): @@ -29,7 +35,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_traceback): pass -class SendOutputByEmail: +class SendOutputByEmail(OutputProcessor): """Sends output to an email address Args: @@ -44,8 +50,6 @@ class SendOutputByEmail: SMTP_ENCRYPTION_STARTTLS="STARTTLS" SMTP_ENCRYPTION_SSL="SSL" - __logger = logging.getLogger(__name__) - def __init__(self, mail_from, mail_to, server, port, username, password, encryption): self.__mail_from = mail_from self.__mail_to = mail_to @@ -104,3 +108,27 @@ def process(self, originalMessage, generatedPdfs): logging.info(f"Sending PDF output for '{originalMessage.subject}' to '{self.__mail_to}...") self.__smtp.sendmail(self.__mail_from, self.__mail_to, msg.as_string()) logging.info(f"Sent PDF output for '{originalMessage.subject}' to '{self.__mail_to}...") + +class OutputToFolder(OutputProcessor): + + def __init__(self, output_folder): + self.__output_folder = output_folder + + def process(self, originalMessage, generatedPdfs): + logging.debug(f"Copying output for '{originalMessage.subject}' to output folder...") + output_base_name = f"{originalMessage.date.strftime('%Y%m%d%H%M%S')}_{originalMessage.subject}" + output_base_name = replace_bad_chars(replace_unpleasant_chars(output_base_name)) + output_base_name = f"{output_base_name[:50]}" + + if len(generatedPdfs) == 1: + self._output_file(generatedPdfs[0], f"{output_base_name}.pdf") + else: + for i, file in enumerate(generatedPdfs): + self._output_file(file, f"{output_base_name}_{i}.pdf") + logging.info(f"Finished copying output for '{originalMessage.subject}' to output folder") + + def _output_file(self, source, destination): + full_destination = os.path.join(self.__output_folder, destination) + logging.debug(f"Copying file '{source}' to '{full_destination}'...") + shutil.copyfile(source, full_destination) + logging.debug(f"Copied file '{source}' to '{full_destination}'") \ No newline at end of file From 7d3e3fc747829870835a3bb0baeabc2bc35db379 Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 17:53:18 +0000 Subject: [PATCH 07/12] Add failure threshold so that one error does not stop processing --- src/main.py | 164 ++++++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 74 deletions(-) diff --git a/src/main.py b/src/main.py index a113939..e664178 100755 --- a/src/main.py +++ b/src/main.py @@ -8,7 +8,6 @@ from imap_tools import MailBox, AND, MailMessageFlags import os - def process_mail( output, mark_msg=True, @@ -19,7 +18,8 @@ def process_mail( imap_folder=None, printfailedmessage=None, pdfkit_options=None, - mail_msg_flag=None + mail_msg_flag=None, + failed_messages_threshold=3 ): logging.info("Starting mail processing run") if printfailedmessage: @@ -34,6 +34,8 @@ def process_mail( "Server refused a stream", ] + failed_messages = 0 + with MailBox(imap_url).login(imap_username, imap_password, imap_folder) as mailbox: for i, msg in enumerate( mailbox.fetch( @@ -42,85 +44,99 @@ def process_mail( mark_seen=False, ) ): - if len(msg.attachments) != 0: - logging.warning(f"Attachments found in {msg.subject}. Messages with attachments cannot be converted to PDF. Skipping.") - continue - - if not msg.html.strip() == "": # handle text only emails - logging.debug(f"Message '{msg.subject}' is HTML") - pdftext = ( - '' - + msg.html - ) - else: - logging.debug(f"Message '{msg.subject}' is plain text") - pdftext = msg.text - - filename = replace_bad_chars(replace_unpleasant_chars(msg.subject)) - filename = f"{filename[:50]}.pdf" - logging.debug(f"Using '{filename}' for PDF filename") - - logging.info(f"Exporting message '{msg.subject}' to PDF") - options = {} - if pdfkit_options is not None: - # parse WKHTMLTOPDF Options to dict - options = json.loads(pdfkit_options) try: - pdfkit.from_string(pdftext, filename, options=options) - except OSError as e: - outputMessage = "" - if any([error in str(e) for error in PDF_CONTENT_ERRORS]): - # allow pdfs with missing images if file got created - if os.path.exists(filename): - if printfailedmessage: - outputMessage += f"\n{pdftext}\n" - outputMessage += f"\n **** HANDLED EXCEPTION ****" - outputMessage += f"\n\n{str(e)}\n" - outputMessage += f"\nOne or more remote resources failed to load, continuing without them." - logging.warning(outputMessage) - + if len(msg.attachments) != 0: + logging.warning(f"Attachments found in {msg.subject}. Messages with attachments cannot be converted to PDF. Skipping.") + continue + + if not msg.html.strip() == "": # handle text only emails + logging.debug(f"Message '{msg.subject}' is HTML") + pdftext = ( + '' + + msg.html + ) + else: + logging.debug(f"Message '{msg.subject}' is plain text") + pdftext = msg.text + + filename = replace_bad_chars(replace_unpleasant_chars(msg.subject)) + filename = f"{filename[:50]}.pdf" + logging.debug(f"Using '{filename}' for PDF filename") + + logging.info(f"Exporting message '{msg.subject}' to PDF") + options = {} + if pdfkit_options is not None: + # parse WKHTMLTOPDF Options to dict + options = json.loads(pdfkit_options) + try: + pdfkit.from_string(pdftext, filename, options=options) + except OSError as e: + outputMessage = "" + if any([error in str(e) for error in PDF_CONTENT_ERRORS]): + # allow pdfs with missing images if file got created + if os.path.exists(filename): + if printfailedmessage: + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n **** HANDLED EXCEPTION ****" + outputMessage += f"\n\n{str(e)}\n" + outputMessage += f"\nOne or more remote resources failed to load, continuing without them." + logging.warning(outputMessage) + + else: + if printfailedmessage: + outputMessage += f"\n{pdftext}\n" + outputMessage += f"\n !!!! UNHANDLED EXCEPTION with PDF Content Errors: {PDF_CONTENT_ERRORS} !!!!" + outputMessage += f"\n{str(e)}" + logging.error(outputMessage) + raise e else: if printfailedmessage: outputMessage += f"\n{pdftext}\n" - outputMessage += f"\n !!!! UNHANDLED EXCEPTION with PDF Content Errors: {PDF_CONTENT_ERRORS} !!!!" + outputMessage += f"\n !!!! UNHANDLED EXCEPTION !!!!" outputMessage += f"\n{str(e)}" logging.error(outputMessage) raise e - else: - if printfailedmessage: - outputMessage += f"\n{pdftext}\n" - outputMessage += f"\n !!!! UNHANDLED EXCEPTION !!!!" - outputMessage += f"\n{str(e)}" - logging.error(outputMessage) - raise e - - output.process(msg, [filename]) - - if mark_msg: - flag = None - if mail_msg_flag == "SEEN": - flag = MailMessageFlags.SEEN - elif mail_msg_flag == "ANSWERED": - flag = MailMessageFlags.ANSWERED - elif mail_msg_flag == "FLAGGED": - flag = MailMessageFlags.FLAGGED - elif mail_msg_flag == "DELETED": - flag = MailMessageFlags.DELETED - elif mail_msg_flag == "DRAFT": - flag = MailMessageFlags.DRAFT - elif mail_msg_flag == "RECENT": - flag = MailMessageFlags.RECENT - else: - logging.warning(f"Unrecognised message flag '{mail_msg_flag}'. Using 'SEEN' instead.") - flag = MailMessageFlags.SEEN - logging.info(f"Marking processed message as '{mail_msg_flag}'") - mailbox.flag(msg.uid, flag, True) - - logging.debug(f"Deleting processed PDF '{filename}'...") - os.remove(filename) - logging.info(f"Finished processing of message '{msg.subject}'") - - logging.info("Completed mail processing run") + + output.process(msg, [filename]) + + if mark_msg: + flag = None + if mail_msg_flag == "SEEN": + flag = MailMessageFlags.SEEN + elif mail_msg_flag == "ANSWERED": + flag = MailMessageFlags.ANSWERED + elif mail_msg_flag == "FLAGGED": + flag = MailMessageFlags.FLAGGED + elif mail_msg_flag == "DELETED": + flag = MailMessageFlags.DELETED + elif mail_msg_flag == "DRAFT": + flag = MailMessageFlags.DRAFT + elif mail_msg_flag == "RECENT": + flag = MailMessageFlags.RECENT + else: + logging.warning(f"Unrecognised message flag '{mail_msg_flag}'. Using 'SEEN' instead.") + flag = MailMessageFlags.SEEN + logging.info(f"Marking processed message as '{mail_msg_flag}'") + mailbox.flag(msg.uid, flag, True) + + logging.debug(f"Deleting processed PDF '{filename}'...") + os.remove(filename) + logging.info(f"Finished processing of message '{msg.subject}'") + except Exception as e: + logging.exception(str(e)) + failed_messages += 1 + + if failed_messages >= failed_messages_threshold: + errorMessage = f"The number of errors has reached the failed messages threshold. Processing will be halted. Please resolve issues before resuming." + logging.critical(errorMessage) + raise RuntimeError(errorMessage) + + logging.info("Continuing with next message") + + if failed_messages > 0: + logging.warn("Completed mail processing run with one or more errors") + else: + logging.info("Completed mail processing run") if __name__ == "__main__": From 470132a223f239d538ce2b8e0cddd40ee8567095 Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 21:25:04 +0000 Subject: [PATCH 08/12] Refactor argument discovery and add missing arguments --- src/main.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main.py b/src/main.py index e664178..00011d4 100755 --- a/src/main.py +++ b/src/main.py @@ -156,25 +156,26 @@ def process_mail( logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=log_level) server_imap = os.environ.get("IMAP_URL") - username = os.environ.get("IMAP_USERNAME") - password = os.environ.get("IMAP_PASSWORD") + imap_username = os.environ.get("IMAP_USERNAME") + imap_password = os.environ.get("IMAP_PASSWORD") folder = os.environ.get("IMAP_FOLDER") output_type = os.getenv('OUTPUT_TYPE', 'mailto') - server_smtp = os.environ.get("SMTP_URL") - sender = os.environ.get("MAIL_SENDER") - destination = os.environ.get("MAIL_DESTINATION") - smtp_port = os.getenv("SMTP_PORT", 587) - smtp_encryption = os.getenv("SMTP_ENCRYPTION", SendOutputByEmail.SMTP_ENCRYPTION_STARTTLS) - printfailedmessage = os.getenv("PRINT_FAILED_MSG", "False") == "True" pdfkit_options = os.environ.get("WKHTMLTOPDF_OPTIONS") mail_msg_flag = os.environ.get("MAIL_MESSAGE_FLAG") output=None if output_type == 'mailto': - output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, username, password, smtp_encryption) + server_smtp = os.environ.get("SMTP_SERVER") + smtp_username = os.environ.get("SMTP_USERNAME", imap_username) + smtp_password = os.environ.get("SMTP_PASSWORD", imap_password) + sender = os.environ.get("MAIL_SENDER", smtp_username) + destination = os.environ.get("MAIL_DESTINATION") + smtp_port = os.environ.get("SMTP_PORT", 587) + smtp_encryption = os.environ.get("SMTP_ENCRYPTION", SendOutputByEmail.SMTP_ENCRYPTION_STARTTLS) + output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, smtp_username, smtp_password, smtp_encryption) elif output_type == 'folder': output_folder = os.getenv("OUTPUT_FOLDER") output=OutputToFolder(output_folder) @@ -188,8 +189,8 @@ def process_mail( process_mail( output=output, imap_url=server_imap, - imap_username=username, - imap_password=password, + imap_username=imap_username, + imap_password=imap_password, imap_folder=folder, printfailedmessage=printfailedmessage, pdfkit_options=pdfkit_options, From bda6bf6a88859e5d0b526df79dde467807a33244 Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 21:25:16 +0000 Subject: [PATCH 09/12] Update readme with new arguments --- README.md | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index afbd3a1..fb9c3ec 100644 --- a/README.md +++ b/README.md @@ -19,17 +19,42 @@ The following parameters are used (defaults in parentheses): * `IMAP_USERNAME` * `IMAP_PASSWORD` * `IMAP_FOLDER` Which folder to watch for unread emails -* `SMTP_URL` -* `MAIL_SENDER`: Address the mail with pdf should be sent from -* `MAIL_DESTINATION`: Where to send the resulting pdf -* `SMTP_PORT`: (587) -* `SMTP_TLS`: (True) * `INTER_RUN_INTERVAL`: Time in seconds that the system should wait between running the script * `PRINT_FAILED_MSG`: Flag to control printing of error messages * `HOSTS`: [Semicolon separated list of hosts](https://github.com/rob-luke/emails-html-to-pdf/pull/12) that should be added to /etc/hosts to prevent dns lookup failures * `WKHTMLTOPDF_OPTIONS`: Python dict (json) representation of wkhtmltopdf_options that can be passed to the used pdfkit library * `MAIL_MESSAGE_FLAG`: Flag to apply to email after processing. Must be one of [imap-tools flags](https://github.com/ikvk/imap_tools/blob/7f8fd5e4f3976bbd2efa507843c577affa61d996/imap_tools/consts.py#L10). Values: SEEN (default), ANSWERED, FLAGGED, DELETED, DRAFT, RECENT +* `OUTPUT_TYPE`: (`mailto`) See Outputs section below. + +### Outputs + +#### Send Email (Default) + +Output Type: `mailto` + +This output will send the generated PDF files attached as an email. + +| Argument | Default | Description | +|---|---|---| +| `SMTP_SERVER` | | The address of the SMTP server. | +| `SMTP_PORT` | 587 | The port number for the SMTP server. | +| `SMTP_USERNAME` | Same as IMAP | The username to use for authentication. | +| `SMTP_PASSWORD` | Same as IMAP | The password to use for authentication. | +| `SMTP_ENCRYPTION` | `STARTTLS` | The encryption used for the SMTP connection. Valid options are `STARTTLS` and `SSL`. All other values will attempt to connect without encryption. | +| `MAIL_SENDER` | SMTP Username | The address which mail should be sent from. This can be in either `user@domain.com` or `Name ` format. | +| `MAIL_DESTINATION` | | The address which mail should be sent to. This can be in either `user@domain.com` or `Name ` format. | +| `SMTP_URL` | | Deprecated. Use `SMTP_SERVER` instead. | + +#### Output to Folder + +Output Type: `folder` + +This output will copy the generate PDFs into the specified folder. + +| Argument | Default | Description | +|---|---|---| +| `OUTPUT_FOLDER` | | The folder to output PDFs to. This can be either relative or absolute. Paths are relative to the working directory. | ### Docker-Compose @@ -88,3 +113,17 @@ poetry run src/main.py * add `{"load-media-error-handling":"ignore"}` as `WKHTMLTOPDF_OPTIONS` option (could be the tracking pixel that is not beeing loaded * append `"enable-local-file-access":true` or `"load-error-handling":"ignore"`to `WKHTMLTOPDF_OPTIONS` if you get a `file://...` error * add `127.0.0.1 true` to the `HOSTS` env if you get a `http:///true/...` error + +## Development + +The recommended editor for development is either IntelliJ or Visual Studio Code + +### Visual Studio Code + +For Visual Studio Code, it is recommended to use the devcontainer included in the repository. With the +[Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) +extension installed, you should be prompted to open the devcontainer when opening the folder. + +For debugging, copy the `env.example` file and rename it to just `env`. Then edit the variables inside +to the required values for testing. These will be automatically configured when launching via either the +debug menu or by pressing F5. The `env` file is included in the gitignore. \ No newline at end of file From 72af0edfb5593fa7e6831aa119b4891fd1c2832d Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 21:39:39 +0000 Subject: [PATCH 10/12] Fix formatting issues --- src/filenameutils.py | 10 ++++---- src/main.py | 55 +++++++++++++++++++++++++++++--------------- src/outputs.py | 42 ++++++++++++++++++++++----------- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/src/filenameutils.py b/src/filenameutils.py index cb1c9a8..35a1224 100644 --- a/src/filenameutils.py +++ b/src/filenameutils.py @@ -1,9 +1,9 @@ +BAD_CHARS = ["/", "*", ":", "<", ">", "|", '"', "’", "–"] -BAD_CHARS=["/", "*", ":", "<", ">", "|", '"', "’", "–"] -def replace_bad_chars(string, replace_char='_'): +def replace_bad_chars(string, replace_char="_"): """Replaces characters in the given string which are not valid for some filesystems. - + Args: string (str): The string to perfom the replacement on replace_char (char): The character to replace the bad characters with @@ -12,11 +12,11 @@ def replace_bad_chars(string, replace_char='_'): string = string.replace(char, replace_char) return string + def replace_unpleasant_chars(string): """Replaces characters which are considered unpleasant in filenames (space and period). - + Args: string (str): The string to perfom the replacement on """ return string.replace(".", "_").replace(" ", "-") - diff --git a/src/main.py b/src/main.py index 00011d4..ad56042 100755 --- a/src/main.py +++ b/src/main.py @@ -8,6 +8,7 @@ from imap_tools import MailBox, AND, MailMessageFlags import os + def process_mail( output, mark_msg=True, @@ -19,7 +20,7 @@ def process_mail( printfailedmessage=None, pdfkit_options=None, mail_msg_flag=None, - failed_messages_threshold=3 + failed_messages_threshold=3, ): logging.info("Starting mail processing run") if printfailedmessage: @@ -46,7 +47,9 @@ def process_mail( ): try: if len(msg.attachments) != 0: - logging.warning(f"Attachments found in {msg.subject}. Messages with attachments cannot be converted to PDF. Skipping.") + logging.warning( + f"Attachments found in {msg.subject}. Messages with attachments cannot be converted to PDF. Skipping." + ) continue if not msg.html.strip() == "": # handle text only emails @@ -114,11 +117,13 @@ def process_mail( elif mail_msg_flag == "RECENT": flag = MailMessageFlags.RECENT else: - logging.warning(f"Unrecognised message flag '{mail_msg_flag}'. Using 'SEEN' instead.") + logging.warning( + f"Unrecognised message flag '{mail_msg_flag}'. Using 'SEEN' instead." + ) flag = MailMessageFlags.SEEN logging.info(f"Marking processed message as '{mail_msg_flag}'") mailbox.flag(msg.uid, flag, True) - + logging.debug(f"Deleting processed PDF '{filename}'...") os.remove(filename) logging.info(f"Finished processing of message '{msg.subject}'") @@ -142,43 +147,57 @@ def process_mail( if __name__ == "__main__": log_level = os.environ.get("LOG_LEVEL", "INFO") - if log_level == 'DEBUG': + if log_level == "DEBUG": log_level = logging.DEBUG - elif log_level == 'INFO': + elif log_level == "INFO": log_level = logging.INFO - elif log_level == 'WARNING': + elif log_level == "WARNING": log_level = logging.WARNING - elif log_level == 'ERROR': + elif log_level == "ERROR": log_level = logging.ERROR else: - logging.warning(f"Unrecognised logging level '{log_level}'. Defaulting to INFO level.") + logging.warning( + f"Unrecognised logging level '{log_level}'. Defaulting to INFO level." + ) log_level = logging.INFO - logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=log_level) + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=log_level + ) server_imap = os.environ.get("IMAP_URL") imap_username = os.environ.get("IMAP_USERNAME") imap_password = os.environ.get("IMAP_PASSWORD") folder = os.environ.get("IMAP_FOLDER") - output_type = os.getenv('OUTPUT_TYPE', 'mailto') + output_type = os.getenv("OUTPUT_TYPE", "mailto") printfailedmessage = os.getenv("PRINT_FAILED_MSG", "False") == "True" pdfkit_options = os.environ.get("WKHTMLTOPDF_OPTIONS") mail_msg_flag = os.environ.get("MAIL_MESSAGE_FLAG") - output=None - if output_type == 'mailto': + output = None + if output_type == "mailto": server_smtp = os.environ.get("SMTP_SERVER") smtp_username = os.environ.get("SMTP_USERNAME", imap_username) smtp_password = os.environ.get("SMTP_PASSWORD", imap_password) sender = os.environ.get("MAIL_SENDER", smtp_username) destination = os.environ.get("MAIL_DESTINATION") smtp_port = os.environ.get("SMTP_PORT", 587) - smtp_encryption = os.environ.get("SMTP_ENCRYPTION", SendOutputByEmail.SMTP_ENCRYPTION_STARTTLS) - output=SendOutputByEmail(sender, destination, server_smtp, smtp_port, smtp_username, smtp_password, smtp_encryption) - elif output_type == 'folder': + smtp_encryption = os.environ.get( + "SMTP_ENCRYPTION", SendOutputByEmail.SMTP_ENCRYPTION_STARTTLS + ) + output = SendOutputByEmail( + sender, + destination, + server_smtp, + smtp_port, + smtp_username, + smtp_password, + smtp_encryption, + ) + elif output_type == "folder": output_folder = os.getenv("OUTPUT_FOLDER") - output=OutputToFolder(output_folder) + output = OutputToFolder(output_folder) if not output: raise ValueError(f"Unknown output type '{output_type}'") @@ -194,5 +213,5 @@ def process_mail( imap_folder=folder, printfailedmessage=printfailedmessage, pdfkit_options=pdfkit_options, - mail_msg_flag=mail_msg_flag + mail_msg_flag=mail_msg_flag, ) diff --git a/src/outputs.py b/src/outputs.py index 5ea84a7..74d4e1a 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -1,5 +1,3 @@ - - from abc import abstractmethod from email import encoders from email.mime.base import MIMEBase @@ -35,6 +33,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_traceback): pass + class SendOutputByEmail(OutputProcessor): """Sends output to an email address @@ -47,10 +46,13 @@ class SendOutputByEmail(OutputProcessor): password (str): server auth password encryption (str): Type of encryption to use (if any) """ - SMTP_ENCRYPTION_STARTTLS="STARTTLS" - SMTP_ENCRYPTION_SSL="SSL" - def __init__(self, mail_from, mail_to, server, port, username, password, encryption): + SMTP_ENCRYPTION_STARTTLS = "STARTTLS" + SMTP_ENCRYPTION_SSL = "SSL" + + def __init__( + self, mail_from, mail_to, server, port, username, password, encryption + ): self.__mail_from = mail_from self.__mail_to = mail_to self.__server = server @@ -60,7 +62,9 @@ def __init__(self, mail_from, mail_to, server, port, username, password, encrypt self.__encryption = encryption def __enter__(self): - self.__logger.info(f"Connecting to SMTP server {self.__server}:{self.__port}...") + self.__logger.info( + f"Connecting to SMTP server {self.__server}:{self.__port}..." + ) if self.__encryption == self.SMTP_ENCRYPTION_SSL: self.__logger.debug(f"Using SSL encryption for SMTP") self.__smtp = smtplib.SMTP_SSL(self.__server, self.__port) @@ -105,18 +109,26 @@ def process(self, originalMessage, generatedPdfs): ) msg.attach(part) - logging.info(f"Sending PDF output for '{originalMessage.subject}' to '{self.__mail_to}...") + logging.info( + f"Sending PDF output for '{originalMessage.subject}' to '{self.__mail_to}..." + ) self.__smtp.sendmail(self.__mail_from, self.__mail_to, msg.as_string()) - logging.info(f"Sent PDF output for '{originalMessage.subject}' to '{self.__mail_to}...") + logging.info( + f"Sent PDF output for '{originalMessage.subject}' to '{self.__mail_to}..." + ) -class OutputToFolder(OutputProcessor): +class OutputToFolder(OutputProcessor): def __init__(self, output_folder): self.__output_folder = output_folder def process(self, originalMessage, generatedPdfs): - logging.debug(f"Copying output for '{originalMessage.subject}' to output folder...") - output_base_name = f"{originalMessage.date.strftime('%Y%m%d%H%M%S')}_{originalMessage.subject}" + logging.debug( + f"Copying output for '{originalMessage.subject}' to output folder..." + ) + output_base_name = ( + f"{originalMessage.date.strftime('%Y%m%d%H%M%S')}_{originalMessage.subject}" + ) output_base_name = replace_bad_chars(replace_unpleasant_chars(output_base_name)) output_base_name = f"{output_base_name[:50]}" @@ -125,10 +137,12 @@ def process(self, originalMessage, generatedPdfs): else: for i, file in enumerate(generatedPdfs): self._output_file(file, f"{output_base_name}_{i}.pdf") - logging.info(f"Finished copying output for '{originalMessage.subject}' to output folder") - + logging.info( + f"Finished copying output for '{originalMessage.subject}' to output folder" + ) + def _output_file(self, source, destination): full_destination = os.path.join(self.__output_folder, destination) logging.debug(f"Copying file '{source}' to '{full_destination}'...") shutil.copyfile(source, full_destination) - logging.debug(f"Copied file '{source}' to '{full_destination}'") \ No newline at end of file + logging.debug(f"Copied file '{source}' to '{full_destination}'") From 0e0276eeb2369c8da28a59b1b5326500a88ea08b Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sun, 26 Dec 2021 21:47:06 +0000 Subject: [PATCH 11/12] Fix logger reference --- src/outputs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/outputs.py b/src/outputs.py index 74d4e1a..e9c0464 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -62,29 +62,29 @@ def __init__( self.__encryption = encryption def __enter__(self): - self.__logger.info( + self._logger.info( f"Connecting to SMTP server {self.__server}:{self.__port}..." ) if self.__encryption == self.SMTP_ENCRYPTION_SSL: - self.__logger.debug(f"Using SSL encryption for SMTP") + self._logger.debug(f"Using SSL encryption for SMTP") self.__smtp = smtplib.SMTP_SSL(self.__server, self.__port) else: self.__smtp = smtplib.SMTP(self.__server, self.__port) if self.__encryption == self.SMTP_ENCRYPTION_STARTTLS: - self.__logger.debug(f"Using STARTTLS encryption for SMTP") + self._logger.debug(f"Using STARTTLS encryption for SMTP") self.__smtp.starttls() - self.__logger.debug(f"Logging in to SMTP server as {self.__username}...") + self._logger.debug(f"Logging in to SMTP server as {self.__username}...") self.__smtp.login(self.__username, self.__password) - self.__logger.info("SMTP setup successful") + self._logger.info("SMTP setup successful") return self def __exit__(self, exc_type, exc_value, exc_traceback): - self.__logger.debug("Closing SMTP server connection...") + self._logger.debug("Closing SMTP server connection...") self.__smtp.quit() - self.__logger.info("SMTP server closed gracefully") + self._logger.info("SMTP server closed gracefully") def process(self, originalMessage, generatedPdfs): logging.debug(f"Building output email for '{originalMessage.subject}'...") From 879578b49ef3a534f6926c31ca3a042264bfd737 Mon Sep 17 00:00:00 2001 From: Deosrc Date: Sat, 22 Jan 2022 17:13:44 +0000 Subject: [PATCH 12/12] Fix formatting --- src/outputs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/outputs.py b/src/outputs.py index e9c0464..3d13836 100644 --- a/src/outputs.py +++ b/src/outputs.py @@ -62,9 +62,7 @@ def __init__( self.__encryption = encryption def __enter__(self): - self._logger.info( - f"Connecting to SMTP server {self.__server}:{self.__port}..." - ) + self._logger.info(f"Connecting to SMTP server {self.__server}:{self.__port}...") if self.__encryption == self.SMTP_ENCRYPTION_SSL: self._logger.debug(f"Using SSL encryption for SMTP") self.__smtp = smtplib.SMTP_SSL(self.__server, self.__port)