From b62b7823bb20f1503c81174f559cd97615824751 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 8 Mar 2023 02:44:20 +0100 Subject: [PATCH 01/15] Create a new smtp provider --- airflow/providers/smtp/CHANGELOG.rst | 30 ++ airflow/providers/smtp/__init__.py | 17 ++ airflow/providers/smtp/hooks/__init__.py | 17 ++ airflow/providers/smtp/hooks/smtp.py | 262 ++++++++++++++++++ airflow/providers/smtp/operators/__init__.py | 17 ++ airflow/providers/smtp/operators/smtp.py | 93 +++++++ airflow/providers/smtp/provider.yaml | 48 ++++ .../apache-airflow-providers-smtp/commits.rst | 33 +++ .../connections/smtp.rst | 79 ++++++ docs/apache-airflow-providers-smtp/index.rst | 70 +++++ .../installing-providers-from-sources.rst | 18 ++ docs/integration-logos/smtp/SMTP.png | Bin 0 -> 12065 bytes generated/provider_dependencies.json | 4 + 13 files changed, 688 insertions(+) create mode 100644 airflow/providers/smtp/CHANGELOG.rst create mode 100644 airflow/providers/smtp/__init__.py create mode 100644 airflow/providers/smtp/hooks/__init__.py create mode 100644 airflow/providers/smtp/hooks/smtp.py create mode 100644 airflow/providers/smtp/operators/__init__.py create mode 100644 airflow/providers/smtp/operators/smtp.py create mode 100644 airflow/providers/smtp/provider.yaml create mode 100644 docs/apache-airflow-providers-smtp/commits.rst create mode 100644 docs/apache-airflow-providers-smtp/connections/smtp.rst create mode 100644 docs/apache-airflow-providers-smtp/index.rst create mode 100644 docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst create mode 100644 docs/integration-logos/smtp/SMTP.png diff --git a/airflow/providers/smtp/CHANGELOG.rst b/airflow/providers/smtp/CHANGELOG.rst new file mode 100644 index 0000000000000..f7968bbf49c60 --- /dev/null +++ b/airflow/providers/smtp/CHANGELOG.rst @@ -0,0 +1,30 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. + +Changelog +--------- + +1.0.0 +..... + +Initial version of the provider. diff --git a/airflow/providers/smtp/__init__.py b/airflow/providers/smtp/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/airflow/providers/smtp/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/smtp/hooks/__init__.py b/airflow/providers/smtp/hooks/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/airflow/providers/smtp/hooks/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py new file mode 100644 index 0000000000000..c8c6d831bd661 --- /dev/null +++ b/airflow/providers/smtp/hooks/smtp.py @@ -0,0 +1,262 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This module provides everything to be able to search in mails for a specific attachment +and also to download it. +It uses the imaplib library that is already integrated in python 3. +""" +from __future__ import annotations + +import collections.abc +import os +import re +import smtplib +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formatdate +from typing import Any, Iterable + +from airflow.exceptions import AirflowException, AirflowNotFoundException +from airflow.hooks.base import BaseHook +from airflow.models.connection import Connection + + +class SmtpHook(BaseHook): + """ + TODO: add docstring + + :param smtp_conn_id: The :ref:`imap connection id ` + that contains the information used to authenticate the client. + """ + + conn_name_attr = "smtp_conn_id" + default_conn_name = "smtp_default" + conn_type = "smtp" + hook_name = "SMTP" + + def __init__(self, smtp_conn_id: str = default_conn_name) -> None: + super().__init__() + self.smtp_conn_id = smtp_conn_id + self.smtp_client: smtplib.SMTP_SSL | smtplib.SMTP | None = None + + def __enter__(self) -> SmtpHook: + return self.get_conn() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.smtp_client.close() + + def get_conn(self) -> SmtpHook: + """ + Login to the smtp server. + + .. note:: Please call this Hook as context manager via `with` + to automatically open and close the connection to the smtp server. + + :return: an authorized SmtpHook object. + """ + if not self.smtp_client: + try: + conn = self.get_connection(self.smtp_conn_id) + except AirflowNotFoundException: + raise AirflowException("SMTP connection is not found.") + smtp_user = conn.login + smtp_password = conn.password + smtp_starttls = conn.extra_dejson.get("starttls", True) + smtp_retry_limit = conn.extra_dejson.get("retry_limit", 5) + + for attempt in range(1, smtp_retry_limit + 1): + try: + self.smtp_client = self._build_client(conn=conn) + except smtplib.SMTPServerDisconnected: + if attempt < smtp_retry_limit: + continue + raise AirflowException("Unable to connect to smtp server") + + self.smtp_client.login(conn.login, conn.password) + if smtp_starttls: + self.smtp_client.starttls() + if smtp_user and smtp_password: + self.smtp_client.login(smtp_user, smtp_password) + break + + return self + + def _build_client(self, conn: Connection) -> smtplib.SMTP_SSL | smtplib.SMTP: + + SMTP: type[smtplib.SMTP_SSL] | type[smtplib.SMTP] + if conn.extra_dejson.get("use_ssl", True): + SMTP = smtplib.SMTP_SSL + else: + SMTP = smtplib.SMTP + + smtp_kwargs = {"host": conn.host} + if conn.port: + smtp_kwargs["port"] = conn.port + smtp_kwargs["timeout"] = conn.extra_dejson.get("timeout", 30) + + return SMTP(**smtp_kwargs) + + def send_email_smtp( + self, + to: str | Iterable[str], + from_email: str, + subject: str, + html_content: str, + files: list[str] | None = None, + dryrun: bool = False, + cc: str | Iterable[str] | None = None, + bcc: str | Iterable[str] | None = None, + mime_subtype: str = "mixed", + mime_charset: str = "utf-8", + custom_headers: dict[str, Any] | None = None, + **kwargs, + ) -> None: + """Send an email with html content. + + :param to: Recipient email address or list of addresses. + :param from_email: Sender email address. + :param subject: Email subject. + :param html_content: Email body in HTML format. + :param files: List of file paths to attach to the email. + :param dryrun: If True, the email will not be sent, but all other actions will be performed. + :param cc: Carbon copy recipient email address or list of addresses. + :param bcc: Blind carbon copy recipient email address or list of addresses. + :param mime_subtype: MIME subtype of the email. + :param mime_charset: MIME charset of the email. + :param custom_headers: Dictionary of custom headers to include in the email. + :param kwargs: Additional keyword arguments. + + >>> send_email( + 'test@example.com', 'source@example.com', 'foo', 'Foo bar', ['/dev/null'], dryrun=True + ) + """ + if not self.smtp_client: + raise Exception("The 'smtp_client' should be initialized before!") + + mime_msg, recipients = self._build_mime_message( + mail_from=from_email, + to=to, + subject=subject, + html_content=html_content, + files=files, + cc=cc, + bcc=bcc, + mime_subtype=mime_subtype, + mime_charset=mime_charset, + custom_headers=custom_headers, + ) + if not dryrun: + self.smtp_client.sendmail(from_addr=from_email, to_addrs=recipients, msg=mime_msg.as_string()) + + def _build_mime_message( + self, + mail_from: str | None, + to: str | Iterable[str], + subject: str, + html_content: str, + files: list[str] | None = None, + cc: str | Iterable[str] | None = None, + bcc: str | Iterable[str] | None = None, + mime_subtype: str = "mixed", + mime_charset: str = "utf-8", + custom_headers: dict[str, Any] | None = None, + ) -> tuple[MIMEMultipart, list[str]]: + """ + Build a MIME message that can be used to send an email and returns a full list of recipients. + + :param mail_from: Email address to set as the email's "From" field. + :param to: A string or iterable of strings containing email addresses + to set as the email's "To" field. + :param subject: The subject of the email. + :param html_content: The content of the email in HTML format. + :param files: A list of paths to files to be attached to the email. + :param cc: A string or iterable of strings containing email addresses + to set as the email's "CC" field. + :param bcc: A string or iterable of strings containing email addresses + to set as the email's "BCC" field. + :param mime_subtype: The subtype of the MIME message. Default: "mixed". + :param mime_charset: The charset of the email. Default: "utf-8". + :param custom_headers: Additional headers to add to the MIME message. + No validations are run on these values, and they should be able to be encoded. + :return: A tuple containing the email as a MIMEMultipart object and + a list of recipient email addresses. + """ + to = self._get_email_address_list(to) + + msg = MIMEMultipart(mime_subtype) + msg["Subject"] = subject + msg["From"] = mail_from + msg["To"] = ", ".join(to) + recipients = to + if cc: + cc = self._get_email_address_list(cc) + msg["CC"] = ", ".join(cc) + recipients += cc + + if bcc: + # don't add bcc in header + bcc = self._get_email_address_list(bcc) + recipients += bcc + + msg["Date"] = formatdate(localtime=True) + mime_text = MIMEText(html_content, "html", mime_charset) + msg.attach(mime_text) + + for fname in files or []: + basename = os.path.basename(fname) + with open(fname, "rb") as file: + part = MIMEApplication(file.read(), Name=basename) + part["Content-Disposition"] = f'attachment; filename="{basename}"' + part["Content-ID"] = f"<{basename}>" + msg.attach(part) + + if custom_headers: + for header_key, header_value in custom_headers.items(): + msg[header_key] = header_value + + return msg, recipients + + def _get_email_address_list(self, addresses: str | Iterable[str]) -> list[str]: + """ + Returns a list of email addresses from the provided input. + + :param addresses: A string or iterable of strings containing email addresses. + :return: A list of email addresses. + :raises TypeError: If the input is not a string or iterable of strings. + """ + if isinstance(addresses, str): + return self._get_email_list_from_str(addresses) + elif isinstance(addresses, collections.abc.Iterable): + if not all(isinstance(item, str) for item in addresses): + raise TypeError("The items in your iterable must be strings.") + return list(addresses) + else: + raise TypeError(f"Unexpected argument type: Received '{type(addresses).__name__}'.") + + def _get_email_list_from_str(self, addresses: str) -> list[str]: + """ + Extract a list of email addresses from a string. The string + can contain multiple email addresses separated by + any of the following delimiters: ',' or ';'. + + :param addresses: A string containing one or more email addresses. + :return: A list of email addresses. + """ + pattern = r"\s*[,;]\s*" + return [address for address in re.split(pattern, addresses)] diff --git a/airflow/providers/smtp/operators/__init__.py b/airflow/providers/smtp/operators/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/airflow/providers/smtp/operators/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/smtp/operators/smtp.py b/airflow/providers/smtp/operators/smtp.py new file mode 100644 index 0000000000000..7c18f1c39094f --- /dev/null +++ b/airflow/providers/smtp/operators/smtp.py @@ -0,0 +1,93 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Any, Sequence + +from airflow.models import BaseOperator +from airflow.providers.smtp.hooks.smtp import SmtpHook +from airflow.utils.context import Context + + +class EmailOperator(BaseOperator): + """ + Sends an email. + + :param to: list of emails to send the email to. (templated) + :param from_email: email to send from. (templated) + :param subject: subject line for the email. (templated) + :param html_content: content of the email, html markup + is allowed. (templated) + :param files: file names to attach in email (templated) + :param cc: list of recipients to be added in CC field + :param bcc: list of recipients to be added in BCC field + :param mime_subtype: MIME sub content type + :param mime_charset: character set parameter added to the Content-Type + header. + :param custom_headers: additional headers to add to the MIME message. + """ + + template_fields: Sequence[str] = ("to", "from_email", "subject", "html_content", "files") + template_fields_renderers = {"html_content": "html"} + template_ext: Sequence[str] = (".html",) + ui_color = "#e6faf9" + + def __init__( + self, + *, + to: list[str] | str, + from_email: str, + subject: str, + html_content: str, + files: list | None = None, + cc: list[str] | str | None = None, + bcc: list[str] | str | None = None, + mime_subtype: str = "mixed", + mime_charset: str = "utf-8", + conn_id: str = "smtp_default", + custom_headers: dict[str, Any] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.to = to + self.from_email = from_email + self.subject = subject + self.html_content = html_content + self.files = files or [] + self.cc = cc + self.bcc = bcc + self.mime_subtype = mime_subtype + self.mime_charset = mime_charset + self.conn_id = conn_id + self.custom_headers = custom_headers + + def execute(self, context: Context): + with SmtpHook(smtp_conn_id=self.conn_id) as smtp_hook: + return smtp_hook.send_email_smtp( + self.to, + self.from_email, + self.subject, + self.html_content, + files=self.files, + cc=self.cc, + bcc=self.bcc, + mime_subtype=self.mime_subtype, + mime_charset=self.mime_charset, + conn_id=self.conn_id, + custom_headers=self.custom_headers, + ) diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml new file mode 100644 index 0000000000000..0ad50985f6129 --- /dev/null +++ b/airflow/providers/smtp/provider.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-smtp +name: Simple Mail Transfer Protocol (SMTP) + +description: | + `Simple Mail Transfer Protocol (SMTP) `__ + +versions: + - 1.0.0 + +dependencies: [] + +integrations: + - integration-name: Simple Mail Transfer Protocol (SMTP) + external-doc-url: https://tools.ietf.org/html/rfc5321 + logo: /integration-logos/smtp/SMTP.png + tags: [protocol] + +operators: + - integration-name: Simple Mail Transfer Protocol (SMTP) + python-modules: + - airflow.providers.smtp.operators.smtp + +hooks: + - integration-name: Simple Mail Transfer Protocol (SMTP) + python-modules: + - airflow.providers.smtp.hooks.smtp + +connection-types: + - hook-class-name: airflow.providers.smtp.hooks.smtp.SmtpHook + connection-type: smtp diff --git a/docs/apache-airflow-providers-smtp/commits.rst b/docs/apache-airflow-providers-smtp/commits.rst new file mode 100644 index 0000000000000..64dc7ca944b55 --- /dev/null +++ b/docs/apache-airflow-providers-smtp/commits.rst @@ -0,0 +1,33 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Package apache-airflow-providers-smtp +------------------------------------------------------ + +`Simple Mail Transfer Protocol (SMTP) `__ + + +This is detailed commit list of changes for versions provider package: ``smtp``. +For high-level changelog, see :doc:`package information including changelog `. + + + + +1.0.0 +..... diff --git a/docs/apache-airflow-providers-smtp/connections/smtp.rst b/docs/apache-airflow-providers-smtp/connections/smtp.rst new file mode 100644 index 0000000000000..4c110515d297d --- /dev/null +++ b/docs/apache-airflow-providers-smtp/connections/smtp.rst @@ -0,0 +1,79 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:smtp: + +SMTP Connection +=============== + +The SMTP connection type enables integrations with the SMTP client. + +Authenticating to SMTP +---------------------- + +Authenticate to the SMTP client with the login and password field. +Use standard `SMTP authentication +`_ + +Default Connection IDs +---------------------- + +Hooks, operators, and sensors related to SMTP use ``smtp_default`` by default. + +Configuring the Connection +-------------------------- + +Login + Specify the username used for the SMTP client. + +Password + Specify the password used for the SMTP client. + +Host + Specify the SMTP host url. + +Port + Specify the SMTP port to connect to. The default depends on the whether you use ssl or not. + +Extra (optional) + Specify the extra parameters (as json dictionary) + + * ``use_ssl``: If set to false, then a non-ssl connection is being used. Default is true. Also note that changing the ssl option also influences the default port being used. + * ``timeout``: The SMTP connection creation timeout in seconds. Default is 30. + * ``starttls``: Put the SMTP connection in TLS mode. All SMTP commands that follow will be encrypted. Default is true. + * ``retry_limit``: How many attempts to connect to the server before raising an exception. Default is 5. + +When specifying the connection in environment variable you should specify +it using URI syntax. + +Note that all components of the URI should be URL-encoded. + +For example: + +.. code-block:: bash + + export AIRFLOW_CONN_SMTP_DEFAULT='smtp://username:password@smtp.sendgrid.net:587?use_ssl=true' + +Another example for connecting via a non-SSL connection. + +.. code-block:: bash + + export AIRFLOW_CONN_SMTP_NOSSL='smtp://username:password@smtp.sendgrid.net:587?use_ssl=false' + +Note that you can set the port regardless of whether you choose to use ssl or not. The above examples show default ports for SSL and Non-SSL connections. diff --git a/docs/apache-airflow-providers-smtp/index.rst b/docs/apache-airflow-providers-smtp/index.rst new file mode 100644 index 0000000000000..3f8ca0061d782 --- /dev/null +++ b/docs/apache-airflow-providers-smtp/index.rst @@ -0,0 +1,70 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-smtp`` +================================= + +Content +------- + +.. toctree:: + :maxdepth: 1 + :caption: References + + Connection types + Python API <_api/airflow/providers/smtp/index> + +.. toctree:: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +Package apache-airflow-providers-smtp +------------------------------------------------------ + +`Simple Mail Transfer Protocol (SMTP) `__ + + +Release: 1.0.0 + +Provider package +---------------- + +This is a provider package for ``smtp`` provider. All classes for this provider package +are in ``airflow.providers.smtp`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below) +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-smtp`` + +.. include:: ../../airflow/providers/smtp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst b/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..1c90205d15b3a --- /dev/null +++ b/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../installing-providers-from-sources.rst diff --git a/docs/integration-logos/smtp/SMTP.png b/docs/integration-logos/smtp/SMTP.png new file mode 100644 index 0000000000000000000000000000000000000000..dbdde70a8b704ae0d2bfe2dad8b0cbe5ecf12989 GIT binary patch literal 12065 zcmeHtXIN9)({DhrP~-?w1Obsmnl$NM={pgSjz9nlgpNo;Z;HSXDThEn=_T~0gd#z} zatI0%AP_KA1rb7(7No!1|9$R<`{90hKi>PoL-xbYUVF`&nKkp9-o50IS)J@Zy5XkFPwu8GY;QyEJn^>AcAYoS_keH_s z$R2nrW&r}h$U`8Dt`LaE3kZboVJ^`^3;g2LJ%q6Vg&i-A@8cELC)cE|G^`c+G(0=6(BN^o zs4%hbQ|2o-^SzyJb;Xw#!>~Y^nVsC)=;jX~Kli1^^x}guT`QaNs@v7nlb|%wiA8MS z#Y6SvHSp4Z2Gj^7|NJZPQ%lU12?7Z*ddCcb{7Q-ewejsD6Zq^qMP^=53#k7Y@c)XI zKx6-Re>_gb8BIW+&A7m`FA)hIp7qLPGAj~vtowtLh5K*cd`PXJe(2yjQ~v5huC5)W zx2X3Ef~#v1J9U+Aid9K#yg^CqxJ-ohr74+GhOq7zQ3V{6e2^Pe7EIMji?~u{Pr9Y& zXD*&ht_BVG%_^SfmUb3WyjgsWKLLjeL?;ofgtH^192R+#(hW1+_{w-E;_?r=1z(9h zlBHjrU1lrVKL^RfTQGU26Qdogf#-bHuO3u3dc1CQy}LK>;M=9!S%Tq#@~~E6f~yBc z#W2B@BUBhBIGMM>VJ}*rtIH8FW=AE;nEZX2?ODQz6lMZHqfrpog;mdJh>fiNNLYlyF=gFtGFdQQAmX*d$XL)$?U)FAzlkNI>S}I2-d5CKMSc#}qd^#*r2`!^vBj$+$G*OSe z^YH{|o@Z{m+zo9xX(3;4>vwxzTwN%BRQag|?|g>s><64uCQRgG)5=!V2@J)%!JW^e3-3#i|b~mSvIGDzTvf<;N z<6*S&*E-eP97(tSsL>HiOkC`kb;faOTXl82SK6)$?Oxo3ZO6%myTL3Dv;kw2#k0s> z{tVa(%(H%3@as1AoFe(C^yZFB&DL~l+To-neb`fT*DH3dnJ41JzNThilG$u?rCN!!%P0>&KTo(vDkPtx}} zo=vY@1n#FBtlNQGpkjp6PHaVjJ`G&ZeQB`XT=aZ>VFlaFhJ0%5!j6Otob1Rk2#|R; zQ?{pbNy>Fw09BtcEJs)88A_`GrIFgMm|xVip4t(Y(m4{T?8KUhU@*yZ0Z~fndZKw; zHCymG9n3VQvr1FAYE`hTs(J4uRcfwC%x;t)HL60MDu`q}9hj3d=jy^i6a8`M$7A7( zvly)Gt>WL6(8Os0BENYMQ2Zv%JM&Q*%d{=>6Z?<5mpzobJZvjslr6AHgnIBjSab?2 zOJ%h*hZW8lW7<-+8d4D^+bJXTS%9A|!PVfP_gVr+w8^=hmN_yUc!a6C^bpG`*IqpK zw8|^8oz7tQe53J6ZJ%~tMrkE+2b$P|-AXhH*J#$(_Q=hY_MBwVkFbL??Hc+@M*gmF&7P`Uhh@>Pz6N6c8ZdAlwJPAK^qxMAJx~Nj}Sn|DB!SGORzTk1$qHIF-J~`YbXShJt-G#9h z6c@l~BGJP2v$So23&(IuTG7%O*CKq59^ZFb8MfmR@$UfFEmr=VK&(W^^?#G~S!ygU zqDFD(fp0k&b>taj$cCp4L(R5*9$Ap`U^|PD_D|W8-n^s)^3zn#s&A>`a~q9$IF3^X z8OP^r;X5-Rc@v<>l4x>I!Tk)ih)AKtP3L)JkIA_L6f@e@JejybY1h(7*=IAS1tW*b zZi&+}2E#n#y3kfVM~TBW*7Ple4J|lPY2J;xCtbCTi?5rzS;A45Ck;>=5*gD~mqM!c z$V90eu1n29s{oJ|z{nkNt55Rz;Z;6>IjQtft*s$kr0Gejw03g^b@@DKR+ic zVy&F7>Y!xECgi&t^{TCrPSUI70LD^Luk#(?g1;1f9<mj3;pR+Mt_fOMd|4*^_BF`ip*bnv9R07`@dX+RD-gM4QX zwK!mE`k8Tcz^CJ%ypFW$%ro}b2f2}-)Y{_lF6I6kU1+aY6V-L})T)iz`_#ZWA-ahz zPV%D@yU!U%QxW-vj$L7!AYG%bi>Z|=l^WM4@_loYlL^w{CZ2?^=9F+fzQL0M7}>if zEh7(e!z$vyC3#z^OfbzJanbxPx>($x+r7zy>fZfjDTQi0$itD#{(3 z?pegKuu%`YJFB8evte@!?bhM5V^w`Fg&z*$DA>|V0_{?#jM(KHlLM7Ok?S^9@I} zOgj!~;xON`t?5_SbgT2Sq$U-<3Bk=tzeRv{j^&|j;DU&EuC{zC$?>`?hdj9h_H2gR zMj|5I6?wa;8o{E9IMlpU6F2?UVKaEba&gu2?kcnA`aMrcQSUE_ER}7Vd5N{u;lQri zMa`dfbnZOvYWmHIUWRB6R){h|X&9_NeAHoTN?9e*6eE<}b897$e@CU`Pc%fmMN<6W zEm`(d)t{au8jE@-Z{Ix(;9^6Zx3dC0qzUW5IL$Y^9KjNTOqa+n&?gKf6F60ga;Epv z0#s7l@ns026YVOQwl^vQ_Zkj;GM%1*c<1#}*}g=9yedv#sUKTzxP6=6h_I#|Lo00& zJ6{&v`fC&}@;5EYRI0qL`XIe&t@CqKN+fFJTYze;xlfsjcfv#5t}A(E`fjew9;+Sr zA~RL;YC)}|6M~Y8Guy3AGJ7F>$*TpLsLGBs5IX(o*yNV@r~6R6TdLhgp{N*wMB_^) zoOT~qQvMMM>ak?gqh;I?wm{_rH+S(yNVNBu{(@lP_t5uhny_&HV>@z- zhD55-_!U=)F!O@>@#qoQxtt(Up_1@1(h3_K&kw90&1}w*_PRi;3v_3Uj8?o1+I<{N|~0_S1|nVc*W%Ff`02Tc*-e+%bFkcVg?!?^JD7F6Gm!R=4hB zrd4FLSKWUr7^ot`z^I)+;5PqbR=p`aY9U1?HGm;blRp?REaiM!CtdhwxeDm>_rQF6 zRq%7q63_&6a7O$0#wfp2Ym3GECL|ARWeuIr7Rf8>DWYr4jy;8=gb7yHs%(3NS?wDP zN(uGWZqo%7^7F7iIB;o6NCWl;)GRlNR$A0c2J4bZxCm^=w+wHkzvF)`iO`L3FsA&( z*0CYYZ!9799)4oRO4ONnm5N2g7}3W%wP z0~m7yd4kfj%R?lZU^1Zyan?5rnkd*R(r?lxHF=x&-CrbHC!DBQ{GKq#sBBVZ*ZDR_{&@ue!Sne}S^Ha&1FL1Y`jOuklZgBT59Sqb)sEnLv@{S1z z5&?3~jVpTJvvD9=M|S5|_XL}zL8*Jy`iw8E$R=^IowJUhmZzPnH+G`SJ}hTZ%1ov-h=jK5qURsG{D2o$hCbO9ZeNApUmQ94-9)i-PW$v2^2s7Zvi zbo5u(Pf}TUu=p#8yovL_R7;Vo|HSwrV4pf%EJqzEmq5TU(?^^(!J6vi7{61B3t37-xm8qbVi%Bjf3!^q&gN z`f%^=Vn?JyGo-^1TuCn1GExQh=J=Tu@>hae_vAIlPr69@lx-&)GRXCLMt@jkyh!qi zHG%7HSqAHT{~q(Xy4~t4GWGZWB_#3+|6cNAAIw+v4a(M~-s_NuXZ|gI<2|avA5OG; zkQbYh(>qWrL2MfiT*<47+BTLt>MgT+f}1K5T}P2<+nQS@8EL$TFwuk{Rg+@1g?u=F zOaE1igSt}_^Eor&*Z<0HX&2tTCw#^6>)|ON@O1y&QoRY%5GP9nsFNv{cSi#IT2Ui6 zYD7D8%a@f#BCAWtb8Lao6HS24Tz`_OfW~mb@p~}mN!y6 zaPA(dv2ttLA^)}$#*x@U{Z-i)4PU%Yb{5%d^3!YiF}$h*T!>ag&ue@##jgIE__+#J z2A!9@kKzxabvrC99rH-%k{)_5FlQ#jC@1=$=dGkXXrBh2Ry96&wwJPtlS(Ln@vwRh zNZ~#R5ftTNUtQ8zS+_kw+knQh`ob^Yr|OF5q-| zDso$>Dr)ZZ;&pSpXf@@V?NE9wi6#;kWtua= z4rKipW*7i1P>Z*23uZwvyVxihx?%r{UxC58&W? zSV#h1IhhcB_RvA4&F8^d!zF(UEZUT-Yd?O~JI7H|^LU-Pf5ix^vqE?e*1gj7cJ;TX zI{Tv5`ip!E@BCOlagwQeuwk%(BSAzxU6?QwzQIzp8kM9JJk`+k1KAZ}L`dv%YpdsD zTWsCpwb{`!i<@`8wKl1`Jaw8MrR$Z8ys)?JEgG!+O;{Xe5dkKF>bO;(oT>+%A~JM~ ze&4lBgY_k?=9Y5G{p<*h5w(45#=Kv@?}XK;V8&7ZZG22p@1U9~1BUMPFQ&T_Od z!Fm#m|7&?KeG|Tg_SZbNsP4def%RAa7u)IcsOpI2cNwmCq&UpZlG7UV1A^cSqczkA zryUEv#yI+3^$gySUWP+kzr>)Ggb?5|}8w&$b{03b2AaG&l>1s$0zkn1O zcPvhC-c;IqQ;u)a8V!09z<6xo%vPB9qv_pYL5IK<*V-$y%b&4RP*3NuFJ>x<))E!y zbyJW~NMt>JDPvlJpTAVM$o%`USj3*RWo;S7zus={ot1USY8-lU6oes%Nrb89-8E~P z#7L@O#CR-J1|&MNp^2j$hXRdn-peV?F6(b}@-FGajW65XVGIQem)La*MRVUk{B{;j z7C`asc|DI&K1ZT;Clh#pEOtS+#U-23X^~qyU-mw_G_d@VR(DhLLD$sG)Szs3zk4q3 z?{--R?|jm)aLL#Tnq3BpT+&q@72`dbR@+MP30R4IkOjI6T-dzu^>?21afy9;x`b@< z>}D|bc(6J)nQ;7T;_3hzE5X=%_4zLn>i)@VoJj;1Y~$RkTJEdXloOe$I(9g*elz3Z zc)z??Z*!IJMcVDS2;atJeWWUHGf}TH%0>NXe}C;F(ZI6$w0`Hgm4DUti}7nGuieGs zV`17kakV1c&9Iyc#}i) zEU=B~0nL@F4&H7gS`I8y8Z89>IFj0U%|{3eI=((1^&UBG>&3i^xVXhc8*4#JJ$_qj z^wHaZOyu|=11q6(rTeG(CXJt|S8R4!Yj$}FCZz4tUjr{lR1~qp?A#v?|J+Bnir3%y zB4zS%=c`op?=_UoWOQO!f13*OlTJixEg!V@e(s^of=&wv6g<7^(jEiYbGR*oX0@Xl zXyTpV7H0M2dy#MoNqX}vaG_qz++6hBdInxvyXCA>o=?e7k)PC#07gK;%K2B1TyqZ} zrR#|{t&Z%d;A?ezy0jcf#?uT zbocIAv({OdryQ{KGJkFG6>h#xjuJypYorAPp@2S}dv99tKaEZ-+Tf zAKtI_#&=*7&HHUJsrPM&2j<&lCsA;zi>P|m$#OII+)044UbMttq7Bmq9g7XPu*OAv zarIF%K3uD%mCCOxTi;|i5}0#CyXh}g)8gOPfzi2$oR=h8Ylk4rRXHXA%nyP$IM3{CW_W_xS&*tgX3hF##|ALMu|e#}ZukocdX0~D^?A)< zRpr^`OF-x5#PT^1*WYIsX?ZrkZvkU5-FVg$jm18fmQq* zG*Qm7=vV1`Z9?WXK$SyAB7E`T!=ZsttPV)Ui=kKe$!hyK(8M*{y^3Ga33Vt~gyypu ze`#&jd;6mER7^Gj8fkNM?v68pUPmceh2$BmuVQtUTSXECQK-2MR;GG}r89@Pg%&{3 zg!z_idDD|QmPfxVAJR0LZDTK@T0ku*N?oY4wG-90M17lS@D)6g^5mXfeloCK+_7BQ zs48v|@%F=NR+E&@Rr=@uP}!>j#1}@2WHMNKGw-t5V;k*2t|3*e%)O#Mbd_X1r|Zki z-V~)$JnR<{CBh#s`C))H&aKl@SAiTA^N=c#M3}}kc0dJ*nKtt3IT{n1Xi=tI7=2B6 z+|jBeH#u`?fhtKui^tdm=LV>>q^W~)f>|sLU%lj%Eln39(QE}d=piH;okTm%Y|O)V z6)oLiDF>OwXs#}Q4x~v5KMIA=7z!jfx|y@4{JX$25Frc_;VT?$9c}Qf{}`+v!AEVjsjU%(0YRlfu||Q4FLFx@=+Z2!u26;SjK({h5v;)rp7HV;I z+`IDRqbq`gdyNj>$s?H{BnvdS?Ex|e)9}66?yJWI_ha1)9ACK&6wfhA;CivS9x zG-GtKCKm)bMR*!^F^@YqFQEBtu6W9TQ4(Pk<=r@fMMo`!kvt*+RdB}afjFg@HH(^5 zx6Wuo%hc{5ITi9=9nS*PUzw~kQIP{iAaHSm@Zjfl!>w;~0)l!6l#AE@!z#3>l6j%N zl9i&mhRGi_sZ65MY%4f*R8uvyFqM%5Uj)-)br&ErUas5%7?o(Jlpg3emMc4H}YZoau7aq3F1*j^o&OydV zKC{aLqJCe_WQv2k%Z^5@(#Wds%jbeReYZ&>l!F{~zfp4sjQ)N=Ir-N)VE(D>6?0Hm zCPY!RQ13e1tdFMO#+e`V-?zWRuegS$8LZ1e4}CLwU6jas&I^euxVYLD)ZHZn){hfu z<=*SF*p_#wg=Q*wV@*jedJb{;G|hJBB$M|gti&p!`1e_0x+0JUNC~3kIFRBF z=>ZI`+2yNuzK%Ij@*X3`#GcJ?#zi4Fz&z&ExBJ%zdXh8K{pRNZLCHq-Ema9IO^H~E zDLwk*FZK0-yIQk8KDl`!M6<@zv&&!=oBH6ccQ}yBB5@y!7t`IAngH z{-$#hK~*Xb>&VTaQmo?s*qySw-^4`Q{HNGU* zWDBmYV`=(XDyLmJKS~wqBzkgo`93WG;o>tO0|woY@qt6~^8m&vK|wn3xnA(dX|RrX zVD&`NI@*{Kjx#$89(9EUuyMa`vWx>iXxx0+tV%_jPlAo^j;vJ5X%5bVy`pyVo(bwlj7f}fO2qe*;B%k2%#zL zN)7e<5<*1?C{l6$7p=L6a~G ziaSPj+B8~FFMltr9iN_XKpg`vO!E?DLxTUNY$|O2A`j$w$0)+IY47%h+;Z~- zKnhJrXT%9}f!X%zvl)Hpq&!%LSW%cJe()#nD^;sqfv@5m)}WE(|$XSU6wf1Aq8z zF`-(JAo)I%(+;>`2QvId+aIZcaLz8MQ;rpz#~k1Qk$(<^dt^e-vSd zn%xX@B{G?P5`Y|`o!-HV-@Gt*jYHe#^&V#q`{(v;ey%oRY#&1>tI%RCX$%$e&-RP3 z?C#~XlMOKw3Wu;nZmxt!!x#AUD&M({2TJzX=vnEwf%bw8rs|24}kro zN4wsnJ1X){DbPfha&&rkv;b;{6`6`c1bVgHXSV%`F72N8`BvFpfZ`N|cqsa6KLO=# zhL_9FXk?kIlKlqGeV2MR1Ar&R_`UCo>Oj4yQ!>zrPX)%nLoM-i2}XPEO%wl5m6KU| z!8K_O1U|W~;R)@TlPrb4O}?N8e`_TlB|l`FT^26#-Wu+&h4rO1`sMYOyP2Qe;NJMk zKB=^c!LEXx4p?X+qKIwN_2atgx|F#|m;}0;Se9H#lyD*-Hn;fU;8#VNPxDkM}VX019^~t z3Yzcy$TwKWV4L~pHjF^OKlj!bt=lrVpsQVQH}}weLBMf1+H;r5``rTotid{5D5}YX z;5gZcP*55;ML}IzR~vn-r$E*Lvy>5b>KffG*|#n6NyA#}$|Dc0n8BuqKl=5>u3{S< z@_NDWwvYH}&O9gG^ z`0{iK=M01Omwmss?gm}3&i}rsg9&-Z95d%hI}ci?96sjJ*h``r>y0i{7PNV7C7(%E zCJSF6l2jBj>F;^ynT^$WBq6O<9Cm0%;UUp91qPo%6L&Jv*p1`wo(|1l1tGVMd;eqT z^sR3aS^i*`hdH=;8QXB6pgPdA4sR}iT4P7v*961w6a3DPfvqK2q%kG9;T;TUL1Y)9 zPp^4mzl886id!7WmU?rx^j30iU&Eqr#;&qJWGYLU!Ob$Ag&vm3;*=+wokf!c@`DfHetdV2qqt<|+r2*xzMN|P^5K0xy3p1qbCliGB?IXxK!o(QfM<68td3DcJ z=ucR&`C6&HA5$Xy{kO*zDMY z=Il>v9Fi#5RG;;6yp)n{uznhP=$6sz%YDW1U-u$54<=nDvas@zJESmB$q<|>L>VyJ zPL&F^r;t_}DnUO)0b}6vF7o7`t@|qRpN!7y@)J|88oyzVvH45NRfhjJNM{w`&{PSL z;FN%%tk=r43Gf61o-Ijl!pQ1gp|6l=aYakZ@@!CT0pj0RgXF)nLTpVd2xsR1VC8b} zt%uu1Dy((|xUB{-zXT=|&VH`~@hl>V5H5*Kz#tdGvOhh?swj|!g^7PbJZy@XNB$HH zlJp!*U7r$}cWmz047keMhkv!2P{8J`JpIwHK1l@Sy_wNw4@IZ1?TbW$;v#QS|9T#%)@Bq})fZN%;-4($B&C)6r0>LP@v4F=VF z;L}QcNiCu-4dpt~d4~dOLl3=yf{eywJW%BQEoRCJXyZZ3sVNEPbI4B~Ndz`1@}EP231_gu zun*RbpzOi>wUQpm&YdbfoiObMR?=Hb4O(E=3)pT`K~b%36*x{WL1JyQwr(tRD4&>J zMu7acp6AzSLkcB8g$DDK0b1f#$N7DWl{|hg?t5X&;#PbN?v&9<=if<7O3d%*91I&lAgZ#!N-K7pu9`%)=x&=1.3.1", From 73b3dca07f6b4cf704c805ba293b8790c94ba30f Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 8 Mar 2023 23:06:11 +0100 Subject: [PATCH 02/15] typo --- airflow/providers/smtp/hooks/smtp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index c8c6d831bd661..b2a0306247a03 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -18,7 +18,7 @@ """ This module provides everything to be able to search in mails for a specific attachment and also to download it. -It uses the imaplib library that is already integrated in python 3. +It uses the smtplib library that is already integrated in python 3. """ from __future__ import annotations @@ -41,7 +41,7 @@ class SmtpHook(BaseHook): """ TODO: add docstring - :param smtp_conn_id: The :ref:`imap connection id ` + :param smtp_conn_id: The :ref:`smtp connection id ` that contains the information used to authenticate the client. """ From e1a953f8812f56964729f2526ad858175dc2bbb8 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 9 Mar 2023 00:17:56 +0100 Subject: [PATCH 03/15] fix static checks and add smtp to preinstalled providers --- .github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml | 1 + CONTRIBUTING.rst | 2 +- INSTALL | 2 +- docs/apache-airflow/extra-packages-ref.rst | 2 ++ setup.py | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 756fd1cffee8d..3a7efad358925 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -91,6 +91,7 @@ body: - sftp - singularity - slack + - smtp - snowflake - sqlite - ssh diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e5d989f61f19d..8b539430cb4bd 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -621,7 +621,7 @@ imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.az microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, -snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, +smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE diff --git a/INSTALL b/INSTALL index e3848a3716ba7..87a759bc02489 100644 --- a/INSTALL +++ b/INSTALL @@ -105,7 +105,7 @@ imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.az microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, -snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, +smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk # END EXTRAS HERE diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 457a34aafa6c6..1339b2936f89b 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -294,6 +294,8 @@ These are extras that provide support for integration with external systems via +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | sftp | ``pip install 'apache-airflow[sftp]'`` | SFTP hooks, operators and sensors | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ +| smtp | ``pip install 'apache-airflow[smtp]'`` | SMTP hooks and operators | * | ++---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | sqlite | ``pip install 'apache-airflow[sqlite]'`` | SQLite hooks and operators | * | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | ssh | ``pip install 'apache-airflow[ssh]'`` | SSH hooks and operators | | diff --git a/setup.py b/setup.py index f29a923c7695e..9fe57e133797b 100644 --- a/setup.py +++ b/setup.py @@ -695,6 +695,7 @@ def sort_extras_dependencies() -> dict[str, list[str]]: "ftp", "http", "imap", + "smtp", "sqlite", ] From 0fe271bfd17ee711c66b35bac02fa9f88c9a2ef6 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 9 Mar 2023 00:25:30 +0100 Subject: [PATCH 04/15] fix breeze images --- images/breeze/output-commands-hash.txt | 10 +-- images/breeze/output-commands.svg | 90 +++++++++---------- images/breeze/output_build-docs.svg | 12 +-- ...ement_generate-issue-content-providers.svg | 4 +- ...agement_prepare-provider-documentation.svg | 6 +- ...e-management_prepare-provider-packages.svg | 44 ++++----- 6 files changed, 85 insertions(+), 81 deletions(-) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index c9158d14134b8..531bd3b6f6335 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,7 +2,7 @@ # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:83de6a9bf2b1afecd1f9ce4cd0493733 -build-docs:18235f12f85f8df82f3eb245e429f62d +build-docs:61ddf42565c6e39b2a1b228b2a0e89de ci:find-newer-dependencies:8fa2b57f5f0523c928743b235ee3ab5a ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd ci:free-space:47234aa0a60b0efd84972e6e797379f8 @@ -37,15 +37,15 @@ prod-image:verify:31bc5efada1d70a0a31990025db1a093 prod-image:4f98deab35e53ebddbdc9950a50555a4 release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957 -release-management:generate-issue-content-providers:9b8b5b39e9310b01e6e0fb6b41483063 +release-management:generate-issue-content-providers:f3c00ba74e3afc054fe29b65156740ac release-management:prepare-airflow-package:3ac14ea6d2b09614959c0ec4fd564789 -release-management:prepare-provider-documentation:452b6165f09755d052501f6003ae3ea1 -release-management:prepare-provider-packages:c85c2997e01b7f9db95dc9a7abb9ea48 +release-management:prepare-provider-documentation:40d540fdfebf6c8ddc4cd151d52b88e6 +release-management:prepare-provider-packages:72dd7c3b19f85024bc9a4939cac5d87c release-management:release-prod-images:c9bc40938e0efad49e51ef66e83f9527 release-management:start-rc-process:6aafbaceabd7b67b9a1af4c2f59abc4c release-management:start-release:acb384d86e02ff5fde1bf971897be17c release-management:verify-provider-packages:88bd609aff6d09d52ab8d80d6e055e7b -release-management:edbaabee1315498f15a27bd853bcd1cc +release-management:926400d9c6d5c491f0182c5520bbfd68 setup:autocomplete:03343478bf1d0cf9c101d454cdb63b68 setup:check-all-params-in-groups:4d0f8c19cbdb56290055d863b08a3376 setup:config:3ffcd35dd24b486ddf1d08b797e3d017 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 784715a8cce83..790b9fe55da49 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #868887 } -.breeze-help-r5 { fill: #68a0b3;font-weight: bold } +.breeze-help-r4 { fill: #68a0b3;font-weight: bold } +.breeze-help-r5 { fill: #868887 } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -190,50 +190,50 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd |    │ -│statsd | trino)                                                                          │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd |    │ +│statsd | trino)                                                                          │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index 84653d88b8b30..2a0435a831639 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -281,12 +281,12 @@ │apache-airflow-providers-samba | apache-airflow-providers-segment |                        │ │apache-airflow-providers-sendgrid | apache-airflow-providers-sftp |                        │ │apache-airflow-providers-singularity | apache-airflow-providers-slack |                    │ -│apache-airflow-providers-snowflake | apache-airflow-providers-sqlite |                     │ -│apache-airflow-providers-ssh | apache-airflow-providers-tableau |                          │ -│apache-airflow-providers-tabular | apache-airflow-providers-telegram |                     │ -│apache-airflow-providers-trino | apache-airflow-providers-vertica |                        │ -│apache-airflow-providers-yandex | apache-airflow-providers-zendesk | docker-stack |        │ -│helm-chart)                                                                                │ +│apache-airflow-providers-smtp | apache-airflow-providers-snowflake |                       │ +│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                           │ +│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                      │ +│apache-airflow-providers-telegram | apache-airflow-providers-trino |                       │ +│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                       │ +│apache-airflow-providers-zendesk | docker-stack | helm-chart)                              │ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index 044fc3b1a354c..ace464046a763 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -176,8 +176,8 @@                                                                   oracle | pagerduty | papermill | plexus | postgres |                                                                   presto | qubole | redis | salesforce | samba |                                                                   segment | sendgrid | sftp | singularity | slack | -                                                                  snowflake | sqlite | ssh | tableau | tabular | -                                                                  telegram | trino | vertica | yandex | zendesk]... +                                                                  smtp | snowflake | sqlite | ssh | tableau | tabular +                                                                  | telegram | trino | vertica | yandex | zendesk]... Generates content for issue to test the release. diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index 986426ccb13a5..807577a6ef118 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -169,9 +169,9 @@                                                                 mongo | mysql | neo4j | odbc | openfaas | opsgenie |                                                                 oracle | pagerduty | papermill | plexus | postgres |                                                                 presto | qubole | redis | salesforce | samba | segment -                                                                | sendgrid | sftp | singularity | slack | snowflake | -                                                                sqlite | ssh | tableau | tabular | telegram | trino | -                                                                vertica | yandex | zendesk]... +                                                                | sendgrid | sftp | singularity | slack | smtp | +                                                                snowflake | sqlite | ssh | tableau | tabular | +                                                                telegram | trino | vertica | yandex | zendesk]... Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index 8d98428323ea4..d75838e2621eb 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -1,4 +1,4 @@ - +                                                            | mysql | neo4j | odbc | openfaas | opsgenie | oracle |                                                            pagerduty | papermill | plexus | postgres | presto | qubole                                                            | redis | salesforce | samba | segment | sendgrid | sftp | -                                                           singularity | slack | snowflake | sqlite | ssh | tableau | -                                                           tabular | telegram | trino | vertica | yandex | zendesk]... - -Prepare sdist/whl packages of Airflow Providers. - -╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                           singularity | slack | smtp | snowflake | sqlite | ssh | +                                                           tableau | tabular | telegram | trino | vertica | yandex | +                                                           zendesk]... + +Prepare sdist/whl packages of Airflow Providers. + +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ From 5a515bd9d5eaa2471044b836e48e620ed4ed9460 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 9 Mar 2023 01:02:29 +0100 Subject: [PATCH 05/15] revert preinstall smtp because the package is not released yet --- docs/apache-airflow/extra-packages-ref.rst | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 1339b2936f89b..7cb7df771b623 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -294,7 +294,7 @@ These are extras that provide support for integration with external systems via +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | sftp | ``pip install 'apache-airflow[sftp]'`` | SFTP hooks, operators and sensors | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| smtp | ``pip install 'apache-airflow[smtp]'`` | SMTP hooks and operators | * | +| smtp | ``pip install 'apache-airflow[smtp]'`` | SMTP hooks and operators | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | sqlite | ``pip install 'apache-airflow[sqlite]'`` | SQLite hooks and operators | * | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ diff --git a/setup.py b/setup.py index 9fe57e133797b..f29a923c7695e 100644 --- a/setup.py +++ b/setup.py @@ -695,7 +695,6 @@ def sort_extras_dependencies() -> dict[str, list[str]]: "ftp", "http", "imap", - "smtp", "sqlite", ] From c0fc184e335e6ae8cda2c5a5e019a4a38bcae5e3 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 9 Mar 2023 02:25:22 +0100 Subject: [PATCH 06/15] load conn from widgets, add test method and update arguments --- airflow/providers/smtp/hooks/smtp.py | 45 +++++++++++++++++-- .../connections/smtp.rst | 8 ++-- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index b2a0306247a03..f88e537545063 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -77,8 +77,8 @@ def get_conn(self) -> SmtpHook: raise AirflowException("SMTP connection is not found.") smtp_user = conn.login smtp_password = conn.password - smtp_starttls = conn.extra_dejson.get("starttls", True) - smtp_retry_limit = conn.extra_dejson.get("retry_limit", 5) + smtp_starttls = not bool(conn.extra_dejson.get("disable_tls", False)) + smtp_retry_limit = int(conn.extra_dejson.get("retry_limit", 5)) for attempt in range(1, smtp_retry_limit + 1): try: @@ -100,7 +100,7 @@ def get_conn(self) -> SmtpHook: def _build_client(self, conn: Connection) -> smtplib.SMTP_SSL | smtplib.SMTP: SMTP: type[smtplib.SMTP_SSL] | type[smtplib.SMTP] - if conn.extra_dejson.get("use_ssl", True): + if not bool(conn.extra_dejson.get("disable_ssl", False)): SMTP = smtplib.SMTP_SSL else: SMTP = smtplib.SMTP @@ -108,10 +108,47 @@ def _build_client(self, conn: Connection) -> smtplib.SMTP_SSL | smtplib.SMTP: smtp_kwargs = {"host": conn.host} if conn.port: smtp_kwargs["port"] = conn.port - smtp_kwargs["timeout"] = conn.extra_dejson.get("timeout", 30) + smtp_kwargs["timeout"] = int(conn.extra_dejson.get("timeout", 30)) return SMTP(**smtp_kwargs) + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Returns connection widgets to add to connection form""" + from flask_appbuilder.fieldwidgets import BS3TextFieldWidget + from flask_babel import lazy_gettext + from wtforms import BooleanField, IntegerField + from wtforms.validators import NumberRange + + return { + "timeout": IntegerField( + lazy_gettext("Connection timeout"), + validators=[NumberRange(min=0)], + widget=BS3TextFieldWidget(), + default=30, + ), + "retry_limit": IntegerField( + lazy_gettext("Number of Retries"), + validators=[NumberRange(min=0)], + widget=BS3TextFieldWidget(), + default=5, + ), + "disable_tls": BooleanField(lazy_gettext("Disable TLS"), default=False), + "disable_ssl": BooleanField(lazy_gettext("Disable SSL"), default=False), + } + + def test_connection(self) -> tuple[bool, str]: + """Test SMTP connectivity from UI""" + try: + smtp_client = self.get_conn().smtp_client + if smtp_client: + status = smtp_client.noop()[0] + if status == 250: + return True, "Connection successfully tested" + except Exception as e: + return False, str(e) + return False, "Failed to establish connection" + def send_email_smtp( self, to: str | Iterable[str], diff --git a/docs/apache-airflow-providers-smtp/connections/smtp.rst b/docs/apache-airflow-providers-smtp/connections/smtp.rst index 4c110515d297d..f046fce9be6a1 100644 --- a/docs/apache-airflow-providers-smtp/connections/smtp.rst +++ b/docs/apache-airflow-providers-smtp/connections/smtp.rst @@ -54,9 +54,9 @@ Port Extra (optional) Specify the extra parameters (as json dictionary) - * ``use_ssl``: If set to false, then a non-ssl connection is being used. Default is true. Also note that changing the ssl option also influences the default port being used. + * ``disable_ssl``: If set to true, then a non-ssl connection is being used. Default is false. Also note that changing the ssl option also influences the default port being used. * ``timeout``: The SMTP connection creation timeout in seconds. Default is 30. - * ``starttls``: Put the SMTP connection in TLS mode. All SMTP commands that follow will be encrypted. Default is true. + * ``disable_tls``: By default the SMTP connection is created in TLS mode. Set to false to disable tls mode. * ``retry_limit``: How many attempts to connect to the server before raising an exception. Default is 5. When specifying the connection in environment variable you should specify @@ -68,12 +68,12 @@ For example: .. code-block:: bash - export AIRFLOW_CONN_SMTP_DEFAULT='smtp://username:password@smtp.sendgrid.net:587?use_ssl=true' + export AIRFLOW_CONN_SMTP_DEFAULT='smtp://username:password@smtp.sendgrid.net:587' Another example for connecting via a non-SSL connection. .. code-block:: bash - export AIRFLOW_CONN_SMTP_NOSSL='smtp://username:password@smtp.sendgrid.net:587?use_ssl=false' + export AIRFLOW_CONN_SMTP_NOSSL='smtp://username:password@smtp.sendgrid.net:587?disable_ssl=true' Note that you can set the port regardless of whether you choose to use ssl or not. The above examples show default ports for SSL and Non-SSL connections. From 2e358a6160eb7280e07aa8627a95d17e2eec6c1c Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Fri, 10 Mar 2023 00:24:53 +0100 Subject: [PATCH 07/15] add from_email to connection extra --- airflow/providers/smtp/hooks/smtp.py | 20 ++++++++++++------- .../connections/smtp.rst | 1 + 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index f88e537545063..04dd135dfbf3a 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -52,6 +52,7 @@ class SmtpHook(BaseHook): def __init__(self, smtp_conn_id: str = default_conn_name) -> None: super().__init__() + self.from_email = None self.smtp_conn_id = smtp_conn_id self.smtp_client: smtplib.SMTP_SSL | smtplib.SMTP | None = None @@ -79,6 +80,7 @@ def get_conn(self) -> SmtpHook: smtp_password = conn.password smtp_starttls = not bool(conn.extra_dejson.get("disable_tls", False)) smtp_retry_limit = int(conn.extra_dejson.get("retry_limit", 5)) + self.from_email = conn.extra_dejson.get("from_email") for attempt in range(1, smtp_retry_limit + 1): try: @@ -117,10 +119,11 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: """Returns connection widgets to add to connection form""" from flask_appbuilder.fieldwidgets import BS3TextFieldWidget from flask_babel import lazy_gettext - from wtforms import BooleanField, IntegerField + from wtforms import BooleanField, IntegerField, StringField from wtforms.validators import NumberRange return { + "from_email": StringField(lazy_gettext("From email"), widget=BS3TextFieldWidget()), "timeout": IntegerField( lazy_gettext("Connection timeout"), validators=[NumberRange(min=0)], @@ -152,9 +155,9 @@ def test_connection(self) -> tuple[bool, str]: def send_email_smtp( self, to: str | Iterable[str], - from_email: str, subject: str, html_content: str, + from_email: str | None = None, files: list[str] | None = None, dryrun: bool = False, cc: str | Iterable[str] | None = None, @@ -167,9 +170,10 @@ def send_email_smtp( """Send an email with html content. :param to: Recipient email address or list of addresses. - :param from_email: Sender email address. :param subject: Email subject. :param html_content: Email body in HTML format. + :param from_email: Sender email address. If it's None, the hook will check if there is an email + provided in the connection, and raises an exception if not. :param files: List of file paths to attach to the email. :param dryrun: If True, the email will not be sent, but all other actions will be performed. :param cc: Carbon copy recipient email address or list of addresses. @@ -179,15 +183,17 @@ def send_email_smtp( :param custom_headers: Dictionary of custom headers to include in the email. :param kwargs: Additional keyword arguments. - >>> send_email( - 'test@example.com', 'source@example.com', 'foo', 'Foo bar', ['/dev/null'], dryrun=True + >>> send_email_smtp( + 'test@example.com', 'foo', 'Foo bar', ['/dev/null'], dryrun=True ) """ if not self.smtp_client: - raise Exception("The 'smtp_client' should be initialized before!") + raise AirflowException("The 'smtp_client' should be initialized before!") + if not from_email and not self.from_email: + raise AirflowException("You should provide `from_email` or define it in the connection.") mime_msg, recipients = self._build_mime_message( - mail_from=from_email, + mail_from=from_email or self.from_email, to=to, subject=subject, html_content=html_content, diff --git a/docs/apache-airflow-providers-smtp/connections/smtp.rst b/docs/apache-airflow-providers-smtp/connections/smtp.rst index f046fce9be6a1..3ad02c6e8b077 100644 --- a/docs/apache-airflow-providers-smtp/connections/smtp.rst +++ b/docs/apache-airflow-providers-smtp/connections/smtp.rst @@ -54,6 +54,7 @@ Port Extra (optional) Specify the extra parameters (as json dictionary) + * ``from_email``: The email address from which you want to send the email. * ``disable_ssl``: If set to true, then a non-ssl connection is being used. Default is false. Also note that changing the ssl option also influences the default port being used. * ``timeout``: The SMTP connection creation timeout in seconds. Default is 30. * ``disable_tls``: By default the SMTP connection is created in TLS mode. Set to false to disable tls mode. From dbb3b59ad478ae25f3c8231517896c77a205d03b Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Fri, 10 Mar 2023 00:36:32 +0100 Subject: [PATCH 08/15] fix duplicated login --- airflow/providers/smtp/hooks/smtp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index 04dd135dfbf3a..810fb5552f2bf 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -90,7 +90,6 @@ def get_conn(self) -> SmtpHook: continue raise AirflowException("Unable to connect to smtp server") - self.smtp_client.login(conn.login, conn.password) if smtp_starttls: self.smtp_client.starttls() if smtp_user and smtp_password: From ff2ccf3ab26dd6d06332436af83a7963fcc68a20 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Fri, 10 Mar 2023 01:13:09 +0100 Subject: [PATCH 09/15] init provider test --- tests/providers/smtp/__init__.py | 17 ++++ tests/providers/smtp/hooks/__init__.py | 17 ++++ tests/providers/smtp/hooks/smtp.py | 133 +++++++++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 tests/providers/smtp/__init__.py create mode 100644 tests/providers/smtp/hooks/__init__.py create mode 100644 tests/providers/smtp/hooks/smtp.py diff --git a/tests/providers/smtp/__init__.py b/tests/providers/smtp/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/tests/providers/smtp/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/smtp/hooks/__init__.py b/tests/providers/smtp/hooks/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/tests/providers/smtp/hooks/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/smtp/hooks/smtp.py b/tests/providers/smtp/hooks/smtp.py new file mode 100644 index 0000000000000..675d4aef238b4 --- /dev/null +++ b/tests/providers/smtp/hooks/smtp.py @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +import smtplib +from unittest.mock import Mock, patch + +import pytest + +from airflow.models import Connection +from airflow.providers.smtp.hooks.smtp import SmtpHook +from airflow.utils import db + +smtplib_string = "airflow.providers.smtp.hooks.smtp.smtplib" + +TEST_EMAILS = ["test1@example.com", "test2@example.com"] + + +def _create_fake_smtp(mock_smtplib, use_ssl=True): + if use_ssl: + mock_conn = Mock(spec=smtplib.SMTP_SSL) + mock_smtplib.SMTP_SSL.return_value = mock_conn + else: + mock_conn = Mock(spec=smtplib.SMTP) + mock_smtplib.SMTP.return_value = mock_conn + + mock_conn.close.return_value = ("OK", []) + + return mock_conn + + +class TestSmtpHook: + def setup_method(self): + db.merge_conn( + Connection( + conn_id="smtp_default", + conn_type="smtp", + host="smtp_server_address", + login="smtp_user", + password="smtp_password", + port=465, + ) + ) + db.merge_conn( + Connection( + conn_id="smtp_nonssl", + conn_type="smtp", + host="smtp_server_address", + login="smtp_user", + password="smtp_password", + port=587, + extra=json.dumps(dict(disable_ssl=True)), + ) + ) + + @patch(smtplib_string) + def test_connect_and_disconnect(self, mock_smtplib): + mock_conn = _create_fake_smtp(mock_smtplib) + + with SmtpHook(): + pass + + mock_smtplib.SMTP_SSL.assert_called_once_with(host="smtp_server_address", port=465, timeout=30) + mock_conn.login.assert_called_once_with("smtp_user", "smtp_password") + assert mock_conn.close.call_count == 1 + + @patch(smtplib_string) + def test_connect_and_disconnect_via_nonssl(self, mock_smtplib): + mock_conn = _create_fake_smtp(mock_smtplib, use_ssl=False) + + with SmtpHook(smtp_conn_id="smtp_nonssl"): + pass + + mock_smtplib.SMTP.assert_called_once_with(host="smtp_server_address", port=587, timeout=30) + mock_conn.login.assert_called_once_with("smtp_user", "smtp_password") + assert mock_conn.close.call_count == 1 + + @patch(smtplib_string) + def test_get_email_address_single_email(self, mock_smtplib): + with SmtpHook() as smtp_hook: + assert smtp_hook._get_email_address_list("test1@example.com") == ["test1@example.com"] + + @patch(smtplib_string) + def test_get_email_address_comma_sep_string(self, mock_smtplib): + with SmtpHook() as smtp_hook: + assert smtp_hook._get_email_address_list("test1@example.com, test2@example.com") == TEST_EMAILS + + @patch(smtplib_string) + def test_get_email_address_colon_sep_string(self, mock_smtplib): + with SmtpHook() as smtp_hook: + assert smtp_hook._get_email_address_list("test1@example.com; test2@example.com") == TEST_EMAILS + + @patch(smtplib_string) + def test_get_email_address_list(self, mock_smtplib): + with SmtpHook() as smtp_hook: + assert ( + smtp_hook._get_email_address_list(["test1@example.com", "test2@example.com"]) == TEST_EMAILS + ) + + @patch(smtplib_string) + def test_get_email_address_tuple(self, mock_smtplib): + with SmtpHook() as smtp_hook: + assert ( + smtp_hook._get_email_address_list(("test1@example.com", "test2@example.com")) == TEST_EMAILS + ) + + @patch(smtplib_string) + def test_get_email_address_invalid_type(self, mock_smtplib): + with pytest.raises(TypeError): + with SmtpHook() as smtp_hook: + smtp_hook._get_email_address_list(1) + + @patch(smtplib_string) + def test_get_email_address_invalid_type_in_iterable(self, mock_smtplib): + with pytest.raises(TypeError): + with SmtpHook() as smtp_hook: + smtp_hook._get_email_address_list(["test1@example.com", 2]) From 42c009c4c84218380e127bab9b08c4af1559f7b8 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sat, 11 Mar 2023 01:54:00 +0100 Subject: [PATCH 10/15] add full tests and fix some bugs --- airflow/providers/smtp/hooks/smtp.py | 91 +++++++++++---- tests/providers/smtp/hooks/smtp.py | 159 ++++++++++++++++++++++++++- 2 files changed, 227 insertions(+), 23 deletions(-) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index 810fb5552f2bf..c0552d6207ec9 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -52,8 +52,8 @@ class SmtpHook(BaseHook): def __init__(self, smtp_conn_id: str = default_conn_name) -> None: super().__init__() - self.from_email = None self.smtp_conn_id = smtp_conn_id + self.smtp_connection: Connection | None = None self.smtp_client: smtplib.SMTP_SSL | smtplib.SMTP | None = None def __enter__(self) -> SmtpHook: @@ -73,43 +73,38 @@ def get_conn(self) -> SmtpHook: """ if not self.smtp_client: try: - conn = self.get_connection(self.smtp_conn_id) + self.smtp_connection = self.get_connection(self.smtp_conn_id) except AirflowNotFoundException: raise AirflowException("SMTP connection is not found.") - smtp_user = conn.login - smtp_password = conn.password - smtp_starttls = not bool(conn.extra_dejson.get("disable_tls", False)) - smtp_retry_limit = int(conn.extra_dejson.get("retry_limit", 5)) - self.from_email = conn.extra_dejson.get("from_email") - for attempt in range(1, smtp_retry_limit + 1): + for attempt in range(1, self.smtp_retry_limit + 1): try: - self.smtp_client = self._build_client(conn=conn) + self.smtp_client = self._build_client() except smtplib.SMTPServerDisconnected: - if attempt < smtp_retry_limit: + if attempt < self.smtp_retry_limit: continue raise AirflowException("Unable to connect to smtp server") - if smtp_starttls: + if self.smtp_starttls: self.smtp_client.starttls() - if smtp_user and smtp_password: - self.smtp_client.login(smtp_user, smtp_password) + if self.smtp_user and self.smtp_password: + self.smtp_client.login(self.smtp_user, self.smtp_password) break return self - def _build_client(self, conn: Connection) -> smtplib.SMTP_SSL | smtplib.SMTP: + def _build_client(self) -> smtplib.SMTP_SSL | smtplib.SMTP: SMTP: type[smtplib.SMTP_SSL] | type[smtplib.SMTP] - if not bool(conn.extra_dejson.get("disable_ssl", False)): + if self.use_ssl: SMTP = smtplib.SMTP_SSL else: SMTP = smtplib.SMTP - smtp_kwargs = {"host": conn.host} - if conn.port: - smtp_kwargs["port"] = conn.port - smtp_kwargs["timeout"] = int(conn.extra_dejson.get("timeout", 30)) + smtp_kwargs: dict[str, Any] = {"host": self.host} + if self.port: + smtp_kwargs["port"] = self.port + smtp_kwargs["timeout"] = self.timeout return SMTP(**smtp_kwargs) @@ -188,11 +183,12 @@ def send_email_smtp( """ if not self.smtp_client: raise AirflowException("The 'smtp_client' should be initialized before!") - if not from_email and not self.from_email: + from_email = from_email or self.from_email + if not from_email: raise AirflowException("You should provide `from_email` or define it in the connection.") mime_msg, recipients = self._build_mime_message( - mail_from=from_email or self.from_email, + mail_from=from_email, to=to, subject=subject, html_content=html_content, @@ -204,7 +200,16 @@ def send_email_smtp( custom_headers=custom_headers, ) if not dryrun: - self.smtp_client.sendmail(from_addr=from_email, to_addrs=recipients, msg=mime_msg.as_string()) + for attempt in range(1, self.smtp_retry_limit + 1): + try: + self.smtp_client.sendmail( + from_addr=from_email, to_addrs=recipients, msg=mime_msg.as_string() + ) + except smtplib.SMTPServerDisconnected as e: + if attempt < self.smtp_retry_limit: + continue + raise e + break def _build_mime_message( self, @@ -302,3 +307,45 @@ def _get_email_list_from_str(self, addresses: str) -> list[str]: """ pattern = r"\s*[,;]\s*" return [address for address in re.split(pattern, addresses)] + + @property + def conn(self) -> Connection: + if not self.smtp_connection: + raise AirflowException("The smtp connection should be loaded before!") + return self.smtp_connection + + @property + def smtp_retry_limit(self) -> int: + return int(self.conn.extra_dejson.get("retry_limit", 5)) + + @property + def from_email(self) -> str | None: + return self.conn.extra_dejson.get("from_email") + + @property + def smtp_user(self) -> str: + return self.conn.login + + @property + def smtp_password(self) -> str: + return self.conn.password + + @property + def smtp_starttls(self) -> bool: + return not bool(self.conn.extra_dejson.get("disable_tls", False)) + + @property + def host(self) -> str: + return self.conn.host + + @property + def port(self) -> int: + return self.conn.port + + @property + def timeout(self) -> int: + return int(self.conn.extra_dejson.get("timeout", 30)) + + @property + def use_ssl(self) -> bool: + return not bool(self.conn.extra_dejson.get("disable_ssl", False)) diff --git a/tests/providers/smtp/hooks/smtp.py b/tests/providers/smtp/hooks/smtp.py index 675d4aef238b4..567f5580a29f6 100644 --- a/tests/providers/smtp/hooks/smtp.py +++ b/tests/providers/smtp/hooks/smtp.py @@ -18,7 +18,10 @@ from __future__ import annotations import json +import os import smtplib +import tempfile +from email.mime.application import MIMEApplication from unittest.mock import Mock, patch import pytest @@ -26,6 +29,7 @@ from airflow.models import Connection from airflow.providers.smtp.hooks.smtp import SmtpHook from airflow.utils import db +from airflow.utils.session import create_session smtplib_string = "airflow.providers.smtp.hooks.smtp.smtplib" @@ -55,6 +59,7 @@ def setup_method(self): login="smtp_user", password="smtp_password", port=465, + extra=json.dumps(dict(from_email="from")), ) ) db.merge_conn( @@ -65,7 +70,7 @@ def setup_method(self): login="smtp_user", password="smtp_password", port=587, - extra=json.dumps(dict(disable_ssl=True)), + extra=json.dumps(dict(disable_ssl=True, from_email="from")), ) ) @@ -131,3 +136,155 @@ def test_get_email_address_invalid_type_in_iterable(self, mock_smtplib): with pytest.raises(TypeError): with SmtpHook() as smtp_hook: smtp_hook._get_email_address_list(["test1@example.com", 2]) + + @patch(smtplib_string) + def test_build_mime_message(self, mock_smtplib): + mail_from = "from@example.com" + mail_to = "to@example.com" + subject = "test subject" + html_content = "Test" + custom_headers = {"Reply-To": "reply_to@example.com"} + with SmtpHook() as smtp_hook: + msg, recipients = smtp_hook._build_mime_message( + mail_from=mail_from, + to=mail_to, + subject=subject, + html_content=html_content, + custom_headers=custom_headers, + ) + + assert "From" in msg + assert "To" in msg + assert "Subject" in msg + assert "Reply-To" in msg + assert [mail_to] == recipients + assert msg["To"] == ",".join(recipients) + + @patch(smtplib_string) + def test_send_smtp(self, mock_smtplib): + mock_send_mime = mock_smtplib.SMTP_SSL().sendmail + with SmtpHook() as smtp_hook, tempfile.NamedTemporaryFile() as attachment: + attachment.write(b"attachment") + attachment.seek(0) + smtp_hook.send_email_smtp("to", "subject", "content", files=[attachment.name]) + assert mock_send_mime.called + _, call_args = mock_send_mime.call_args + assert "from" == call_args["from_addr"] + assert ["to"] == call_args["to_addrs"] + msg = call_args["msg"] + assert "Subject: subject" in msg + assert "From: from" in msg + filename = 'attachment; filename="' + os.path.basename(attachment.name) + '"' + assert filename in msg + mimeapp = MIMEApplication("attachment") + assert mimeapp.get_payload() in msg + + @patch("airflow.providers.smtp.hooks.smtp.SmtpHook.get_connection") + @patch(smtplib_string) + def test_hook_conn(self, mock_smtplib, mock_hook_conn): + mock_conn = Mock() + mock_conn.login = "user" + mock_conn.password = "password" + mock_conn.extra_dejson = { + "disable_ssl": False, + } + mock_hook_conn.return_value = mock_conn + smtp_client_mock = mock_smtplib.SMTP_SSL() + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp("to", "subject", "content", from_email="from") + mock_hook_conn.assert_called_with("smtp_default") + smtp_client_mock.login.assert_called_once_with("user", "password") + smtp_client_mock.sendmail.assert_called_once() + assert smtp_client_mock.close.called + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + def test_send_mime_ssl(self, mock_smtp, mock_smtp_ssl): + mock_smtp_ssl.return_value = Mock() + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp("to", "subject", "content", from_email="from") + assert not mock_smtp.called + mock_smtp_ssl.assert_called_once_with(host="smtp_server_address", port=465, timeout=30) + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + def test_send_mime_nossl(self, mock_smtp, mock_smtp_ssl): + mock_smtp.return_value = Mock() + with SmtpHook(smtp_conn_id="smtp_nonssl") as smtp_hook: + smtp_hook.send_email_smtp("to", "subject", "content", from_email="from") + assert not mock_smtp_ssl.called + mock_smtp.assert_called_once_with(host="smtp_server_address", port=587, timeout=30) + + @patch("smtplib.SMTP") + def test_send_mime_noauth(self, mock_smtp): + mock_smtp.return_value = Mock() + conn = Connection( + conn_id="smtp_noauth", + conn_type="smtp", + host="smtp_server_address", + login=None, + password="None", + port=587, + extra=json.dumps(dict(disable_ssl=True, from_email="from")), + ) + db.merge_conn(conn) + with SmtpHook(smtp_conn_id="smtp_noauth") as smtp_hook: + smtp_hook.send_email_smtp("to", "subject", "content", from_email="from") + mock_smtp.assert_called_once_with(host="smtp_server_address", port=587, timeout=30) + assert not mock_smtp.login.called + with create_session() as session: + session.query(Connection).filter(Connection.id == conn.id).delete() + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + def test_send_mime_dryrun(self, mock_smtp, mock_smtp_ssl): + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp("to", "subject", "content", dryrun=True) + assert not mock_smtp.sendmail.called + assert not mock_smtp_ssl.sendmail.called + + @patch("smtplib.SMTP_SSL") + def test_send_mime_ssl_complete_failure(self, mock_smtp_ssl): + mock_smtp_ssl().sendmail.side_effect = smtplib.SMTPServerDisconnected() + with SmtpHook() as smtp_hook: + with pytest.raises(smtplib.SMTPServerDisconnected): + smtp_hook.send_email_smtp("to", "subject", "content") + assert mock_smtp_ssl().sendmail.call_count == 5 + + @patch("email.message.Message.as_string") + @patch("smtplib.SMTP_SSL") + def test_send_mime_partial_failure(self, mock_smtp_ssl, mime_message_mock): + mime_message_mock.return_value = "msg" + final_mock = Mock() + side_effects = [smtplib.SMTPServerDisconnected(), smtplib.SMTPServerDisconnected(), final_mock] + mock_smtp_ssl.side_effect = side_effects + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp("to", "subject", "content") + assert mock_smtp_ssl.call_count == side_effects.index(final_mock) + 1 + assert final_mock.starttls.called + final_mock.sendmail.assert_called_once_with(from_addr="from", to_addrs=["to"], msg="msg") + assert final_mock.close.called + + @patch("airflow.models.connection.Connection") + @patch("smtplib.SMTP_SSL") + def test_send_mime_custom_timeout_retrylimit(self, mock_smtp_ssl, connection_mock): + mock_smtp_ssl().sendmail.side_effect = smtplib.SMTPServerDisconnected() + custom_retry_limit = 10 + custom_timeout = 60 + fake_conn = Connection( + conn_id="mock_conn", + conn_type="smtp", + host="smtp_server_address", + login="smtp_user", + password="smtp_password", + port=465, + extra=json.dumps(dict(from_email="from", timeout=custom_timeout, retry_limit=custom_retry_limit)), + ) + connection_mock.get_connection_from_secrets.return_value = fake_conn + with SmtpHook() as smtp_hook: + with pytest.raises(smtplib.SMTPServerDisconnected): + smtp_hook.send_email_smtp("to", "subject", "content") + mock_smtp_ssl.assert_any_call( + host=fake_conn.host, port=fake_conn.port, timeout=fake_conn.extra_dejson["timeout"] + ) + assert mock_smtp_ssl().sendmail.call_count == 10 From 3f286eca37164493600945d69993595488061b95 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sat, 11 Mar 2023 01:56:20 +0100 Subject: [PATCH 11/15] hide unused fields from the connections UI --- airflow/providers/smtp/hooks/smtp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index c0552d6207ec9..dac80dabb03d5 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -349,3 +349,11 @@ def timeout(self) -> int: @property def use_ssl(self) -> bool: return not bool(self.conn.extra_dejson.get("disable_ssl", False)) + + @staticmethod + def get_ui_field_behaviour() -> dict[str, Any]: + """Returns custom field behaviour""" + return { + "hidden_fields": ["schema", "extra"], + "relabeling": {}, + } From 5e90c9892de3ca0d63540f4b519ce1c4cf4e7ba8 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sat, 11 Mar 2023 02:01:25 +0100 Subject: [PATCH 12/15] add hook docstring --- airflow/providers/smtp/hooks/smtp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index dac80dabb03d5..521e47fca135f 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -39,7 +39,10 @@ class SmtpHook(BaseHook): """ - TODO: add docstring + This hook connects to a mail server by using the smtp protocol. + + .. note:: Please call this Hook as context manager via `with` + to automatically open and close the connection to the mail server. :param smtp_conn_id: The :ref:`smtp connection id ` that contains the information used to authenticate the client. From 72d94f11195d025549d1a2398d10927701d63c2b Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sat, 11 Mar 2023 02:07:40 +0100 Subject: [PATCH 13/15] empty changelog --- airflow/providers/smtp/CHANGELOG.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/airflow/providers/smtp/CHANGELOG.rst b/airflow/providers/smtp/CHANGELOG.rst index f7968bbf49c60..320da6fc450c4 100644 --- a/airflow/providers/smtp/CHANGELOG.rst +++ b/airflow/providers/smtp/CHANGELOG.rst @@ -23,8 +23,3 @@ Changelog --------- - -1.0.0 -..... - -Initial version of the provider. From ab29aa4792e984c75f10430fbd6764d3b5622d13 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Mon, 13 Mar 2023 21:36:02 +0100 Subject: [PATCH 14/15] add 1.0.0 to Changelog --- airflow/providers/smtp/CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/smtp/CHANGELOG.rst b/airflow/providers/smtp/CHANGELOG.rst index 320da6fc450c4..f7968bbf49c60 100644 --- a/airflow/providers/smtp/CHANGELOG.rst +++ b/airflow/providers/smtp/CHANGELOG.rst @@ -23,3 +23,8 @@ Changelog --------- + +1.0.0 +..... + +Initial version of the provider. From 3d71320d792b7581e527b30fe9d92ac308da0ae8 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Mon, 13 Mar 2023 21:40:03 +0100 Subject: [PATCH 15/15] add minimum Airflow 2.3.0 as dependency --- airflow/providers/smtp/provider.yaml | 3 ++- generated/provider_dependencies.json | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml index 0ad50985f6129..b7b96eba23aa9 100644 --- a/airflow/providers/smtp/provider.yaml +++ b/airflow/providers/smtp/provider.yaml @@ -25,7 +25,8 @@ description: | versions: - 1.0.0 -dependencies: [] +dependencies: + - apache-airflow>=2.3.0 integrations: - integration-name: Simple Mail Transfer Protocol (SMTP) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 4bc75054dad53..121ebb5f79d09 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -683,7 +683,9 @@ ] }, "smtp": { - "deps": [], + "deps": [ + "apache-airflow>=2.3.0" + ], "cross-providers-deps": [] }, "snowflake": {