diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py new file mode 100644 index 0000000000000..e2ac1a35a0d9b --- /dev/null +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module contains a web hook for Chime.""" +from __future__ import annotations + +import json +import re +from typing import Any + +from airflow.exceptions import AirflowException +from airflow.providers.http.hooks.http import HttpHook + + +class ChimeWebhookHook(HttpHook): + """Interact with Chime Web Hooks to create notifications. + + .. warning:: This hook is only designed to work with web hooks and not chat bots. + + :param chime_conn_id: Chime connection ID with Endpoint as "https://hooks.chime.aws" and + the webhook token in the form of ```{webhook.id}?token{webhook.token}``` + + """ + + conn_name_attr = "chime_conn_id" + default_conn_name = "chime_default" + conn_type = "chime" + hook_name = "Chime Web Hook" + + def __init__( + self, + chime_conn_id: str, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self.webhook_endpoint = self._get_webhook_endpoint(chime_conn_id) + + def _get_webhook_endpoint(self, conn_id: str) -> str: + """ + Given a Chime conn_id return the default webhook endpoint. + + :param conn_id: The provided connection ID. + :return: Endpoint(str) for chime webhook. + """ + conn = self.get_connection(conn_id) + token = conn.get_password() + if token is None: + raise AirflowException("Webhook token field is missing and is required.") + url = conn.schema + "://" + conn.host + endpoint = url + token + # Check to make sure the endpoint matches what Chime expects + if not re.match(r"^[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token): + raise AirflowException( + "Expected Chime webhook token in the form of '{webhook.id}?token={webhook.token}'." + ) + + return endpoint + + def _build_chime_payload(self, message: str) -> str: + """ + Builds payload for Chime and ensures messages do not exceed max length allowed. + + :param message: The message you want to send to your Chime room. + (max 4096 characters) + """ + payload: dict[str, Any] = {} + # We need to make sure that the message does not exceed the max length for Chime + if len(message) > 4096: + raise AirflowException("Chime message must be 4096 characters or less.") + + payload["Content"] = message + return json.dumps(payload) + + def send_message(self, message: str) -> None: + """Execute calling the Chime webhook endpoint. + + :param message: The message you want to send to your Chime room. + (max 4096 characters) + + """ + chime_payload = self._build_chime_payload(message) + self.run( + endpoint=self.webhook_endpoint, data=chime_payload, headers={"Content-type": "application/json"} + ) + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Returns custom field behaviour to only get what is needed for Chime webhooks to function.""" + return { + "hidden_fields": ["login", "port", "extra"], + "relabeling": { + "host": "Chime Webhook Endpoint", + "password": "Webhook Token", + }, + "placeholders": { + "schema": "https", + "host": "hooks.chime.aws/incomingwebhook/", + "password": "T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX", + }, + } diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 223bc553efb78..5439f9c8cbb76 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -61,6 +61,7 @@ versions: dependencies: - apache-airflow>=2.4.0 - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-http - boto3>=1.24.0 - asgiref # watchtower 3 has been released end Jan and introduced breaking change across the board that might @@ -84,6 +85,10 @@ integrations: how-to-guide: - /docs/apache-airflow-providers-amazon/operators/athena.rst tags: [aws] + - integration-name: Amazon Chime + external-doc-url: https://aws.amazon.com/chime/ + logo: /integration-logos/aws/Amazon-Chime-light-bg.png + tags: [aws] - integration-name: Amazon CloudFormation external-doc-url: https://aws.amazon.com/cloudformation/ logo: /integration-logos/aws/AWS-CloudFormation_light-bg@4x.png @@ -407,6 +412,9 @@ hooks: - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.hooks.athena + - integration-name: Amazon Chime + python-modules: + - airflow.providers.amazon.aws.hooks.chime - integration-name: Amazon DynamoDB python-modules: - airflow.providers.amazon.aws.hooks.dynamodb @@ -624,11 +632,14 @@ extra-links: connection-types: - hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook connection-type: aws + - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook + connection-type: chime - hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook connection-type: emr - hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook connection-type: redshift + secrets-backends: - airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend - airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index dac17ce81560a..0081b6a2a795b 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -191,7 +191,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/providers/http/file.py", ), { - "affected-providers-list-as-string": "airbyte apache.livy " + "affected-providers-list-as-string": "airbyte amazon apache.livy " "dbt.cloud dingding discord http", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -200,11 +200,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", - "run-amazon-tests": "false", + "run-amazon-tests": "true", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always " + "parallel-test-types-list-as-string": "Providers[amazon] Always " "Providers[airbyte,apache.livy,dbt.cloud,dingding,discord,http]", }, id="Helm tests, http and all relevant providers, kubernetes tests and " @@ -311,7 +311,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/__init__.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google imap " + "common.sql exasol ftp google http imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -325,7 +325,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,imap," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap," "mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", @@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/file.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google imap " + "common.sql exasol ftp google http imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -368,7 +368,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always " "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp," - "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", + "http,imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", ), diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst b/docs/apache-airflow-providers-amazon/connections/chime.rst new file mode 100644 index 0000000000000..94a356f81d805 --- /dev/null +++ b/docs/apache-airflow-providers-amazon/connections/chime.rst @@ -0,0 +1,60 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:chime: + +Amazon Chime Connection +========================== + +The Chime connection works with calling Chime webhooks to send messages to a chime room. + +Authenticating to Amazon Chime +--------------------------------- +When a webhook is created in a Chime room a token will be included in the url for authentication. + + +Default Connection IDs +---------------------- + +The default connection ID is ``chime_default``. + +Configuring the Connection +-------------------------- +Chime Webhook Endpoint: + Specify the entire url or the base of the url for the service. + + +Chime Webhook token: + The token for authentication including the webhook ID. + +Schema: + Whether or not the endpoint should be http or https + + +Examples +-------- + +**Connection** + +* **Chime Webhook Endpoint**: hooks.chime.aws +* **Chime Webhook Token**: + +.. code-block:: text + + abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken + +* **Schema**: https diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index dff699d55a864..e0b148425c035 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -134,6 +134,7 @@ Dependent package `apache-airflow-providers-exasol `_ ``exasol`` `apache-airflow-providers-ftp `_ ``ftp`` `apache-airflow-providers-google `_ ``google`` +`apache-airflow-providers-http `_ ``http`` `apache-airflow-providers-imap `_ ``imap`` `apache-airflow-providers-mongo `_ ``mongo`` `apache-airflow-providers-salesforce `_ ``salesforce`` diff --git a/docs/integration-logos/aws/Amazon-Chime-light-bg.png b/docs/integration-logos/aws/Amazon-Chime-light-bg.png new file mode 100644 index 0000000000000..bd412e9815147 Binary files /dev/null and b/docs/integration-logos/aws/Amazon-Chime-light-bg.png differ diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 2a1ae40e3faf8..2755e9f08bc1c 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -20,6 +20,7 @@ "amazon": { "deps": [ "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-http", "apache-airflow>=2.4.0", "asgiref", "asgiref", @@ -40,6 +41,7 @@ "exasol", "ftp", "google", + "http", "imap", "mongo", "salesforce", diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py new file mode 100644 index 0000000000000..d5130964f4d27 --- /dev/null +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -0,0 +1,105 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json + +import pytest + +from airflow.exceptions import AirflowException +from airflow.models import Connection +from airflow.providers.amazon.aws.hooks.chime import ChimeWebhookHook +from airflow.utils import db + + +class TestChimeWebhookHook: + + _config = { + "chime_conn_id": "default-chime-webhook", + "webhook_endpoint": "incomingwebhooks/abcd-1134-ZeDA?token=somechimetoken-111", + "message": "your message here", + } + + expected_payload_dict = { + "Content": _config["message"], + } + + expected_payload = json.dumps(expected_payload_dict) + + def setup_method(self): + db.merge_conn( + Connection( + conn_id="default-chime-webhook", + conn_type="chime", + host="hooks.chime.aws/incomingwebhooks/", + password="abcd-1134-ZeDA?token=somechimetoken111", + schema="https", + ) + ) + db.merge_conn( + Connection( + conn_id="chime-bad-url", + conn_type="chime", + host="https://hooks.chime.aws/", + password="somebadurl", + schema="https", + ) + ) + + def test_get_webhook_endpoint_invalid_url(self): + # Given + + # When/Then + expected_message = r"Expected Chime webhook token in the form" + with pytest.raises(AirflowException, match=expected_message): + ChimeWebhookHook(chime_conn_id="chime-bad-url") + + def test_get_webhook_endpoint_conn_id(self): + # Given + conn_id = "default-chime-webhook" + hook = ChimeWebhookHook(chime_conn_id=conn_id) + expected_webhook_endpoint = ( + "https://hooks.chime.aws/incomingwebhooks/abcd-1134-ZeDA?token=somechimetoken111" + ) + + # When + webhook_endpoint = hook._get_webhook_endpoint(conn_id) + + # Then + assert webhook_endpoint == expected_webhook_endpoint + + def test_build_chime_payload(self): + # Given + hook = ChimeWebhookHook(self._config["chime_conn_id"]) + message = self._config["message"] + # When + payload = hook._build_chime_payload(message) + # Then + assert self.expected_payload == payload + + def test_build_chime_payload_message_length(self): + # Given + self._config.copy() + # create message over the character limit + message = "c" * 4097 + hook = ChimeWebhookHook(self._config["chime_conn_id"]) + + # When/Then + expected_message = "Chime message must be 4096 characters or less." + with pytest.raises(AirflowException, match=expected_message): + hook._build_chime_payload(message)