diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 268653ffa58d9..cd3958cdffacb 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -43,7 +43,7 @@ body: - celery - cloudant - cncf-kubernetes - - core-sql + - common-sql - databricks - datadog - dbt-cloud diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 92b98dfca1c40..323172ad31474 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -620,15 +620,15 @@ This is the full list of those extras: airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas, aws, azure, cassandra, -celery, cgroups, cloudant, cncf.kubernetes, core.sql, crypto, dask, databricks, datadog, dbt.cloud, -deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, -elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, -grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, -leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, -neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, -postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, -sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, -vertica, virtualenv, webhdfs, winrm, yandex, zendesk +celery, cgroups, cloudant, cncf.kubernetes, common.sql, crypto, dask, databricks, datadog, +dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, +druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, +google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, +kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, +mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, +pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, +sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, +telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE Provider packages diff --git a/INSTALL b/INSTALL index 35b39b0b89918..164b9622a3cf0 100644 --- a/INSTALL +++ b/INSTALL @@ -97,15 +97,15 @@ The list of available extras: airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas, aws, azure, cassandra, -celery, cgroups, cloudant, cncf.kubernetes, core.sql, crypto, dask, databricks, datadog, dbt.cloud, -deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, -elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, -grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, -leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, -neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, -postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, -sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, -vertica, virtualenv, webhdfs, winrm, yandex, zendesk +celery, cgroups, cloudant, cncf.kubernetes, common.sql, crypto, dask, databricks, datadog, +dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, +druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, +google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, +kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, +mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, +pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, +sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, +telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/hooks/dbapi.py b/airflow/hooks/dbapi.py index d3d3fa5fc8972..1dc2908eb9062 100644 --- a/airflow/hooks/dbapi.py +++ b/airflow/hooks/dbapi.py @@ -15,367 +15,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from contextlib import closing -from datetime import datetime -from typing import Any, Optional +import warnings -from sqlalchemy import create_engine +from airflow.providers.common.sql.hooks.sql import ConnectorProtocol # noqa +from airflow.providers.common.sql.hooks.sql import DbApiHook # noqa -from airflow.exceptions import AirflowException -from airflow.hooks.base import BaseHook -from airflow.typing_compat import Protocol - - -class ConnectorProtocol(Protocol): - """A protocol where you can connect to a database.""" - - def connect(self, host: str, port: int, username: str, schema: str) -> Any: - """ - Connect to a database. - - :param host: The database host to connect to. - :param port: The database port to connect to. - :param username: The database username used for the authentication. - :param schema: The database schema to connect to. - :return: the authorized connection object. - """ - - -######################################################################################### -# # -# Note! Be extra careful when changing this file. This hook is used as a base for # -# a number of DBApi-related hooks and providers depend on the methods implemented # -# here. Whatever you add here, has to backwards compatible unless # -# `>=` is added to providers' requirements using the new feature # -# # -######################################################################################### -class DbApiHook(BaseHook): - """ - Abstract base class for sql hooks. - - :param schema: Optional DB schema that overrides the schema specified in the connection. Make sure that - if you change the schema parameter value in the constructor of the derived Hook, such change - should be done before calling the ``DBApiHook.__init__()``. - :param log_sql: Whether to log SQL query when it's executed. Defaults to *True*. - """ - - # Override to provide the connection name. - conn_name_attr = None # type: str - # Override to have a default connection id for a particular dbHook - default_conn_name = 'default_conn_id' - # Override if this db supports autocommit. - supports_autocommit = False - # Override with the object that exposes the connect method - connector = None # type: Optional[ConnectorProtocol] - # Override with db-specific query to check connection - _test_connection_sql = "select 1" - - def __init__(self, *args, schema: Optional[str] = None, log_sql: bool = True, **kwargs): - super().__init__() - if not self.conn_name_attr: - raise AirflowException("conn_name_attr is not defined") - elif len(args) == 1: - setattr(self, self.conn_name_attr, args[0]) - elif self.conn_name_attr not in kwargs: - setattr(self, self.conn_name_attr, self.default_conn_name) - else: - setattr(self, self.conn_name_attr, kwargs[self.conn_name_attr]) - # We should not make schema available in deriving hooks for backwards compatibility - # If a hook deriving from DBApiHook has a need to access schema, then it should retrieve it - # from kwargs and store it on its own. We do not run "pop" here as we want to give the - # Hook deriving from the DBApiHook to still have access to the field in it's constructor - self.__schema = schema - self.log_sql = log_sql - - def get_conn(self): - """Returns a connection object""" - db = self.get_connection(getattr(self, self.conn_name_attr)) - return self.connector.connect(host=db.host, port=db.port, username=db.login, schema=db.schema) - - def get_uri(self) -> str: - """ - Extract the URI from the connection. - - :return: the extracted uri. - """ - conn = self.get_connection(getattr(self, self.conn_name_attr)) - conn.schema = self.__schema or conn.schema - return conn.get_uri() - - def get_sqlalchemy_engine(self, engine_kwargs=None): - """ - Get an sqlalchemy_engine object. - - :param engine_kwargs: Kwargs used in :func:`~sqlalchemy.create_engine`. - :return: the created engine. - """ - if engine_kwargs is None: - engine_kwargs = {} - return create_engine(self.get_uri(), **engine_kwargs) - - def get_pandas_df(self, sql, parameters=None, **kwargs): - """ - Executes the sql and returns a pandas dataframe - - :param sql: the sql statement to be executed (str) or a list of - sql statements to execute - :param parameters: The parameters to render the SQL query with. - :param kwargs: (optional) passed into pandas.io.sql.read_sql method - """ - try: - from pandas.io import sql as psql - except ImportError: - raise Exception("pandas library not installed, run: pip install 'apache-airflow[pandas]'.") - - with closing(self.get_conn()) as conn: - return psql.read_sql(sql, con=conn, params=parameters, **kwargs) - - def get_pandas_df_by_chunks(self, sql, parameters=None, *, chunksize, **kwargs): - """ - Executes the sql and returns a generator - - :param sql: the sql statement to be executed (str) or a list of - sql statements to execute - :param parameters: The parameters to render the SQL query with - :param chunksize: number of rows to include in each chunk - :param kwargs: (optional) passed into pandas.io.sql.read_sql method - """ - try: - from pandas.io import sql as psql - except ImportError: - raise Exception("pandas library not installed, run: pip install 'apache-airflow[pandas]'.") - - with closing(self.get_conn()) as conn: - yield from psql.read_sql(sql, con=conn, params=parameters, chunksize=chunksize, **kwargs) - - def get_records(self, sql, parameters=None): - """ - Executes the sql and returns a set of records. - - :param sql: the sql statement to be executed (str) or a list of - sql statements to execute - :param parameters: The parameters to render the SQL query with. - """ - with closing(self.get_conn()) as conn: - with closing(conn.cursor()) as cur: - if parameters is not None: - cur.execute(sql, parameters) - else: - cur.execute(sql) - return cur.fetchall() - - def get_first(self, sql, parameters=None): - """ - Executes the sql and returns the first resulting row. - - :param sql: the sql statement to be executed (str) or a list of - sql statements to execute - :param parameters: The parameters to render the SQL query with. - """ - with closing(self.get_conn()) as conn: - with closing(conn.cursor()) as cur: - if parameters is not None: - cur.execute(sql, parameters) - else: - cur.execute(sql) - return cur.fetchone() - - def run(self, sql, autocommit=False, parameters=None, handler=None): - """ - Runs a command or a list of commands. Pass a list of sql - statements to the sql parameter to get them to execute - sequentially - - :param sql: the sql statement to be executed (str) or a list of - sql statements to execute - :param autocommit: What to set the connection's autocommit setting to - before executing the query. - :param parameters: The parameters to render the SQL query with. - :param handler: The result handler which is called with the result of each statement. - :return: query results if handler was provided. - """ - scalar = isinstance(sql, str) - if scalar: - sql = [sql] - - if sql: - self.log.debug("Executing %d statements", len(sql)) - else: - raise ValueError("List of SQL statements is empty") - - with closing(self.get_conn()) as conn: - if self.supports_autocommit: - self.set_autocommit(conn, autocommit) - - with closing(conn.cursor()) as cur: - results = [] - for sql_statement in sql: - self._run_command(cur, sql_statement, parameters) - if handler is not None: - result = handler(cur) - results.append(result) - - # If autocommit was set to False for db that supports autocommit, - # or if db does not supports autocommit, we do a manual commit. - if not self.get_autocommit(conn): - conn.commit() - - if handler is None: - return None - - if scalar: - return results[0] - - return results - - def _run_command(self, cur, sql_statement, parameters): - """Runs a statement using an already open cursor.""" - if self.log_sql: - self.log.info("Running statement: %s, parameters: %s", sql_statement, parameters) - - if parameters: - cur.execute(sql_statement, parameters) - else: - cur.execute(sql_statement) - - # According to PEP 249, this is -1 when query result is not applicable. - if cur.rowcount >= 0: - self.log.info("Rows affected: %s", cur.rowcount) - - def set_autocommit(self, conn, autocommit): - """Sets the autocommit flag on the connection""" - if not self.supports_autocommit and autocommit: - self.log.warning( - "%s connection doesn't support autocommit but autocommit activated.", - getattr(self, self.conn_name_attr), - ) - conn.autocommit = autocommit - - def get_autocommit(self, conn): - """ - Get autocommit setting for the provided connection. - Return True if conn.autocommit is set to True. - Return False if conn.autocommit is not set or set to False or conn - does not support autocommit. - - :param conn: Connection to get autocommit setting from. - :return: connection autocommit setting. - :rtype: bool - """ - return getattr(conn, 'autocommit', False) and self.supports_autocommit - - def get_cursor(self): - """Returns a cursor""" - return self.get_conn().cursor() - - @staticmethod - def _generate_insert_sql(table, values, target_fields, replace, **kwargs): - """ - Static helper method that generates the INSERT SQL statement. - The REPLACE variant is specific to MySQL syntax. - - :param table: Name of the target table - :param values: The row to insert into the table - :param target_fields: The names of the columns to fill in the table - :param replace: Whether to replace instead of insert - :return: The generated INSERT or REPLACE SQL statement - :rtype: str - """ - placeholders = [ - "%s", - ] * len(values) - - if target_fields: - target_fields = ", ".join(target_fields) - target_fields = f"({target_fields})" - else: - target_fields = '' - - if not replace: - sql = "INSERT INTO " - else: - sql = "REPLACE INTO " - sql += f"{table} {target_fields} VALUES ({','.join(placeholders)})" - return sql - - def insert_rows(self, table, rows, target_fields=None, commit_every=1000, replace=False, **kwargs): - """ - A generic way to insert a set of tuples into a table, - a new transaction is created every commit_every rows - - :param table: Name of the target table - :param rows: The rows to insert into the table - :param target_fields: The names of the columns to fill in the table - :param commit_every: The maximum number of rows to insert in one - transaction. Set to 0 to insert all rows in one transaction. - :param replace: Whether to replace instead of insert - """ - i = 0 - with closing(self.get_conn()) as conn: - if self.supports_autocommit: - self.set_autocommit(conn, False) - - conn.commit() - - with closing(conn.cursor()) as cur: - for i, row in enumerate(rows, 1): - lst = [] - for cell in row: - lst.append(self._serialize_cell(cell, conn)) - values = tuple(lst) - sql = self._generate_insert_sql(table, values, target_fields, replace, **kwargs) - self.log.debug("Generated sql: %s", sql) - cur.execute(sql, values) - if commit_every and i % commit_every == 0: - conn.commit() - self.log.info("Loaded %s rows into %s so far", i, table) - - conn.commit() - self.log.info("Done loading. Loaded a total of %s rows", i) - - @staticmethod - def _serialize_cell(cell, conn=None): - """ - Returns the SQL literal of the cell as a string. - - :param cell: The cell to insert into the table - :param conn: The database connection - :return: The serialized cell - :rtype: str - """ - if cell is None: - return None - if isinstance(cell, datetime): - return cell.isoformat() - return str(cell) - - def bulk_dump(self, table, tmp_file): - """ - Dumps a database table into a tab-delimited file - - :param table: The name of the source table - :param tmp_file: The path of the target file - """ - raise NotImplementedError() - - def bulk_load(self, table, tmp_file): - """ - Loads a tab-delimited file into a database table - - :param table: The name of the target table - :param tmp_file: The path of the file to load into the table - """ - raise NotImplementedError() - - def test_connection(self): - """Tests the connection using db-specific query""" - status, message = False, '' - try: - if self.get_first(self._test_connection_sql): - status = True - message = 'Connection successfully tested' - except Exception as e: - status = False - message = str(e) - - return status, message +warnings.warn( + "This module is deprecated. Please use `airflow.providers.common.sql.hooks.sql`.", + DeprecationWarning, + stacklevel=2, +) diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py index 4a441b0f50d59..6445db78814d9 100644 --- a/airflow/hooks/dbapi_hook.py +++ b/airflow/hooks/dbapi_hook.py @@ -19,8 +19,10 @@ import warnings -from airflow.hooks.dbapi import DbApiHook # noqa +from airflow.providers.common.sql.hooks.sql import DbApiHook # noqa warnings.warn( - "This module is deprecated. Please use `airflow.hooks.dbapi`.", DeprecationWarning, stacklevel=2 + "This module is deprecated. Please use `airflow.providers.common.sql.hooks.sql`.", + DeprecationWarning, + stacklevel=2, ) diff --git a/airflow/operators/sql.py b/airflow/operators/sql.py index efa5d0d81a8f0..cb8b664875971 100644 --- a/airflow/operators/sql.py +++ b/airflow/operators/sql.py @@ -20,8 +20,8 @@ from airflow.compat.functools import cached_property from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook from airflow.models import BaseOperator, SkipMixin +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.context import Context diff --git a/airflow/providers/amazon/aws/hooks/redshift_sql.py b/airflow/providers/amazon/aws/hooks/redshift_sql.py index 692b600b38e3a..304e5cb898e83 100644 --- a/airflow/providers/amazon/aws/hooks/redshift_sql.py +++ b/airflow/providers/amazon/aws/hooks/redshift_sql.py @@ -23,7 +23,7 @@ from sqlalchemy.engine.url import URL from airflow.compat.functools import cached_property -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class RedshiftSQLHook(DbApiHook): diff --git a/airflow/providers/amazon/aws/transfers/sql_to_s3.py b/airflow/providers/amazon/aws/transfers/sql_to_s3.py index f399c271416e4..d0a15e1609911 100644 --- a/airflow/providers/amazon/aws/transfers/sql_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/sql_to_s3.py @@ -27,9 +27,9 @@ from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook +from airflow.providers.common.sql.hooks.sql import DbApiHook if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index ee7f246298816..2311e37905646 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -43,6 +43,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - boto3>=1.15.0 # watchtower 3 has been released end Jan and introduced breaking change across the board that might # change logging behaviour: diff --git a/airflow/providers/apache/drill/hooks/drill.py b/airflow/providers/apache/drill/hooks/drill.py index 5baf1f9ecb47b..218b31bf18ab0 100644 --- a/airflow/providers/apache/drill/hooks/drill.py +++ b/airflow/providers/apache/drill/hooks/drill.py @@ -21,7 +21,7 @@ from sqlalchemy import create_engine from sqlalchemy.engine import Connection -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class DrillHook(DbApiHook): diff --git a/airflow/providers/apache/drill/provider.yaml b/airflow/providers/apache/drill/provider.yaml index dcdf128e500e2..127455184ea3e 100644 --- a/airflow/providers/apache/drill/provider.yaml +++ b/airflow/providers/apache/drill/provider.yaml @@ -31,6 +31,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - sqlalchemy-drill>=1.1.0 - sqlparse>=0.4.1 diff --git a/airflow/providers/apache/druid/hooks/druid.py b/airflow/providers/apache/druid/hooks/druid.py index a10519eea476f..2415bc4bc1262 100644 --- a/airflow/providers/apache/druid/hooks/druid.py +++ b/airflow/providers/apache/druid/hooks/druid.py @@ -24,7 +24,7 @@ from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class DruidHook(BaseHook): diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index a1af570a157ba..0139af4292c19 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -38,6 +38,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - pydruid>=0.4.1 integrations: diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py index 559c9727a8ad8..63e4652f204da 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/airflow/providers/apache/hive/hooks/hive.py @@ -32,7 +32,7 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.security import utils from airflow.utils.helpers import as_flattened_list from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index cd9321c4cd754..285393bc82b78 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -40,6 +40,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - hmsclient>=0.1.0 - pandas>=0.17.1 - pyhive[hive]>=0.6.0 diff --git a/airflow/providers/apache/hive/sensors/metastore_partition.py b/airflow/providers/apache/hive/sensors/metastore_partition.py index ea6c1525a1d57..f4295f0d00fa2 100644 --- a/airflow/providers/apache/hive/sensors/metastore_partition.py +++ b/airflow/providers/apache/hive/sensors/metastore_partition.py @@ -17,7 +17,7 @@ # under the License. from typing import TYPE_CHECKING, Any, Sequence -from airflow.sensors.sql import SqlSensor +from airflow.providers.common.sql.sensors.sql import SqlSensor if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/airflow/providers/apache/pinot/hooks/pinot.py b/airflow/providers/apache/pinot/hooks/pinot.py index 4943b37adcb7c..fa31b9f33d18f 100644 --- a/airflow/providers/apache/pinot/hooks/pinot.py +++ b/airflow/providers/apache/pinot/hooks/pinot.py @@ -24,8 +24,8 @@ from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook from airflow.models import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook class PinotAdminHook(BaseHook): diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index 8c2708c2fb767..d2f1f0e8508d8 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -33,6 +33,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql # pinotdb v0.1.1 may still work with older versions of Apache Pinot, but we've confirmed that it # causes a problem with newer versions. - pinotdb>0.1.2 diff --git a/airflow/providers/core/__init__.py b/airflow/providers/common/__init__.py similarity index 100% rename from airflow/providers/core/__init__.py rename to airflow/providers/common/__init__.py diff --git a/airflow/providers/core/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst similarity index 86% rename from airflow/providers/core/sql/CHANGELOG.rst rename to airflow/providers/common/sql/CHANGELOG.rst index a5b4657ed3277..e8cd07d33bc85 100644 --- a/airflow/providers/core/sql/CHANGELOG.rst +++ b/airflow/providers/common/sql/CHANGELOG.rst @@ -23,4 +23,5 @@ Changelog ..... Initial version of the provider. -Adds SQLColumnCheckOperator and SQLTableCheckOperator. +Adds ``SQLColumnCheckOperator`` and ``SQLTableCheckOperator``. +Moves ``DBApiHook``, ``SQLSensor`` and ``ConnectorProtocol`` to the provider. diff --git a/airflow/providers/core/sql/__init__.py b/airflow/providers/common/sql/__init__.py similarity index 100% rename from airflow/providers/core/sql/__init__.py rename to airflow/providers/common/sql/__init__.py diff --git a/airflow/providers/core/sql/example_dags/__init__.py b/airflow/providers/common/sql/hooks/__init__.py similarity index 100% rename from airflow/providers/core/sql/example_dags/__init__.py rename to airflow/providers/common/sql/hooks/__init__.py diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py new file mode 100644 index 0000000000000..efd4a9dcfed3b --- /dev/null +++ b/airflow/providers/common/sql/hooks/sql.py @@ -0,0 +1,402 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import warnings +from contextlib import closing +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import create_engine +from typing_extensions import Protocol + +from airflow import AirflowException +from airflow.hooks.base import BaseHook +from airflow.providers_manager import ProvidersManager +from airflow.utils.module_loading import import_string + + +def _backported_get_hook(connection, *, hook_params=None): + """Return hook based on conn_type + For supporting Airflow versions < 2.3, we backport "get_hook()" method. This should be removed + when "apache-airflow-providers-slack" will depend on Airflow >= 2.3. + """ + hook = ProvidersManager().hooks.get(connection.conn_type, None) + + if hook is None: + raise AirflowException(f'Unknown hook type "{connection.conn_type}"') + try: + hook_class = import_string(hook.hook_class_name) + except ImportError: + warnings.warn( + f"Could not import {hook.hook_class_name} when discovering {hook.hook_name} {hook.package_name}", + ) + raise + if hook_params is None: + hook_params = {} + return hook_class(**{hook.connection_id_attribute_name: connection.conn_id}, **hook_params) + + +class ConnectorProtocol(Protocol): + """A protocol where you can connect to a database.""" + + def connect(self, host: str, port: int, username: str, schema: str) -> Any: + """ + Connect to a database. + + :param host: The database host to connect to. + :param port: The database port to connect to. + :param username: The database username used for the authentication. + :param schema: The database schema to connect to. + :return: the authorized connection object. + """ + + +class DbApiHook(BaseHook): + """ + Abstract base class for sql hooks. + + :param schema: Optional DB schema that overrides the schema specified in the connection. Make sure that + if you change the schema parameter value in the constructor of the derived Hook, such change + should be done before calling the ``DBApiHook.__init__()``. + :param log_sql: Whether to log SQL query when it's executed. Defaults to *True*. + """ + + # Override to provide the connection name. + conn_name_attr = None # type: str + # Override to have a default connection id for a particular dbHook + default_conn_name = 'default_conn_id' + # Override if this db supports autocommit. + supports_autocommit = False + # Override with the object that exposes the connect method + connector = None # type: Optional[ConnectorProtocol] + # Override with db-specific query to check connection + _test_connection_sql = "select 1" + + def __init__(self, *args, schema: Optional[str] = None, log_sql: bool = True, **kwargs): + super().__init__() + if not self.conn_name_attr: + raise AirflowException("conn_name_attr is not defined") + elif len(args) == 1: + setattr(self, self.conn_name_attr, args[0]) + elif self.conn_name_attr not in kwargs: + setattr(self, self.conn_name_attr, self.default_conn_name) + else: + setattr(self, self.conn_name_attr, kwargs[self.conn_name_attr]) + # We should not make schema available in deriving hooks for backwards compatibility + # If a hook deriving from DBApiHook has a need to access schema, then it should retrieve it + # from kwargs and store it on its own. We do not run "pop" here as we want to give the + # Hook deriving from the DBApiHook to still have access to the field in it's constructor + self.__schema = schema + self.log_sql = log_sql + + def get_conn(self): + """Returns a connection object""" + db = self.get_connection(getattr(self, self.conn_name_attr)) + return self.connector.connect(host=db.host, port=db.port, username=db.login, schema=db.schema) + + def get_uri(self) -> str: + """ + Extract the URI from the connection. + + :return: the extracted uri. + """ + conn = self.get_connection(getattr(self, self.conn_name_attr)) + conn.schema = self.__schema or conn.schema + return conn.get_uri() + + def get_sqlalchemy_engine(self, engine_kwargs=None): + """ + Get an sqlalchemy_engine object. + + :param engine_kwargs: Kwargs used in :func:`~sqlalchemy.create_engine`. + :return: the created engine. + """ + if engine_kwargs is None: + engine_kwargs = {} + return create_engine(self.get_uri(), **engine_kwargs) + + def get_pandas_df(self, sql, parameters=None, **kwargs): + """ + Executes the sql and returns a pandas dataframe + + :param sql: the sql statement to be executed (str) or a list of + sql statements to execute + :param parameters: The parameters to render the SQL query with. + :param kwargs: (optional) passed into pandas.io.sql.read_sql method + """ + try: + from pandas.io import sql as psql + except ImportError: + raise Exception( + "pandas library not installed, run: pip install " + "'apache-airflow-providers-common-sql[pandas]'." + ) + + with closing(self.get_conn()) as conn: + return psql.read_sql(sql, con=conn, params=parameters, **kwargs) + + def get_pandas_df_by_chunks(self, sql, parameters=None, *, chunksize, **kwargs): + """ + Executes the sql and returns a generator + + :param sql: the sql statement to be executed (str) or a list of + sql statements to execute + :param parameters: The parameters to render the SQL query with + :param chunksize: number of rows to include in each chunk + :param kwargs: (optional) passed into pandas.io.sql.read_sql method + """ + try: + from pandas.io import sql as psql + except ImportError: + raise Exception( + "pandas library not installed, run: pip install " + "'apache-airflow-providers-common-sql[pandas]'." + ) + + with closing(self.get_conn()) as conn: + yield from psql.read_sql(sql, con=conn, params=parameters, chunksize=chunksize, **kwargs) + + def get_records(self, sql, parameters=None): + """ + Executes the sql and returns a set of records. + + :param sql: the sql statement to be executed (str) or a list of + sql statements to execute + :param parameters: The parameters to render the SQL query with. + """ + with closing(self.get_conn()) as conn: + with closing(conn.cursor()) as cur: + if parameters is not None: + cur.execute(sql, parameters) + else: + cur.execute(sql) + return cur.fetchall() + + def get_first(self, sql, parameters=None): + """ + Executes the sql and returns the first resulting row. + + :param sql: the sql statement to be executed (str) or a list of + sql statements to execute + :param parameters: The parameters to render the SQL query with. + """ + with closing(self.get_conn()) as conn: + with closing(conn.cursor()) as cur: + if parameters is not None: + cur.execute(sql, parameters) + else: + cur.execute(sql) + return cur.fetchone() + + def run(self, sql, autocommit=False, parameters=None, handler=None): + """ + Runs a command or a list of commands. Pass a list of sql + statements to the sql parameter to get them to execute + sequentially + + :param sql: the sql statement to be executed (str) or a list of + sql statements to execute + :param autocommit: What to set the connection's autocommit setting to + before executing the query. + :param parameters: The parameters to render the SQL query with. + :param handler: The result handler which is called with the result of each statement. + :return: query results if handler was provided. + """ + scalar = isinstance(sql, str) + if scalar: + sql = [sql] + + if sql: + self.log.debug("Executing %d statements", len(sql)) + else: + raise ValueError("List of SQL statements is empty") + + with closing(self.get_conn()) as conn: + if self.supports_autocommit: + self.set_autocommit(conn, autocommit) + + with closing(conn.cursor()) as cur: + results = [] + for sql_statement in sql: + self._run_command(cur, sql_statement, parameters) + if handler is not None: + result = handler(cur) + results.append(result) + + # If autocommit was set to False for db that supports autocommit, + # or if db does not supports autocommit, we do a manual commit. + if not self.get_autocommit(conn): + conn.commit() + + if handler is None: + return None + + if scalar: + return results[0] + + return results + + def _run_command(self, cur, sql_statement, parameters): + """Runs a statement using an already open cursor.""" + if self.log_sql: + self.log.info("Running statement: %s, parameters: %s", sql_statement, parameters) + + if parameters: + cur.execute(sql_statement, parameters) + else: + cur.execute(sql_statement) + + # According to PEP 249, this is -1 when query result is not applicable. + if cur.rowcount >= 0: + self.log.info("Rows affected: %s", cur.rowcount) + + def set_autocommit(self, conn, autocommit): + """Sets the autocommit flag on the connection""" + if not self.supports_autocommit and autocommit: + self.log.warning( + "%s connection doesn't support autocommit but autocommit activated.", + getattr(self, self.conn_name_attr), + ) + conn.autocommit = autocommit + + def get_autocommit(self, conn): + """ + Get autocommit setting for the provided connection. + Return True if conn.autocommit is set to True. + Return False if conn.autocommit is not set or set to False or conn + does not support autocommit. + + :param conn: Connection to get autocommit setting from. + :return: connection autocommit setting. + :rtype: bool + """ + return getattr(conn, 'autocommit', False) and self.supports_autocommit + + def get_cursor(self): + """Returns a cursor""" + return self.get_conn().cursor() + + @staticmethod + def _generate_insert_sql(table, values, target_fields, replace, **kwargs): + """ + Static helper method that generates the INSERT SQL statement. + The REPLACE variant is specific to MySQL syntax. + + :param table: Name of the target table + :param values: The row to insert into the table + :param target_fields: The names of the columns to fill in the table + :param replace: Whether to replace instead of insert + :return: The generated INSERT or REPLACE SQL statement + :rtype: str + """ + placeholders = [ + "%s", + ] * len(values) + + if target_fields: + target_fields = ", ".join(target_fields) + target_fields = f"({target_fields})" + else: + target_fields = '' + + if not replace: + sql = "INSERT INTO " + else: + sql = "REPLACE INTO " + sql += f"{table} {target_fields} VALUES ({','.join(placeholders)})" + return sql + + def insert_rows(self, table, rows, target_fields=None, commit_every=1000, replace=False, **kwargs): + """ + A generic way to insert a set of tuples into a table, + a new transaction is created every commit_every rows + + :param table: Name of the target table + :param rows: The rows to insert into the table + :param target_fields: The names of the columns to fill in the table + :param commit_every: The maximum number of rows to insert in one + transaction. Set to 0 to insert all rows in one transaction. + :param replace: Whether to replace instead of insert + """ + i = 0 + with closing(self.get_conn()) as conn: + if self.supports_autocommit: + self.set_autocommit(conn, False) + + conn.commit() + + with closing(conn.cursor()) as cur: + for i, row in enumerate(rows, 1): + lst = [] + for cell in row: + lst.append(self._serialize_cell(cell, conn)) + values = tuple(lst) + sql = self._generate_insert_sql(table, values, target_fields, replace, **kwargs) + self.log.debug("Generated sql: %s", sql) + cur.execute(sql, values) + if commit_every and i % commit_every == 0: + conn.commit() + self.log.info("Loaded %s rows into %s so far", i, table) + + conn.commit() + self.log.info("Done loading. Loaded a total of %s rows", i) + + @staticmethod + def _serialize_cell(cell, conn=None): + """ + Returns the SQL literal of the cell as a string. + + :param cell: The cell to insert into the table + :param conn: The database connection + :return: The serialized cell + :rtype: str + """ + if cell is None: + return None + if isinstance(cell, datetime): + return cell.isoformat() + return str(cell) + + def bulk_dump(self, table, tmp_file): + """ + Dumps a database table into a tab-delimited file + + :param table: The name of the source table + :param tmp_file: The path of the target file + """ + raise NotImplementedError() + + def bulk_load(self, table, tmp_file): + """ + Loads a tab-delimited file into a database table + + :param table: The name of the target table + :param tmp_file: The path of the file to load into the table + """ + raise NotImplementedError() + + def test_connection(self): + """Tests the connection using db-specific query""" + status, message = False, '' + try: + if self.get_first(self._test_connection_sql): + status = True + message = 'Connection successfully tested' + except Exception as e: + status = False + message = str(e) + + return status, message diff --git a/airflow/providers/core/sql/operators/__init__.py b/airflow/providers/common/sql/operators/__init__.py similarity index 100% rename from airflow/providers/core/sql/operators/__init__.py rename to airflow/providers/common/sql/operators/__init__.py diff --git a/airflow/providers/core/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py similarity index 99% rename from airflow/providers/core/sql/operators/sql.py rename to airflow/providers/common/sql/operators/sql.py index 1d9945f403ec3..63b6457c759f2 100644 --- a/airflow/providers/core/sql/operators/sql.py +++ b/airflow/providers/common/sql/operators/sql.py @@ -121,7 +121,7 @@ def execute(self, context=None): if not records: raise AirflowException(f"The following query returned zero rows: {self.sql}") - self.log.info(f"Record: {records}") + self.log.info("Record: %s", records) for idx, result in enumerate(records): tolerance = self.column_mapping[column][checks[idx]].get("tolerance") @@ -298,7 +298,7 @@ def execute(self, context=None): if not records: raise AirflowException(f"The following query returned zero rows: {self.sql}") - self.log.info(f"Record: {records}") + self.log.info("Record: %s", records) for check in self.checks.keys(): for result in records: diff --git a/airflow/providers/core/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml similarity index 61% rename from airflow/providers/core/sql/provider.yaml rename to airflow/providers/common/sql/provider.yaml index f9519842679de..a277f327ccac5 100644 --- a/airflow/providers/core/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -16,26 +16,40 @@ # under the License. --- -package-name: apache-airflow-providers-core-sql -name: Core SQL +package-name: apache-airflow-providers-common-sql +name: Common SQL description: | - `Core SQL Provider `__ + `Common SQL Provider `__ versions: - 1.0.0 -dependencies: - - apache-airflow>=2.2.0 +dependencies: [] + +additional-extras: + - name: pandas + dependencies: + - pandas>=0.17.1 integrations: - - integration-name: Core SQL + - integration-name: Common SQL external-doc-url: https://en.wikipedia.org/wiki/SQL how-to-guide: - - /docs/apache-airflow-providers-core-sql/operators.rst + - /docs/apache-airflow-providers-common-sql/operators.rst logo: /integration-logos/core/sql/sql.png tags: [software] operators: - - integration-name: Core SQL + - integration-name: Common SQL + python-modules: + - airflow.providers.common.sql.operators.sql + +hooks: + - integration-name: Common SQL + python-modules: + - airflow.providers.common.sql.hooks.sql + +sensors: + - integration-name: Common SQL python-modules: - - airflow.providers.core.sql.operators.sql + - airflow.providers.common.sql.sensors.sql diff --git a/tests/providers/core/__init__.py b/airflow/providers/common/sql/sensors/__init__.py similarity index 100% rename from tests/providers/core/__init__.py rename to airflow/providers/common/sql/sensors/__init__.py diff --git a/airflow/providers/common/sql/sensors/sql.py b/airflow/providers/common/sql/sensors/sql.py new file mode 100644 index 0000000000000..c9f66568e6c4b --- /dev/null +++ b/airflow/providers/common/sql/sensors/sql.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Any, Sequence + +from packaging.version import Version + +from airflow import AirflowException +from airflow.hooks.base import BaseHook +from airflow.providers.common.sql.hooks.sql import DbApiHook, _backported_get_hook +from airflow.sensors.base import BaseSensorOperator +from airflow.version import version + + +class SqlSensor(BaseSensorOperator): + """ + Runs a sql statement repeatedly until a criteria is met. It will keep trying until + success or failure criteria are met, or if the first cell is not in (0, '0', '', None). + Optional success and failure callables are called with the first cell returned as the argument. + If success callable is defined the sensor will keep retrying until the criteria is met. + If failure callable is defined and the criteria is met the sensor will raise AirflowException. + Failure criteria is evaluated before success criteria. A fail_on_empty boolean can also + be passed to the sensor in which case it will fail if no rows have been returned + + :param conn_id: The connection to run the sensor against + :param sql: The sql to run. To pass, it needs to return at least one cell + that contains a non-zero / empty string value. + :param parameters: The parameters to render the SQL query with (optional). + :param success: Success criteria for the sensor is a Callable that takes first_cell + as the only argument, and returns a boolean (optional). + :param failure: Failure criteria for the sensor is a Callable that takes first_cell + as the only argument and return a boolean (optional). + :param fail_on_empty: Explicitly fail on no rows returned. + :param hook_params: Extra config params to be passed to the underlying hook. + Should match the desired hook constructor params. + """ + + template_fields: Sequence[str] = ('sql',) + template_ext: Sequence[str] = ( + '.hql', + '.sql', + ) + ui_color = '#7c7287' + + def __init__( + self, + *, + conn_id, + sql, + parameters=None, + success=None, + failure=None, + fail_on_empty=False, + hook_params=None, + **kwargs, + ): + self.conn_id = conn_id + self.sql = sql + self.parameters = parameters + self.success = success + self.failure = failure + self.fail_on_empty = fail_on_empty + self.hook_params = hook_params + super().__init__(**kwargs) + + def _get_hook(self): + conn = BaseHook.get_connection(self.conn_id) + if Version(version) >= Version('2.3'): + # "hook_params" were introduced to into "get_hook()" only in Airflow 2.3. + hook = conn.get_hook(hook_params=self.hook_params) # ignore airflow compat check + else: + # For supporting Airflow versions < 2.3, we backport "get_hook()" method. This should be removed + # when "apache-airflow-providers-common-sql" will depend on Airflow >= 2.3. + hook = _backported_get_hook(conn, hook_params=self.hook_params) + if not isinstance(hook, DbApiHook): + raise AirflowException( + f'The connection type is not supported by {self.__class__.__name__}. ' + f'The associated hook should be a subclass of `DbApiHook`. Got {hook.__class__.__name__}' + ) + return hook + + def poke(self, context: Any): + hook = self._get_hook() + + self.log.info('Poking: %s (with parameters %s)', self.sql, self.parameters) + records = hook.get_records(self.sql, self.parameters) + if not records: + if self.fail_on_empty: + raise AirflowException("No rows returned, raising as per fail_on_empty flag") + else: + return False + first_cell = records[0][0] + if self.failure is not None: + if callable(self.failure): + if self.failure(first_cell): + raise AirflowException(f"Failure criteria met. self.failure({first_cell}) returned True") + else: + raise AirflowException(f"self.failure is present, but not callable -> {self.failure}") + if self.success is not None: + if callable(self.success): + return self.success(first_cell) + else: + raise AirflowException(f"self.success is present, but not callable -> {self.success}") + return bool(first_cell) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 9d86b4dbe3ae6..6c5800170d255 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -25,7 +25,7 @@ from airflow import __version__ from airflow.exceptions import AirflowException -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook LIST_SQL_ENDPOINTS_ENDPOINT = ('GET', 'api/2.0/sql/endpoints') diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index d20878bef58e9..b0d4d44bc4f8b 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -38,6 +38,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - requests>=2.27,<3 - databricks-sql-connector>=2.0.0, <3.0.0 - aiohttp>=3.6.3, <4 diff --git a/airflow/providers/elasticsearch/hooks/elasticsearch.py b/airflow/providers/elasticsearch/hooks/elasticsearch.py index b48511670ffe5..493ca7f082d65 100644 --- a/airflow/providers/elasticsearch/hooks/elasticsearch.py +++ b/airflow/providers/elasticsearch/hooks/elasticsearch.py @@ -20,8 +20,8 @@ from es.elastic.api import Connection as ESConnection, connect -from airflow.hooks.dbapi import DbApiHook from airflow.models.connection import Connection as AirflowConnection +from airflow.providers.common.sql.hooks.sql import DbApiHook class ElasticsearchHook(DbApiHook): diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index e9f4a2ddcafce..25ef669963d52 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -40,6 +40,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - elasticsearch>7 - elasticsearch-dbapi - elasticsearch-dsl>=5.0.0 diff --git a/airflow/providers/exasol/hooks/exasol.py b/airflow/providers/exasol/hooks/exasol.py index 2233ce1e2c347..784c57cde0661 100644 --- a/airflow/providers/exasol/hooks/exasol.py +++ b/airflow/providers/exasol/hooks/exasol.py @@ -23,7 +23,7 @@ import pyexasol from pyexasol import ExaConnection -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class ExasolHook(DbApiHook): diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index b0594fb6532cd..da8479d95911c 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -35,6 +35,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - pyexasol>=0.5.1 - pandas>=0.17.1 diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index 1be077d790046..0049143aea461 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -52,7 +52,7 @@ from sqlalchemy import create_engine from airflow.exceptions import AirflowException -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.providers.google.common.consts import CLIENT_INFO from airflow.providers.google.common.hooks.base_google import GoogleBaseHook from airflow.utils.helpers import convert_camel_to_snake diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index a4962fb714bbe..5d6c0b739e880 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -52,6 +52,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql # Google has very clear rules on what dependencies should be used. All the limits below # follow strict guidelines of Google Libraries as quoted here: # While this issue is open, dependents of google-api-core, google-cloud-core. and google-auth diff --git a/airflow/providers/jdbc/hooks/jdbc.py b/airflow/providers/jdbc/hooks/jdbc.py index 734afecb5ba18..a33a2b31ee279 100644 --- a/airflow/providers/jdbc/hooks/jdbc.py +++ b/airflow/providers/jdbc/hooks/jdbc.py @@ -20,8 +20,8 @@ import jaydebeapi -from airflow.hooks.dbapi import DbApiHook from airflow.models.connection import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook class JdbcHook(DbApiHook): diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index f12f371ab887e..958a7e65e5e2d 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -34,6 +34,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - jaydebeapi>=1.1.1 integrations: diff --git a/airflow/providers/microsoft/mssql/hooks/mssql.py b/airflow/providers/microsoft/mssql/hooks/mssql.py index 75241f1f296d3..ba236efbc25b9 100644 --- a/airflow/providers/microsoft/mssql/hooks/mssql.py +++ b/airflow/providers/microsoft/mssql/hooks/mssql.py @@ -20,7 +20,7 @@ import pymssql -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class MsSqlHook(DbApiHook): diff --git a/airflow/providers/microsoft/mssql/operators/mssql.py b/airflow/providers/microsoft/mssql/operators/mssql.py index 7082c7c52d90b..5a5738eb6878c 100644 --- a/airflow/providers/microsoft/mssql/operators/mssql.py +++ b/airflow/providers/microsoft/mssql/operators/mssql.py @@ -23,7 +23,7 @@ from airflow.www import utils as wwwutils if TYPE_CHECKING: - from airflow.hooks.dbapi import DbApiHook + from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.context import Context diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index b2feb654fdaef..9f3a35daaf175 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -35,6 +35,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - pymssql>=2.1.5; platform_machine != "aarch64" integrations: diff --git a/airflow/providers/mysql/hooks/mysql.py b/airflow/providers/mysql/hooks/mysql.py index 9f0cb58cd4d54..586cfb5b8c718 100644 --- a/airflow/providers/mysql/hooks/mysql.py +++ b/airflow/providers/mysql/hooks/mysql.py @@ -20,8 +20,8 @@ import json from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union -from airflow.hooks.dbapi import DbApiHook from airflow.models import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook if TYPE_CHECKING: from mysql.connector.abstracts import MySQLConnectionAbstract diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index 21fa45f649b0b..7947686519a37 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -37,6 +37,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - mysql-connector-python>=8.0.11; platform_machine != "aarch64" - mysqlclient>=1.3.6; platform_machine != "aarch64" diff --git a/airflow/providers/odbc/hooks/odbc.py b/airflow/providers/odbc/hooks/odbc.py index 9ce32fa337118..d5e288bb3d7fe 100644 --- a/airflow/providers/odbc/hooks/odbc.py +++ b/airflow/providers/odbc/hooks/odbc.py @@ -20,7 +20,7 @@ import pyodbc -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.helpers import merge_dicts @@ -178,7 +178,10 @@ def get_conn(self) -> pyodbc.Connection: return conn def get_uri(self) -> str: - """URI invoked in :py:meth:`~airflow.hooks.dbapi.DbApiHook.get_sqlalchemy_engine` method""" + """ + URI invoked in :py:meth:`~airflow.providers.common.sql.hooks.sql.DbApiHook.get_sqlalchemy_engine` + method. + """ quoted_conn_str = quote_plus(self.odbc_connection_string) uri = f"{self.sqlalchemy_scheme}:///?odbc_connect={quoted_conn_str}" return uri diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 12dc46f5eb60b..18e980038bd72 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -33,6 +33,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - pyodbc integrations: diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index eb94b435e8970..65db1ee7abb75 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -28,7 +28,7 @@ except ImportError: numpy = None # type: ignore -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook PARAM_TYPES = {bool, float, int, str} diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index 513cc34fc07eb..135e04c43e768 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -37,6 +37,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - oracledb>=1.0.0 integrations: diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py index 884a1c92927e9..09c07c9b8f7bd 100644 --- a/airflow/providers/postgres/hooks/postgres.py +++ b/airflow/providers/postgres/hooks/postgres.py @@ -26,8 +26,8 @@ from psycopg2.extensions import connection from psycopg2.extras import DictCursor, NamedTupleCursor, RealDictCursor -from airflow.hooks.dbapi import DbApiHook from airflow.models.connection import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook CursorType = Union[DictCursor, RealDictCursor, NamedTupleCursor] diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 9c4c70ad3e9bc..8f81558f470cd 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -38,6 +38,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - psycopg2-binary>=2.7.4 integrations: diff --git a/airflow/providers/presto/hooks/presto.py b/airflow/providers/presto/hooks/presto.py index 95ecf86b52dda..22afc71577341 100644 --- a/airflow/providers/presto/hooks/presto.py +++ b/airflow/providers/presto/hooks/presto.py @@ -26,8 +26,8 @@ from airflow import AirflowException from airflow.configuration import conf -from airflow.hooks.dbapi import DbApiHook from airflow.models import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING try: diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index 26d7db6f8a4a7..dd665140ee0f4 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -36,6 +36,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - presto-python-client>=0.8.2 - pandas>=0.17.1 diff --git a/airflow/providers/qubole/hooks/qubole_check.py b/airflow/providers/qubole/hooks/qubole_check.py index 5dba31c5cbe97..59ae4396a3dd3 100644 --- a/airflow/providers/qubole/hooks/qubole_check.py +++ b/airflow/providers/qubole/hooks/qubole_check.py @@ -23,7 +23,7 @@ from qds_sdk.commands import Command from airflow.exceptions import AirflowException -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.providers.qubole.hooks.qubole import QuboleHook log = logging.getLogger(__name__) diff --git a/airflow/providers/qubole/provider.yaml b/airflow/providers/qubole/provider.yaml index 1826d2299cd23..a35168931345d 100644 --- a/airflow/providers/qubole/provider.yaml +++ b/airflow/providers/qubole/provider.yaml @@ -35,6 +35,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - qds-sdk>=1.10.4 integrations: diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index 2a8331d1acbfe..57ff668b08f7e 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -36,6 +36,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - apache-airflow-providers-http - slack_sdk>=3.0.0 diff --git a/airflow/providers/slack/transfers/sql_to_slack.py b/airflow/providers/slack/transfers/sql_to_slack.py index 8b9ff04af3d5a..58c723feca2a4 100644 --- a/airflow/providers/slack/transfers/sql_to_slack.py +++ b/airflow/providers/slack/transfers/sql_to_slack.py @@ -14,49 +14,23 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import warnings from typing import TYPE_CHECKING, Iterable, Mapping, Optional, Sequence, Union +from packaging.version import Version from pandas import DataFrame from tabulate import tabulate from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook from airflow.models import BaseOperator +from airflow.providers.common.sql.hooks.sql import DbApiHook, _backported_get_hook from airflow.providers.slack.hooks.slack_webhook import SlackWebhookHook -from airflow.providers_manager import ProvidersManager -from airflow.utils.module_loading import import_string from airflow.version import version if TYPE_CHECKING: from airflow.utils.context import Context -def _backported_get_hook(connection, *, hook_params=None): - """Return hook based on conn_type - For supporting Airflow versions < 2.3, we backport "get_hook()" method. This should be removed - when "apache-airflow-providers-slack" will depend on Airflow >= 2.3. - """ - hook = ProvidersManager().hooks.get(connection.conn_type, None) - - if hook is None: - raise AirflowException(f'Unknown hook type "{connection.conn_type}"') - try: - hook_class = import_string(hook.hook_class_name) - except ImportError: - warnings.warn( - "Could not import %s when discovering %s %s", - hook.hook_class_name, - hook.hook_name, - hook.package_name, - ) - raise - if hook_params is None: - hook_params = {} - return hook_class(**{hook.connection_id_attribute_name: connection.conn_id}, **hook_params) - - class SqlToSlackOperator(BaseOperator): """ Executes an SQL statement in a given SQL connection and sends the results to Slack. The results of the @@ -126,7 +100,7 @@ def __init__( def _get_hook(self) -> DbApiHook: self.log.debug("Get connection for %s", self.sql_conn_id) conn = BaseHook.get_connection(self.sql_conn_id) - if version >= '2.3': + if Version(version) >= Version('2.3'): # "hook_params" were introduced to into "get_hook()" only in Airflow 2.3. hook = conn.get_hook(hook_params=self.sql_hook_params) # ignore airflow compat check else: diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/airflow/providers/snowflake/hooks/snowflake.py index 29a4b63156c6d..3dee0989ed210 100644 --- a/airflow/providers/snowflake/hooks/snowflake.py +++ b/airflow/providers/snowflake/hooks/snowflake.py @@ -30,7 +30,7 @@ from sqlalchemy import create_engine from airflow import AirflowException -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.strings import to_boolean diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index c1c035bde4eaa..f097795685507 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -43,6 +43,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - snowflake-connector-python>=2.4.1 - snowflake-sqlalchemy>=1.1.0 diff --git a/airflow/providers/sqlite/hooks/sqlite.py b/airflow/providers/sqlite/hooks/sqlite.py index 6fe055c149d6f..8559b3040150e 100644 --- a/airflow/providers/sqlite/hooks/sqlite.py +++ b/airflow/providers/sqlite/hooks/sqlite.py @@ -18,7 +18,7 @@ import sqlite3 -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class SqliteHook(DbApiHook): diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index d663a06e3c44c..d69853378d351 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -33,7 +33,8 @@ versions: - 1.0.1 - 1.0.0 -dependencies: [] +dependencies: + - apache-airflow-providers-common-sql integrations: - integration-name: SQLite diff --git a/airflow/providers/trino/hooks/trino.py b/airflow/providers/trino/hooks/trino.py index cd8fa78c67e23..9170e19a547ae 100644 --- a/airflow/providers/trino/hooks/trino.py +++ b/airflow/providers/trino/hooks/trino.py @@ -29,8 +29,8 @@ from airflow import AirflowException from airflow.configuration import conf -from airflow.hooks.dbapi import DbApiHook from airflow.models import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING try: diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index f701e6c899049..3195e764b0db2 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -35,6 +35,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - pandas>=0.17.1 - trino>=0.301.0 diff --git a/airflow/providers/vertica/hooks/vertica.py b/airflow/providers/vertica/hooks/vertica.py index 9530c5965c5a8..5a6e1c125c229 100644 --- a/airflow/providers/vertica/hooks/vertica.py +++ b/airflow/providers/vertica/hooks/vertica.py @@ -19,7 +19,7 @@ from vertica_python import connect -from airflow.hooks.dbapi import DbApiHook +from airflow.providers.common.sql.hooks.sql import DbApiHook class VerticaHook(DbApiHook): diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index 0ebe1989acae0..62a8921780bf7 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -34,6 +34,7 @@ versions: dependencies: - apache-airflow>=2.2.0 + - apache-airflow-providers-common-sql - vertica-python>=0.5.1 integrations: diff --git a/airflow/sensors/sql.py b/airflow/sensors/sql.py index a35d7566ceb41..c52fe691434e1 100644 --- a/airflow/sensors/sql.py +++ b/airflow/sensors/sql.py @@ -15,97 +15,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import warnings -from typing import Sequence +from airflow.providers.common.sql.sensors.sql import SqlSensor # noqa -from airflow.exceptions import AirflowException -from airflow.hooks.base import BaseHook -from airflow.hooks.dbapi import DbApiHook -from airflow.sensors.base import BaseSensorOperator -from airflow.utils.context import Context - - -class SqlSensor(BaseSensorOperator): - """ - Runs a sql statement repeatedly until a criteria is met. It will keep trying until - success or failure criteria are met, or if the first cell is not in (0, '0', '', None). - Optional success and failure callables are called with the first cell returned as the argument. - If success callable is defined the sensor will keep retrying until the criteria is met. - If failure callable is defined and the criteria is met the sensor will raise AirflowException. - Failure criteria is evaluated before success criteria. A fail_on_empty boolean can also - be passed to the sensor in which case it will fail if no rows have been returned - - :param conn_id: The connection to run the sensor against - :param sql: The sql to run. To pass, it needs to return at least one cell - that contains a non-zero / empty string value. - :param parameters: The parameters to render the SQL query with (optional). - :param success: Success criteria for the sensor is a Callable that takes first_cell - as the only argument, and returns a boolean (optional). - :param failure: Failure criteria for the sensor is a Callable that takes first_cell - as the only argument and return a boolean (optional). - :param fail_on_empty: Explicitly fail on no rows returned. - :param hook_params: Extra config params to be passed to the underlying hook. - Should match the desired hook constructor params. - """ - - template_fields: Sequence[str] = ('sql',) - template_ext: Sequence[str] = ( - '.hql', - '.sql', - ) - ui_color = '#7c7287' - - def __init__( - self, - *, - conn_id, - sql, - parameters=None, - success=None, - failure=None, - fail_on_empty=False, - hook_params=None, - **kwargs, - ): - self.conn_id = conn_id - self.sql = sql - self.parameters = parameters - self.success = success - self.failure = failure - self.fail_on_empty = fail_on_empty - self.hook_params = hook_params - super().__init__(**kwargs) - - def _get_hook(self): - conn = BaseHook.get_connection(self.conn_id) - hook = conn.get_hook(hook_params=self.hook_params) - if not isinstance(hook, DbApiHook): - raise AirflowException( - f'The connection type is not supported by {self.__class__.__name__}. ' - f'The associated hook should be a subclass of `DbApiHook`. Got {hook.__class__.__name__}' - ) - return hook - - def poke(self, context: Context): - hook = self._get_hook() - - self.log.info('Poking: %s (with parameters %s)', self.sql, self.parameters) - records = hook.get_records(self.sql, self.parameters) - if not records: - if self.fail_on_empty: - raise AirflowException("No rows returned, raising as per fail_on_empty flag") - else: - return False - first_cell = records[0][0] - if self.failure is not None: - if callable(self.failure): - if self.failure(first_cell): - raise AirflowException(f"Failure criteria met. self.failure({first_cell}) returned True") - else: - raise AirflowException(f"self.failure is present, but not callable -> {self.failure}") - if self.success is not None: - if callable(self.success): - return self.success(first_cell) - else: - raise AirflowException(f"self.success is present, but not callable -> {self.success}") - return bool(first_cell) +warnings.warn( + "This module is deprecated. Please use `airflow.providers.common.sql.sensors.sql`.", + DeprecationWarning, + stacklevel=2, +) diff --git a/airflow/sensors/sql_sensor.py b/airflow/sensors/sql_sensor.py index 8a077db534496..fafc5335a2353 100644 --- a/airflow/sensors/sql_sensor.py +++ b/airflow/sensors/sql_sensor.py @@ -19,8 +19,10 @@ import warnings -from airflow.sensors.sql import SqlSensor # noqa +from airflow.providers.common.sql.sensors.sql import SqlSensor # noqa warnings.warn( - "This module is deprecated. Please use `airflow.sensors.sql`.", DeprecationWarning, stacklevel=2 + "This module is deprecated. Please use `airflow.providers.common.sql.sensors.sql`.", + DeprecationWarning, + stacklevel=2, ) diff --git a/docs/apache-airflow-providers-core-sql/commits.rst b/docs/apache-airflow-providers-common-sql/commits.rst similarity index 84% rename from docs/apache-airflow-providers-core-sql/commits.rst rename to docs/apache-airflow-providers-common-sql/commits.rst index 8292a11950bf7..b25291f136b4e 100644 --- a/docs/apache-airflow-providers-core-sql/commits.rst +++ b/docs/apache-airflow-providers-common-sql/commits.rst @@ -15,11 +15,11 @@ specific language governing permissions and limitations under the License. -Package apache-airflow-providers-core-sql ------------------------------------------- +Package apache-airflow-providers-common-sql +------------------------------------------- -`Core SQL Provider `__ +`Common SQL Provider `__ -This is detailed commit list of changes for versions provider package: ``core.sql``. +This is detailed commit list of changes for versions provider package: ``common.sql``. For high-level changelog, see :doc:`package information including changelog `. diff --git a/docs/apache-airflow-providers-core-sql/connections.rst b/docs/apache-airflow-providers-common-sql/connections.rst similarity index 100% rename from docs/apache-airflow-providers-core-sql/connections.rst rename to docs/apache-airflow-providers-common-sql/connections.rst diff --git a/docs/apache-airflow-providers-core-sql/index.rst b/docs/apache-airflow-providers-common-sql/index.rst similarity index 74% rename from docs/apache-airflow-providers-core-sql/index.rst rename to docs/apache-airflow-providers-common-sql/index.rst index abc6186b68557..c1f8ddbc9f758 100644 --- a/docs/apache-airflow-providers-core-sql/index.rst +++ b/docs/apache-airflow-providers-common-sql/index.rst @@ -15,8 +15,8 @@ specific language governing permissions and limitations under the License. -``apache-airflow-providers-core-sql`` -============================================ +``apache-airflow-providers-common-sql`` +======================================= Content ------- @@ -32,14 +32,20 @@ Content :maxdepth: 1 :caption: References - Python API <_api/airflow/providers/core/sql/index> + Python API <_api/airflow/providers/common/sql/index> + +.. toctree:: + :hidden: + :caption: System tests + + System Tests <_api/tests/system/providers/common/sql/index> .. toctree:: :maxdepth: 1 :caption: Resources - Example DAGs - PyPI Repository + Example DAGs + PyPI Repository Installing from sources .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! @@ -50,5 +56,5 @@ Content Detailed list of commits -Package apache-airflow-providers-core-sql ------------------------------------------------------- +Package apache-airflow-providers-common-sql +------------------------------------------- diff --git a/docs/apache-airflow-providers-core-sql/installing-providers-from-sources.rst b/docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst similarity index 100% rename from docs/apache-airflow-providers-core-sql/installing-providers-from-sources.rst rename to docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-core-sql/operators.rst b/docs/apache-airflow-providers-common-sql/operators.rst similarity index 90% rename from docs/apache-airflow-providers-core-sql/operators.rst rename to docs/apache-airflow-providers-common-sql/operators.rst index c2b04767d4857..039e10b966cc1 100644 --- a/docs/apache-airflow-providers-core-sql/operators.rst +++ b/docs/apache-airflow-providers-common-sql/operators.rst @@ -16,7 +16,7 @@ under the License. SQL Operators -=================== +============= These operators perform various queries against a SQL database, including column- and table-level data quality checks. @@ -26,7 +26,7 @@ column- and table-level data quality checks. Check SQL Table Columns ~~~~~~~~~~~~~~~~~~~~~~~ -Use the :class:`~airflow.providers.core.sql.operators.sql.SQLColumnCheckOperator` to run data quality +Use the :class:`~airflow.providers.common.sql.operators.sql.SQLColumnCheckOperator` to run data quality checks against columns of a given table. As well as a connection ID and table, a column_mapping describing the relationship between columns and tests to run must be supplied. An example column mapping is a set of three nested dictionaries and looks like: @@ -71,7 +71,7 @@ be out of bounds but still considered successful. The below example demonstrates how to instantiate the SQLColumnCheckOperator task. -.. exampleinclude:: /../../airflow/providers/core/sql/example_dags/example_sql_column_table_check.py +.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_column_check] @@ -82,7 +82,7 @@ The below example demonstrates how to instantiate the SQLColumnCheckOperator tas Check SQL Table Values ~~~~~~~~~~~~~~~~~~~~~~~ -Use the :class:`~airflow.providers.core.sql.operators.sql.SQLTableCheckOperator` to run data quality +Use the :class:`~airflow.providers.common.sql.operators.sql.SQLTableCheckOperator` to run data quality checks against a given table. As well as a connection ID and table, a checks dictionary describing the relationship between the table and tests to run must be supplied. An example checks argument is a set of two nested dictionaries and looks like: @@ -105,7 +105,7 @@ airflow.operators.sql.parse_boolean). The below example demonstrates how to instantiate the SQLTableCheckOperator task. -.. exampleinclude:: /../../airflow/providers/core/sql/example_dags/example_sql_column_table_check.py +.. exampleinclude:: /../../tests/system/providers/common/sql/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_table_check] diff --git a/docs/apache-airflow-providers-odbc/connections/odbc.rst b/docs/apache-airflow-providers-odbc/connections/odbc.rst index 5d715596aa820..176977d8fec04 100644 --- a/docs/apache-airflow-providers-odbc/connections/odbc.rst +++ b/docs/apache-airflow-providers-odbc/connections/odbc.rst @@ -65,7 +65,7 @@ Extra (optional) * key-value pairs under ``connect_kwargs`` will be passed onto ``pyodbc.connect`` as kwargs - ``sqlalchemy_scheme`` * This is only used when ``get_uri`` is invoked in - :py:meth:`~airflow.hooks.dbapi.DbApiHook.get_sqlalchemy_engine`. By default, the hook uses + :py:meth:`~airflow.providers.common.sql.hooks.sql.DbApiHook.get_sqlalchemy_engine`. By default, the hook uses scheme ``mssql+pyodbc``. You may pass a string value here to override. .. note:: diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 0e7094ce179e4..a79b5625e7657 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -270,7 +270,7 @@ These are extras that provide support for integration with external systems via +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | extra | install command | enables | Preinstalled | +=====================+=====================================================+======================================+==============+ -| core.sql | ``pip install 'apache-airflow[core.sql]'`` | Core SQL Operators | | +| common.sql | ``pip install 'apache-airflow[common.sql]'`` | Core SQL Operators | * | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | ftp | ``pip install 'apache-airflow[ftp]'`` | FTP hooks and operators | * | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index e14ee8af7150a..70ad0658fbf34 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -17,6 +17,7 @@ }, "amazon": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "boto3>=1.15.0", "jsonpath_ng>=1.5.3", @@ -31,6 +32,7 @@ "cross-providers-deps": [ "apache.hive", "cncf.kubernetes", + "common.sql", "exasol", "ftp", "google", @@ -58,19 +60,24 @@ }, "apache.drill": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "sqlalchemy-drill>=1.1.0", "sqlparse>=0.4.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "apache.druid": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pydruid>=0.4.1" ], "cross-providers-deps": [ - "apache.hive" + "apache.hive", + "common.sql" ] }, "apache.hdfs": { @@ -83,6 +90,7 @@ }, "apache.hive": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "hmsclient>=0.1.0", "pandas>=0.17.1", @@ -92,6 +100,7 @@ ], "cross-providers-deps": [ "amazon", + "common.sql", "microsoft.mssql", "mysql", "presto", @@ -123,10 +132,13 @@ }, "apache.pinot": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pinotdb>0.1.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "apache.spark": { "deps": [ @@ -178,20 +190,21 @@ ], "cross-providers-deps": [] }, - "core.sql": { - "deps": [ - "apache-airflow>=2.2.0" - ], + "common.sql": { + "deps": [], "cross-providers-deps": [] }, "databricks": { "deps": [ "aiohttp>=3.6.3, <4", + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "databricks-sql-connector>=2.0.0, <3.0.0", "requests>=2.27,<3" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "datadog": { "deps": [ @@ -236,20 +249,26 @@ }, "elasticsearch": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "elasticsearch-dbapi", "elasticsearch-dsl>=5.0.0", "elasticsearch>7" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "exasol": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pandas>=0.17.1", "pyexasol>=0.5.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "facebook": { "deps": [ @@ -272,6 +291,7 @@ "google": { "deps": [ "PyOpenSSL", + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "google-ads>=15.1.1", "google-api-core>=2.7.0,<3.0.0", @@ -322,6 +342,7 @@ "apache.beam", "apache.cassandra", "cncf.kubernetes", + "common.sql", "facebook", "microsoft.azure", "microsoft.mssql", @@ -373,10 +394,13 @@ }, "jdbc": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "jaydebeapi>=1.1.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "jenkins": { "deps": [ @@ -418,10 +442,13 @@ }, "microsoft.mssql": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pymssql>=2.1.5; platform_machine != \"aarch64\"" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "microsoft.psrp": { "deps": [ @@ -446,12 +473,14 @@ }, "mysql": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "mysql-connector-python>=8.0.11; platform_machine != \"aarch64\"", "mysqlclient>=1.3.6; platform_machine != \"aarch64\"" ], "cross-providers-deps": [ "amazon", + "common.sql", "presto", "trino", "vertica" @@ -466,10 +495,13 @@ }, "odbc": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pyodbc" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "openfaas": { "deps": [ @@ -486,10 +518,13 @@ }, "oracle": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "oracledb>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "pagerduty": { "deps": [ @@ -515,30 +550,37 @@ }, "postgres": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "psycopg2-binary>=2.7.4" ], "cross-providers-deps": [ - "amazon" + "amazon", + "common.sql" ] }, "presto": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pandas>=0.17.1", "presto-python-client>=0.8.2" ], "cross-providers-deps": [ + "common.sql", "google", "slack" ] }, "qubole": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "qds-sdk>=1.10.4" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "redis": { "deps": [ @@ -594,27 +636,35 @@ }, "slack": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow-providers-http", "apache-airflow>=2.2.0", "slack_sdk>=3.0.0" ], "cross-providers-deps": [ + "common.sql", "http" ] }, "snowflake": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "snowflake-connector-python>=2.4.1", "snowflake-sqlalchemy>=1.1.0" ], "cross-providers-deps": [ + "common.sql", "slack" ] }, "sqlite": { - "deps": [], - "cross-providers-deps": [] + "deps": [ + "apache-airflow-providers-common-sql" + ], + "cross-providers-deps": [ + "common.sql" + ] }, "ssh": { "deps": [ @@ -646,20 +696,25 @@ }, "trino": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "pandas>=0.17.1", "trino>=0.301.0" ], "cross-providers-deps": [ + "common.sql", "google" ] }, "vertica": { "deps": [ + "apache-airflow-providers-common-sql", "apache-airflow>=2.2.0", "vertica-python>=0.5.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [ + "common.sql" + ] }, "yandex": { "deps": [ diff --git a/images/breeze/output-build-docs.svg b/images/breeze/output-build-docs.svg index 50c566d055002..5a05287b24205 100644 --- a/images/breeze/output-build-docs.svg +++ b/images/breeze/output-build-docs.svg @@ -19,265 +19,265 @@ font-weight: 700; } - .terminal-1768597619-matrix { + .terminal-3772557587-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1768597619-title { + .terminal-3772557587-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1768597619-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1768597619-r2 { fill: #c5c8c6 } -.terminal-1768597619-r3 { fill: #d0b344;font-weight: bold } -.terminal-1768597619-r4 { fill: #868887 } -.terminal-1768597619-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1768597619-r6 { fill: #98a84b;font-weight: bold } -.terminal-1768597619-r7 { fill: #8d7b39 } + .terminal-3772557587-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-3772557587-r2 { fill: #c5c8c6 } +.terminal-3772557587-r3 { fill: #d0b344;font-weight: bold } +.terminal-3772557587-r4 { fill: #868887 } +.terminal-3772557587-r5 { fill: #68a0b3;font-weight: bold } +.terminal-3772557587-r6 { fill: #98a84b;font-weight: bold } +.terminal-3772557587-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: build-docs + Command: build-docs - + - - -Usage: breeze build-docs [OPTIONS] - -Build documentation in the container. - -╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--docs-only-dOnly build documentation.│ -│--spellcheck-only-sOnly run spell checking.│ -│--for-production-pBuilds documentation for official release i.e. all links point to stable version.│ -│--package-filter-pList of packages to consider.                                                                │ -│(apache-airflow | apache-airflow-providers | apache-airflow-providers-airbyte |              │ -│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |                         │ -│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra |           │ -│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |              │ -│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |                │ -│apache-airflow-providers-apache-kylin | apache-airflow-providers-apache-livy |               │ -│apache-airflow-providers-apache-pig | apache-airflow-providers-apache-pinot |                │ -│apache-airflow-providers-apache-spark | apache-airflow-providers-apache-sqoop |              │ -│apache-airflow-providers-arangodb | apache-airflow-providers-asana |                         │ -│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                        │ -│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-core-sql |               │ -│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                     │ -│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                     │ -│apache-airflow-providers-discord | apache-airflow-providers-docker |                         │ -│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                   │ -│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                           │ -│apache-airflow-providers-github | apache-airflow-providers-google |                          │ -│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                         │ -│apache-airflow-providers-http | apache-airflow-providers-imap |                              │ -│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                          │ -│apache-airflow-providers-jenkins | apache-airflow-providers-jira |                           │ -│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql |        │ -│apache-airflow-providers-microsoft-psrp | apache-airflow-providers-microsoft-winrm |         │ -│apache-airflow-providers-mongo | apache-airflow-providers-mysql |                            │ -│apache-airflow-providers-neo4j | apache-airflow-providers-odbc |                             │ -│apache-airflow-providers-openfaas | apache-airflow-providers-opsgenie |                      │ -│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |                       │ -│apache-airflow-providers-papermill | apache-airflow-providers-plexus |                       │ -│apache-airflow-providers-postgres | apache-airflow-providers-presto |                        │ -│apache-airflow-providers-qubole | apache-airflow-providers-redis |                           │ -│apache-airflow-providers-salesforce | apache-airflow-providers-samba |                       │ -│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |                       │ -│apache-airflow-providers-sftp | apache-airflow-providers-singularity |                       │ -│apache-airflow-providers-slack | apache-airflow-providers-snowflake |                        │ -│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                             │ -│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                        │ -│apache-airflow-providers-telegram | apache-airflow-providers-trino |                         │ -│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                         │ -│apache-airflow-providers-zendesk | docker-stack | helm-chart)                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-docs [OPTIONS] + +Build documentation in the container. + +╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--docs-only-dOnly build documentation.│ +│--spellcheck-only-sOnly run spell checking.│ +│--for-production-pBuilds documentation for official release i.e. all links point to stable version.│ +│--package-filter-pList of packages to consider.                                                                │ +│(apache-airflow | apache-airflow-providers | apache-airflow-providers-airbyte |              │ +│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |                         │ +│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra |           │ +│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |              │ +│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |                │ +│apache-airflow-providers-apache-kylin | apache-airflow-providers-apache-livy |               │ +│apache-airflow-providers-apache-pig | apache-airflow-providers-apache-pinot |                │ +│apache-airflow-providers-apache-spark | apache-airflow-providers-apache-sqoop |              │ +│apache-airflow-providers-arangodb | apache-airflow-providers-asana |                         │ +│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                        │ +│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |             │ +│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                     │ +│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                     │ +│apache-airflow-providers-discord | apache-airflow-providers-docker |                         │ +│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                   │ +│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                           │ +│apache-airflow-providers-github | apache-airflow-providers-google |                          │ +│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                         │ +│apache-airflow-providers-http | apache-airflow-providers-imap |                              │ +│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                          │ +│apache-airflow-providers-jenkins | apache-airflow-providers-jira |                           │ +│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql |        │ +│apache-airflow-providers-microsoft-psrp | apache-airflow-providers-microsoft-winrm |         │ +│apache-airflow-providers-mongo | apache-airflow-providers-mysql |                            │ +│apache-airflow-providers-neo4j | apache-airflow-providers-odbc |                             │ +│apache-airflow-providers-openfaas | apache-airflow-providers-opsgenie |                      │ +│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |                       │ +│apache-airflow-providers-papermill | apache-airflow-providers-plexus |                       │ +│apache-airflow-providers-postgres | apache-airflow-providers-presto |                        │ +│apache-airflow-providers-qubole | apache-airflow-providers-redis |                           │ +│apache-airflow-providers-salesforce | apache-airflow-providers-samba |                       │ +│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |                       │ +│apache-airflow-providers-sftp | apache-airflow-providers-singularity |                       │ +│apache-airflow-providers-slack | apache-airflow-providers-snowflake |                        │ +│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                             │ +│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                        │ +│apache-airflow-providers-telegram | apache-airflow-providers-trino |                         │ +│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                         │ +│apache-airflow-providers-zendesk | docker-stack | helm-chart)                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 7b335b475dd0b..40907dae6ba0c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -3,7 +3,7 @@ # Please do not solve it but run `breeze regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:fa4319079b275ce966502346f083f2e3 -build-docs:39e674e357429ab1be7cc363cb793434 +build-docs:5ac8cf0870ec66fc7ebcf2363e823178 build-image:b62509a59badf3aa230e4562df751002 build-prod-image:1902ec077a6d70336de6038d13472ef3 cleanup:9a94bd1063296ea86e895f671db0b330 @@ -16,8 +16,8 @@ fix-ownership:596143cc74217f0a90850a554220ea45 free-space:bb8e7ac63d12ab3ede272a898de2f527 generate-constraints:a5120e79439f30eb7fbee929dca23156 prepare-airflow-package:cff9d88ca313db10f3cc464c6798f6be -prepare-provider-documentation:95c864f8a656a95cac7d9c682cb75773 -prepare-provider-packages:33c0fe04ad4c6068b69ad1361b142057 +prepare-provider-documentation:ff90e2d37c629e0f7b1f5e8bc723d9db +prepare-provider-packages:349292885c763f32db2bb8f99ae0ae59 pull-image:a9bb83372b5da5212f48e2affeedc551 pull-prod-image:6e8467a2b8c833a392c8bdd65189363e regenerate-command-images:4fd2e7ecbfd6eebb18b854f3eb0f29c8 diff --git a/images/breeze/output-prepare-provider-documentation.svg b/images/breeze/output-prepare-provider-documentation.svg index 6390fc20d9b88..c413c49f8c6d7 100644 --- a/images/breeze/output-prepare-provider-documentation.svg +++ b/images/breeze/output-prepare-provider-documentation.svg @@ -19,149 +19,149 @@ font-weight: 700; } - .terminal-1998463419-matrix { + .terminal-1019290203-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1998463419-title { + .terminal-1019290203-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1998463419-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1998463419-r2 { fill: #c5c8c6 } -.terminal-1998463419-r3 { fill: #d0b344;font-weight: bold } -.terminal-1998463419-r4 { fill: #868887 } -.terminal-1998463419-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1998463419-r6 { fill: #98a84b;font-weight: bold } -.terminal-1998463419-r7 { fill: #8d7b39 } + .terminal-1019290203-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-1019290203-r2 { fill: #c5c8c6 } +.terminal-1019290203-r3 { fill: #d0b344;font-weight: bold } +.terminal-1019290203-r4 { fill: #868887 } +.terminal-1019290203-r5 { fill: #68a0b3;font-weight: bold } +.terminal-1019290203-r6 { fill: #98a84b;font-weight: bold } +.terminal-1019290203-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: prepare-provider-documentation + Command: prepare-provider-documentation - + - - -Usage: breeze prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | -                                             apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | -                                             apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | -                                             arangodb | asana | celery | cloudant | cncf.kubernetes | core.sql | -                                             databricks | datadog | dbt.cloud | dingding | discord | docker | -                                             elasticsearch | exasol | facebook | ftp | github | google | grpc | -                                             hashicorp | http | imap | influxdb | jdbc | jenkins | jira | -                                             microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | -                                             mongo | mysql | neo4j | odbc | openfaas | opsgenie | oracle | pagerduty | -                                             papermill | plexus | postgres | presto | qubole | redis | salesforce | -                                             samba | segment | sendgrid | sftp | singularity | slack | snowflake | -                                             sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | -                                             zendesk]... - -Prepare CHANGELOG, README and COMMITS information for providers. - -╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | +                                             apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | +                                             apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | +                                             arangodb | asana | celery | cloudant | cncf.kubernetes | common.sql | +                                             databricks | datadog | dbt.cloud | dingding | discord | docker | +                                             elasticsearch | exasol | facebook | ftp | github | google | grpc | +                                             hashicorp | http | imap | influxdb | jdbc | jenkins | jira | +                                             microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | +                                             mongo | mysql | neo4j | odbc | openfaas | opsgenie | oracle | pagerduty | +                                             papermill | plexus | postgres | presto | qubole | redis | salesforce | +                                             samba | segment | sendgrid | sftp | singularity | slack | snowflake | +                                             sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | +                                             zendesk]... + +Prepare CHANGELOG, README and COMMITS information for providers. + +╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-prepare-provider-packages.svg b/images/breeze/output-prepare-provider-packages.svg index fe0c1733cbf59..aa4ba89f53582 100644 --- a/images/breeze/output-prepare-provider-packages.svg +++ b/images/breeze/output-prepare-provider-packages.svg @@ -19,153 +19,153 @@ font-weight: 700; } - .terminal-3451094975-matrix { + .terminal-2104395871-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3451094975-title { + .terminal-2104395871-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3451094975-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-3451094975-r2 { fill: #c5c8c6 } -.terminal-3451094975-r3 { fill: #d0b344;font-weight: bold } -.terminal-3451094975-r4 { fill: #868887 } -.terminal-3451094975-r5 { fill: #68a0b3;font-weight: bold } -.terminal-3451094975-r6 { fill: #8d7b39 } -.terminal-3451094975-r7 { fill: #98a84b;font-weight: bold } + .terminal-2104395871-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2104395871-r2 { fill: #c5c8c6 } +.terminal-2104395871-r3 { fill: #d0b344;font-weight: bold } +.terminal-2104395871-r4 { fill: #868887 } +.terminal-2104395871-r5 { fill: #68a0b3;font-weight: bold } +.terminal-2104395871-r6 { fill: #8d7b39 } +.terminal-2104395871-r7 { fill: #98a84b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: prepare-provider-packages + Command: prepare-provider-packages - + - - -Usage: breeze prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | -                                        apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | -                                        apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | -                                        arangodb | asana | celery | cloudant | cncf.kubernetes | core.sql | databricks -                                        | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | -                                        facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | -                                        jdbc | jenkins | jira | microsoft.azure | microsoft.mssql | microsoft.psrp | -                                        microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | opsgenie | oracle -                                        | pagerduty | papermill | plexus | postgres | presto | qubole | redis | -                                        salesforce | samba | segment | sendgrid | sftp | singularity | slack | -                                        snowflake | sqlite | ssh | tableau | tabular | telegram | trino | vertica | -                                        yandex | zendesk]... - -Prepare sdist/whl packages of Airflow Providers. - -╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--package-list-fileRead list of packages from text file (one package per line)(FILENAME)│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | +                                        apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | +                                        apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | +                                        arangodb | asana | celery | cloudant | cncf.kubernetes | common.sql | +                                        databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch +                                        | exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | +                                        influxdb | jdbc | jenkins | jira | microsoft.azure | microsoft.mssql | +                                        microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | +                                        opsgenie | oracle | pagerduty | papermill | plexus | postgres | presto | +                                        qubole | redis | salesforce | samba | segment | sendgrid | sftp | singularity +                                        | slack | snowflake | sqlite | ssh | tableau | tabular | telegram | trino | +                                        vertica | yandex | zendesk]... + +Prepare sdist/whl packages of Airflow Providers. + +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--package-list-fileRead list of packages from text file (one package per line)(FILENAME)│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/newsfragments/NEW.significant.rst b/newsfragments/NEW.significant.rst new file mode 100644 index 0000000000000..01a1b678edcf7 --- /dev/null +++ b/newsfragments/NEW.significant.rst @@ -0,0 +1 @@ +The DB related classes: ``DBApiHook``, ``SQLSensor`` have been moved to ``apache-airflow-providers-common-sql`` provider. diff --git a/scripts/ci/installed_providers.txt b/scripts/ci/installed_providers.txt index c6b02bfae16b3..013fb587e3583 100644 --- a/scripts/ci/installed_providers.txt +++ b/scripts/ci/installed_providers.txt @@ -1,6 +1,7 @@ amazon celery cncf.kubernetes +common.sql docker elasticsearch ftp diff --git a/setup.py b/setup.py index 991b6b0617178..2435ac046e5aa 100644 --- a/setup.py +++ b/setup.py @@ -654,6 +654,7 @@ def sort_extras_dependencies() -> Dict[str, List[str]]: # Those providers do not have dependency on airflow2.0 because that would lead to circular dependencies. # This is not a problem for PIP but some tools (pipdeptree) show those as a warning. PREINSTALLED_PROVIDERS = [ + 'common.sql', 'ftp', 'http', 'imap', diff --git a/tests/deprecated_classes.py b/tests/deprecated_classes.py index c06adedbc1352..522e62da72f0e 100644 --- a/tests/deprecated_classes.py +++ b/tests/deprecated_classes.py @@ -19,10 +19,6 @@ "airflow.hooks.base.BaseHook", "airflow.hooks.base_hook.BaseHook", ), - ( - "airflow.hooks.dbapi.DbApiHook", - "airflow.hooks.dbapi_hook.DbApiHook", - ), ( "airflow.providers.apache.cassandra.hooks.cassandra.CassandraHook", "airflow.contrib.hooks.cassandra_hook.CassandraHook", diff --git a/tests/operators/test_generic_transfer.py b/tests/operators/test_generic_transfer.py index 2ce165b126f03..b56666db79677 100644 --- a/tests/operators/test_generic_transfer.py +++ b/tests/operators/test_generic_transfer.py @@ -74,7 +74,7 @@ def test_mysql_to_mysql(self, client): ) op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - @mock.patch('airflow.hooks.dbapi.DbApiHook.insert_rows') + @mock.patch('airflow.providers.common.sql.hooks.sql.DbApiHook.insert_rows') def test_mysql_to_mysql_replace(self, mock_insert): sql = "SELECT * FROM connection LIMIT 10;" op = GenericTransfer( @@ -126,7 +126,7 @@ def test_postgres_to_postgres(self): ) op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - @mock.patch('airflow.hooks.dbapi.DbApiHook.insert_rows') + @mock.patch('airflow.providers.common.sql.hooks.sql.DbApiHook.insert_rows') def test_postgres_to_postgres_replace(self, mock_insert): sql = "SELECT id, conn_id, conn_type FROM connection LIMIT 10;" op = GenericTransfer( diff --git a/tests/providers/core/sql/__init__.py b/tests/providers/common/__init__.py similarity index 100% rename from tests/providers/core/sql/__init__.py rename to tests/providers/common/__init__.py diff --git a/tests/providers/core/sql/operators/__init__.py b/tests/providers/common/sql/__init__.py similarity index 100% rename from tests/providers/core/sql/operators/__init__.py rename to tests/providers/common/sql/__init__.py diff --git a/tests/providers/common/sql/hooks/__init__.py b/tests/providers/common/sql/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/common/sql/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/hooks/test_dbapi.py b/tests/providers/common/sql/hooks/test_dbapi.py similarity index 99% rename from tests/hooks/test_dbapi.py rename to tests/providers/common/sql/hooks/test_dbapi.py index ad5e7ba6af065..a44fa57e075a7 100644 --- a/tests/hooks/test_dbapi.py +++ b/tests/providers/common/sql/hooks/test_dbapi.py @@ -23,8 +23,8 @@ import pytest -from airflow.hooks.dbapi import DbApiHook from airflow.models import Connection +from airflow.providers.common.sql.hooks.sql import DbApiHook class TestDbApiHook(unittest.TestCase): diff --git a/tests/providers/common/sql/operators/__init__.py b/tests/providers/common/sql/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/common/sql/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/core/sql/operators/test_sql.py b/tests/providers/common/sql/operators/test_sql.py similarity index 97% rename from tests/providers/core/sql/operators/test_sql.py rename to tests/providers/common/sql/operators/test_sql.py index d39e2d9faeccf..63ef78ba1fe0f 100644 --- a/tests/providers/core/sql/operators/test_sql.py +++ b/tests/providers/common/sql/operators/test_sql.py @@ -19,7 +19,7 @@ import pytest from airflow.exceptions import AirflowException -from airflow.providers.core.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator +from airflow.providers.common.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator class MockHook: diff --git a/tests/providers/common/sql/sensors/__init__.py b/tests/providers/common/sql/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/common/sql/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/sensors/test_sql_sensor.py b/tests/providers/common/sql/sensors/test_sql.py similarity index 93% rename from tests/sensors/test_sql_sensor.py rename to tests/providers/common/sql/sensors/test_sql.py index 23c31aa170088..9746c073610ec 100644 --- a/tests/sensors/test_sql_sensor.py +++ b/tests/providers/common/sql/sensors/test_sql.py @@ -23,7 +23,8 @@ from airflow.exceptions import AirflowException from airflow.models.dag import DAG -from airflow.sensors.sql import DbApiHook, SqlSensor +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.common.sql.sensors.sql import SqlSensor from airflow.utils.timezone import datetime from tests.providers.apache.hive import TestHiveEnvironment @@ -86,7 +87,7 @@ def test_sql_sensor_postgres(self): ) op2.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', @@ -118,7 +119,7 @@ def test_sql_sensor_postgres_poke(self, mock_hook): mock_get_records.return_value = [['1']] assert op.poke(None) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_fail_on_empty(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', conn_id='postgres_default', sql="SELECT 1", fail_on_empty=True @@ -131,7 +132,7 @@ def test_sql_sensor_postgres_poke_fail_on_empty(self, mock_hook): with pytest.raises(AirflowException): op.poke(None) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_success(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', conn_id='postgres_default', sql="SELECT 1", success=lambda x: x in [1] @@ -149,7 +150,7 @@ def test_sql_sensor_postgres_poke_success(self, mock_hook): mock_get_records.return_value = [['1']] assert not op.poke(None) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_failure(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', conn_id='postgres_default', sql="SELECT 1", failure=lambda x: x in [1] @@ -165,7 +166,7 @@ def test_sql_sensor_postgres_poke_failure(self, mock_hook): with pytest.raises(AirflowException): op.poke(None) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_failure_success(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', @@ -188,7 +189,7 @@ def test_sql_sensor_postgres_poke_failure_success(self, mock_hook): mock_get_records.return_value = [[2]] assert op.poke(None) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_failure_success_same(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', @@ -208,7 +209,7 @@ def test_sql_sensor_postgres_poke_failure_success_same(self, mock_hook): with pytest.raises(AirflowException): op.poke(None) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_invalid_failure(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', @@ -225,7 +226,7 @@ def test_sql_sensor_postgres_poke_invalid_failure(self, mock_hook): op.poke(None) assert "self.failure is present, but not callable -> [1]" == str(ctx.value) - @mock.patch('airflow.sensors.sql.BaseHook') + @mock.patch('airflow.providers.common.sql.sensors.sql.BaseHook') def test_sql_sensor_postgres_poke_invalid_success(self, mock_hook): op = SqlSensor( task_id='sql_sensor_check', diff --git a/tests/providers/microsoft/mssql/hooks/test_mssql.py b/tests/providers/microsoft/mssql/hooks/test_mssql.py index 63e032406d03e..202c7f8749bfe 100644 --- a/tests/providers/microsoft/mssql/hooks/test_mssql.py +++ b/tests/providers/microsoft/mssql/hooks/test_mssql.py @@ -31,7 +31,7 @@ class TestMsSqlHook(unittest.TestCase): @unittest.skipIf(PY38, "Mssql package not available when Python >= 3.8.") @mock.patch('airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook.get_conn') - @mock.patch('airflow.hooks.dbapi.DbApiHook.get_connection') + @mock.patch('airflow.providers.common.sql.hooks.sql.DbApiHook.get_connection') def test_get_conn_should_return_connection(self, get_connection, mssql_get_conn): get_connection.return_value = PYMSSQL_CONN mssql_get_conn.return_value = mock.Mock() @@ -44,7 +44,7 @@ def test_get_conn_should_return_connection(self, get_connection, mssql_get_conn) @unittest.skipIf(PY38, "Mssql package not available when Python >= 3.8.") @mock.patch('airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook.get_conn') - @mock.patch('airflow.hooks.dbapi.DbApiHook.get_connection') + @mock.patch('airflow.providers.common.sql.hooks.sql.DbApiHook.get_connection') def test_set_autocommit_should_invoke_autocommit(self, get_connection, mssql_get_conn): get_connection.return_value = PYMSSQL_CONN mssql_get_conn.return_value = mock.Mock() @@ -59,7 +59,7 @@ def test_set_autocommit_should_invoke_autocommit(self, get_connection, mssql_get @unittest.skipIf(PY38, "Mssql package not available when Python >= 3.8.") @mock.patch('airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook.get_conn') - @mock.patch('airflow.hooks.dbapi.DbApiHook.get_connection') + @mock.patch('airflow.providers.common.sql.hooks.sql.DbApiHook.get_connection') def test_get_autocommit_should_return_autocommit_state(self, get_connection, mssql_get_conn): get_connection.return_value = PYMSSQL_CONN mssql_get_conn.return_value = mock.Mock() diff --git a/tests/providers/presto/hooks/test_presto.py b/tests/providers/presto/hooks/test_presto.py index e6fd7c5ed01b1..61f6dcb911abd 100644 --- a/tests/providers/presto/hooks/test_presto.py +++ b/tests/providers/presto/hooks/test_presto.py @@ -234,7 +234,7 @@ def get_isolation_level(self): self.db_hook = UnitTestPrestoHook() - @patch('airflow.hooks.dbapi.DbApiHook.insert_rows') + @patch('airflow.providers.common.sql.hooks.sql.DbApiHook.insert_rows') def test_insert_rows(self, mock_insert_rows): table = "table" rows = [("hello",), ("world",)] diff --git a/tests/providers/trino/hooks/test_trino.py b/tests/providers/trino/hooks/test_trino.py index a9be6545bfcd5..2fdad8b71e893 100644 --- a/tests/providers/trino/hooks/test_trino.py +++ b/tests/providers/trino/hooks/test_trino.py @@ -195,7 +195,7 @@ def get_isolation_level(self): self.db_hook = UnitTestTrinoHook() - @patch('airflow.hooks.dbapi.DbApiHook.insert_rows') + @patch('airflow.providers.common.sql.hooks.sql.DbApiHook.insert_rows') def test_insert_rows(self, mock_insert_rows): table = "table" rows = [("hello",), ("world",)] diff --git a/tests/providers/vertica/hooks/test_vertica.py b/tests/providers/vertica/hooks/test_vertica.py index 23ee7e5a0a47b..de6fe56bcbd6f 100644 --- a/tests/providers/vertica/hooks/test_vertica.py +++ b/tests/providers/vertica/hooks/test_vertica.py @@ -68,7 +68,7 @@ def get_conn(self): self.db_hook = UnitTestVerticaHook() - @patch('airflow.hooks.dbapi.DbApiHook.insert_rows') + @patch('airflow.providers.common.sql.hooks.sql.DbApiHook.insert_rows') def test_insert_rows(self, mock_insert_rows): table = "table" rows = [("hello",), ("world",)] diff --git a/tests/system/providers/common/__init__.py b/tests/system/providers/common/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/common/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/common/sql/__init__.py b/tests/system/providers/common/sql/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/common/sql/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/core/sql/example_dags/example_sql_column_table_check.py b/tests/system/providers/common/sql/example_sql_column_table_check.py similarity index 89% rename from airflow/providers/core/sql/example_dags/example_sql_column_table_check.py rename to tests/system/providers/common/sql/example_sql_column_table_check.py index e83f0217655a3..034060ae8b882 100644 --- a/airflow/providers/core/sql/example_dags/example_sql_column_table_check.py +++ b/tests/system/providers/common/sql/example_sql_column_table_check.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. from airflow import DAG -from airflow.providers.core.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator +from airflow.providers.common.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator from airflow.utils.dates import datetime AIRFLOW_DB_METADATA_TABLE = "ab_role" @@ -75,3 +75,9 @@ # [END howto_operator_sql_table_check] column_check >> row_count_check + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag)