From 89f31a0c8778c9f6e2c1370eb1c2ca565d1d3a3b Mon Sep 17 00:00:00 2001 From: Benji Lampel Date: Thu, 30 Jun 2022 11:57:00 -0400 Subject: [PATCH 1/2] Fix string formatting in logs --- airflow/providers/core/sql/operators/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/core/sql/operators/sql.py b/airflow/providers/core/sql/operators/sql.py index 1d9945f403ec3..63b6457c759f2 100644 --- a/airflow/providers/core/sql/operators/sql.py +++ b/airflow/providers/core/sql/operators/sql.py @@ -121,7 +121,7 @@ def execute(self, context=None): if not records: raise AirflowException(f"The following query returned zero rows: {self.sql}") - self.log.info(f"Record: {records}") + self.log.info("Record: %s", records) for idx, result in enumerate(records): tolerance = self.column_mapping[column][checks[idx]].get("tolerance") @@ -298,7 +298,7 @@ def execute(self, context=None): if not records: raise AirflowException(f"The following query returned zero rows: {self.sql}") - self.log.info(f"Record: {records}") + self.log.info("Record: %s", records) for check in self.checks.keys(): for result in records: From 927fd268d6d3ab4aa8253633a0ac4f0278208e08 Mon Sep 17 00:00:00 2001 From: Benji Lampel Date: Fri, 1 Jul 2022 14:45:27 -0400 Subject: [PATCH 2/2] Rename core.sql provider to common.sql Conversation around the initial PR suggested core.sql was not a good name, and common.sql seemed to be the consensus choice. --- .../airflow_providers_bug_report.yml | 2 +- CONTRIBUTING.rst | 18 +++++++++--------- INSTALL | 18 +++++++++--------- .../providers/{core => common}/__init__.py | 0 .../{core => common}/sql/CHANGELOG.rst | 0 .../{core => common}/sql/__init__.py | 0 .../sql/example_dags/__init__.py | 0 .../example_sql_column_table_check.py | 2 +- .../sql/operators/__init__.py | 0 .../{core => common}/sql/operators/sql.py | 0 .../{core => common}/sql/provider.yaml | 16 ++++++++-------- .../commits.rst | 6 +++--- .../connections.rst | 0 .../index.rst | 10 +++++----- .../installing-providers-from-sources.rst | 0 .../operators.rst | 8 ++++---- docs/apache-airflow/extra-packages-ref.rst | 2 +- .../{core => common}/sql/sql.png | Bin generated/provider_dependencies.json | 2 +- setup.py | 1 + tests/providers/{core => common}/__init__.py | 0 .../{core => common}/sql/__init__.py | 0 .../sql/operators/__init__.py | 0 .../sql/operators/test_sql.py | 0 24 files changed, 43 insertions(+), 42 deletions(-) rename airflow/providers/{core => common}/__init__.py (100%) rename airflow/providers/{core => common}/sql/CHANGELOG.rst (100%) rename airflow/providers/{core => common}/sql/__init__.py (100%) rename airflow/providers/{core => common}/sql/example_dags/__init__.py (100%) rename airflow/providers/{core => common}/sql/example_dags/example_sql_column_table_check.py (95%) rename airflow/providers/{core => common}/sql/operators/__init__.py (100%) rename airflow/providers/{core => common}/sql/operators/sql.py (100%) rename airflow/providers/{core => common}/sql/provider.yaml (73%) rename docs/{apache-airflow-providers-core-sql => apache-airflow-providers-common-sql}/commits.rst (88%) rename docs/{apache-airflow-providers-core-sql => apache-airflow-providers-common-sql}/connections.rst (100%) rename docs/{apache-airflow-providers-core-sql => apache-airflow-providers-common-sql}/index.rst (88%) rename docs/{apache-airflow-providers-core-sql => apache-airflow-providers-common-sql}/installing-providers-from-sources.rst (100%) rename docs/{apache-airflow-providers-core-sql => apache-airflow-providers-common-sql}/operators.rst (90%) rename docs/integration-logos/{core => common}/sql/sql.png (100%) rename tests/providers/{core => common}/__init__.py (100%) rename tests/providers/{core => common}/sql/__init__.py (100%) rename tests/providers/{core => common}/sql/operators/__init__.py (100%) rename tests/providers/{core => common}/sql/operators/test_sql.py (100%) diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 67be91e8cb8b2..ebaca3b329d12 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -43,7 +43,7 @@ body: - celery - cloudant - cncf-kubernetes - - core-sql + - common-sql - databricks - datadog - dbt-cloud diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 283e430aed3ca..61f8d1f47f091 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -620,15 +620,15 @@ This is the full list of those extras: airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas, aws, azure, cassandra, -celery, cgroups, cloudant, cncf.kubernetes, core.sql, crypto, dask, databricks, datadog, dbt.cloud, -deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, -elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, -grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, -leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, -neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, -postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, -sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, -virtualenv, webhdfs, winrm, yandex, zendesk +celery, cgroups, cloudant, cncf.kubernetes, common.sql, crypto, dask, databricks, datadog, +dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, +druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, +google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, +kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, +mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, +pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, +sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, +telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE Provider packages diff --git a/INSTALL b/INSTALL index 9694ba1f5bdba..b246e923f33ac 100644 --- a/INSTALL +++ b/INSTALL @@ -97,15 +97,15 @@ The list of available extras: airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas, aws, azure, cassandra, -celery, cgroups, cloudant, cncf.kubernetes, core.sql, crypto, dask, databricks, datadog, dbt.cloud, -deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, -elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, -grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, -leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, -neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, -postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, -sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, -virtualenv, webhdfs, winrm, yandex, zendesk +celery, cgroups, cloudant, cncf.kubernetes, common.sql, crypto, dask, databricks, datadog, +dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, +druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, +google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, +kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, +mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, +pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, +sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, +telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/providers/core/__init__.py b/airflow/providers/common/__init__.py similarity index 100% rename from airflow/providers/core/__init__.py rename to airflow/providers/common/__init__.py diff --git a/airflow/providers/core/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst similarity index 100% rename from airflow/providers/core/sql/CHANGELOG.rst rename to airflow/providers/common/sql/CHANGELOG.rst diff --git a/airflow/providers/core/sql/__init__.py b/airflow/providers/common/sql/__init__.py similarity index 100% rename from airflow/providers/core/sql/__init__.py rename to airflow/providers/common/sql/__init__.py diff --git a/airflow/providers/core/sql/example_dags/__init__.py b/airflow/providers/common/sql/example_dags/__init__.py similarity index 100% rename from airflow/providers/core/sql/example_dags/__init__.py rename to airflow/providers/common/sql/example_dags/__init__.py diff --git a/airflow/providers/core/sql/example_dags/example_sql_column_table_check.py b/airflow/providers/common/sql/example_dags/example_sql_column_table_check.py similarity index 95% rename from airflow/providers/core/sql/example_dags/example_sql_column_table_check.py rename to airflow/providers/common/sql/example_dags/example_sql_column_table_check.py index e83f0217655a3..8e72051404998 100644 --- a/airflow/providers/core/sql/example_dags/example_sql_column_table_check.py +++ b/airflow/providers/common/sql/example_dags/example_sql_column_table_check.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. from airflow import DAG -from airflow.providers.core.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator +from airflow.providers.common.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator from airflow.utils.dates import datetime AIRFLOW_DB_METADATA_TABLE = "ab_role" diff --git a/airflow/providers/core/sql/operators/__init__.py b/airflow/providers/common/sql/operators/__init__.py similarity index 100% rename from airflow/providers/core/sql/operators/__init__.py rename to airflow/providers/common/sql/operators/__init__.py diff --git a/airflow/providers/core/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py similarity index 100% rename from airflow/providers/core/sql/operators/sql.py rename to airflow/providers/common/sql/operators/sql.py diff --git a/airflow/providers/core/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml similarity index 73% rename from airflow/providers/core/sql/provider.yaml rename to airflow/providers/common/sql/provider.yaml index f9519842679de..ac92f70167c49 100644 --- a/airflow/providers/core/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -16,10 +16,10 @@ # under the License. --- -package-name: apache-airflow-providers-core-sql -name: Core SQL +package-name: apache-airflow-providers-common-sql +name: Common SQL description: | - `Core SQL Provider `__ + `Common SQL Provider `__ versions: - 1.0.0 @@ -28,14 +28,14 @@ dependencies: - apache-airflow>=2.2.0 integrations: - - integration-name: Core SQL + - integration-name: Common SQL external-doc-url: https://en.wikipedia.org/wiki/SQL how-to-guide: - - /docs/apache-airflow-providers-core-sql/operators.rst - logo: /integration-logos/core/sql/sql.png + - /docs/apache-airflow-providers-common-sql/operators.rst + logo: /integration-logos/common/sql/sql.png tags: [software] operators: - - integration-name: Core SQL + - integration-name: Common SQL python-modules: - - airflow.providers.core.sql.operators.sql + - airflow.providers.common.sql.operators.sql diff --git a/docs/apache-airflow-providers-core-sql/commits.rst b/docs/apache-airflow-providers-common-sql/commits.rst similarity index 88% rename from docs/apache-airflow-providers-core-sql/commits.rst rename to docs/apache-airflow-providers-common-sql/commits.rst index 8292a11950bf7..ab22de4686039 100644 --- a/docs/apache-airflow-providers-core-sql/commits.rst +++ b/docs/apache-airflow-providers-common-sql/commits.rst @@ -15,11 +15,11 @@ specific language governing permissions and limitations under the License. -Package apache-airflow-providers-core-sql +Package apache-airflow-providers-common-sql ------------------------------------------ -`Core SQL Provider `__ +`Common SQL Provider `__ -This is detailed commit list of changes for versions provider package: ``core.sql``. +This is detailed commit list of changes for versions provider package: ``common.sql``. For high-level changelog, see :doc:`package information including changelog `. diff --git a/docs/apache-airflow-providers-core-sql/connections.rst b/docs/apache-airflow-providers-common-sql/connections.rst similarity index 100% rename from docs/apache-airflow-providers-core-sql/connections.rst rename to docs/apache-airflow-providers-common-sql/connections.rst diff --git a/docs/apache-airflow-providers-core-sql/index.rst b/docs/apache-airflow-providers-common-sql/index.rst similarity index 88% rename from docs/apache-airflow-providers-core-sql/index.rst rename to docs/apache-airflow-providers-common-sql/index.rst index abc6186b68557..e197f780228b3 100644 --- a/docs/apache-airflow-providers-core-sql/index.rst +++ b/docs/apache-airflow-providers-common-sql/index.rst @@ -15,7 +15,7 @@ specific language governing permissions and limitations under the License. -``apache-airflow-providers-core-sql`` +``apache-airflow-providers-common-sql`` ============================================ Content @@ -32,14 +32,14 @@ Content :maxdepth: 1 :caption: References - Python API <_api/airflow/providers/core/sql/index> + Python API <_api/airflow/providers/common/sql/index> .. toctree:: :maxdepth: 1 :caption: Resources - Example DAGs - PyPI Repository + Example DAGs + PyPI Repository Installing from sources .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! @@ -50,5 +50,5 @@ Content Detailed list of commits -Package apache-airflow-providers-core-sql +Package apache-airflow-providers-common-sql ------------------------------------------------------ diff --git a/docs/apache-airflow-providers-core-sql/installing-providers-from-sources.rst b/docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst similarity index 100% rename from docs/apache-airflow-providers-core-sql/installing-providers-from-sources.rst rename to docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-core-sql/operators.rst b/docs/apache-airflow-providers-common-sql/operators.rst similarity index 90% rename from docs/apache-airflow-providers-core-sql/operators.rst rename to docs/apache-airflow-providers-common-sql/operators.rst index c2b04767d4857..a93009c39e195 100644 --- a/docs/apache-airflow-providers-core-sql/operators.rst +++ b/docs/apache-airflow-providers-common-sql/operators.rst @@ -26,7 +26,7 @@ column- and table-level data quality checks. Check SQL Table Columns ~~~~~~~~~~~~~~~~~~~~~~~ -Use the :class:`~airflow.providers.core.sql.operators.sql.SQLColumnCheckOperator` to run data quality +Use the :class:`~airflow.providers.common.sql.operators.sql.SQLColumnCheckOperator` to run data quality checks against columns of a given table. As well as a connection ID and table, a column_mapping describing the relationship between columns and tests to run must be supplied. An example column mapping is a set of three nested dictionaries and looks like: @@ -71,7 +71,7 @@ be out of bounds but still considered successful. The below example demonstrates how to instantiate the SQLColumnCheckOperator task. -.. exampleinclude:: /../../airflow/providers/core/sql/example_dags/example_sql_column_table_check.py +.. exampleinclude:: /../../airflow/providers/common/sql/example_dags/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_column_check] @@ -82,7 +82,7 @@ The below example demonstrates how to instantiate the SQLColumnCheckOperator tas Check SQL Table Values ~~~~~~~~~~~~~~~~~~~~~~~ -Use the :class:`~airflow.providers.core.sql.operators.sql.SQLTableCheckOperator` to run data quality +Use the :class:`~airflow.providers.common.sql.operators.sql.SQLTableCheckOperator` to run data quality checks against a given table. As well as a connection ID and table, a checks dictionary describing the relationship between the table and tests to run must be supplied. An example checks argument is a set of two nested dictionaries and looks like: @@ -105,7 +105,7 @@ airflow.operators.sql.parse_boolean). The below example demonstrates how to instantiate the SQLTableCheckOperator task. -.. exampleinclude:: /../../airflow/providers/core/sql/example_dags/example_sql_column_table_check.py +.. exampleinclude:: /../../airflow/providers/common/sql/example_dags/example_sql_column_table_check.py :language: python :dedent: 4 :start-after: [START howto_operator_sql_table_check] diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 188b9a02f9944..dfe2196892662 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -268,7 +268,7 @@ These are extras that provide support for integration with external systems via +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | extra | install command | enables | Preinstalled | +=====================+=====================================================+======================================+==============+ -| core.sql | ``pip install 'apache-airflow[core.sql]'`` | Core SQL Operators | | +| common.sql | ``pip install 'apache-airflow[common.sql]'`` | Common SQL Operators | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | ftp | ``pip install 'apache-airflow[ftp]'`` | FTP hooks and operators | * | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ diff --git a/docs/integration-logos/core/sql/sql.png b/docs/integration-logos/common/sql/sql.png similarity index 100% rename from docs/integration-logos/core/sql/sql.png rename to docs/integration-logos/common/sql/sql.png diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 6226703df35e5..3518a7bc93c30 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -178,7 +178,7 @@ ], "cross-providers-deps": [] }, - "core.sql": { + "common.sql": { "deps": [ "apache-airflow>=2.2.0" ], diff --git a/setup.py b/setup.py index 991b6b0617178..2a41bbfd30d03 100644 --- a/setup.py +++ b/setup.py @@ -562,6 +562,7 @@ def add_all_deprecated_provider_packages() -> None: 'apache.pinot', 'arangodb', 'cloudant', + 'common.sql', 'databricks', 'exasol', 'influxdb', diff --git a/tests/providers/core/__init__.py b/tests/providers/common/__init__.py similarity index 100% rename from tests/providers/core/__init__.py rename to tests/providers/common/__init__.py diff --git a/tests/providers/core/sql/__init__.py b/tests/providers/common/sql/__init__.py similarity index 100% rename from tests/providers/core/sql/__init__.py rename to tests/providers/common/sql/__init__.py diff --git a/tests/providers/core/sql/operators/__init__.py b/tests/providers/common/sql/operators/__init__.py similarity index 100% rename from tests/providers/core/sql/operators/__init__.py rename to tests/providers/common/sql/operators/__init__.py diff --git a/tests/providers/core/sql/operators/test_sql.py b/tests/providers/common/sql/operators/test_sql.py similarity index 100% rename from tests/providers/core/sql/operators/test_sql.py rename to tests/providers/common/sql/operators/test_sql.py