diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8b539430cb4bd..7f73946d9ec22 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -610,14 +610,14 @@ all dependencies needed in the CI environment. This is the full list of those extras: .. START EXTRAS HERE -airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, -apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin, apache.livy, -apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas, -atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, common.sql, -crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, -devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, -gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, -imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, +aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, +apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin, +apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, +async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, +common.sql, crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, +devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, +facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, +hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, diff --git a/Dockerfile b/Dockerfile index 4bc400036d55b..75bd176c5d71a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ # much smaller. # # Use the same builder frontend version for everyone -ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv" +ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv" ARG ADDITIONAL_AIRFLOW_EXTRAS="" ARG ADDITIONAL_PYTHON_DEPS="" diff --git a/INSTALL b/INSTALL index 87a759bc02489..fe10ce91e1014 100644 --- a/INSTALL +++ b/INSTALL @@ -94,14 +94,14 @@ pip install .[async,google,amazon] \ The list of available extras: # START EXTRAS HERE -airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, -apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin, apache.livy, -apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas, -atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, common.sql, -crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, -devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, -gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, -imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, +aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, +apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin, +apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, +async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, +common.sql, crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, +devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, +facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, +hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index d2ea38d4c8209..00887e3ab1be9 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -257,6 +257,7 @@ def get_airflow_extras(): DEFAULT_EXTRAS = [ # BEGINNING OF EXTRAS LIST UPDATED BY PRE COMMIT + "aiobotocore", "amazon", "async", "celery", diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 7cb7df771b623..ff7bd8c6a773e 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -40,6 +40,8 @@ python dependencies for the provided package. +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | extra | install command | enables | +=====================+=====================================================+============================================================================+ +| aiobotocore | ``pip install 'apache-airflow[aiobotocore]'`` | Support for asynchronous (deferrable) operators for Amazon integration | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor (also installs the celery provider package!) | diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index bd762130360c1..a3f959e6d4667 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -80,6 +80,7 @@ List of default extras in the production Dockerfile: .. BEGINNING OF EXTRAS LIST UPDATED BY PRE COMMIT +* aiobotocore * amazon * async * celery diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 3f197d9eabcce..22cd50e976b08 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -31,10 +31,10 @@ k8s:status:2b8aa0b9597a1790c184c7b776ebd9c3 k8s:tests:dc44e907de908d1c4f11cf5d99915da1 k8s:upload-k8s-image:b012457bb6a98524859ecaf5746b4439 k8s:a04e7db8fb49c16a0633ac32f14463ec -prod-image:build:5d0fcf0f0b1f4a8d13378253214162d6 +prod-image:build:a1608071b5a80b3cc442da9d00e1525b prod-image:pull:e3c89dd908fc44adf6e159c2950ebdd0 prod-image:verify:31bc5efada1d70a0a31990025db1a093 -prod-image:2ac8e1e6b4afb259f00390ad0e90a06f +prod-image:79bd4cc9de03ab7e1d75f025d75eee46 release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957 release-management:generate-issue-content-providers:f3c00ba74e3afc054fe29b65156740ac diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index c3b906463ca15..bf8a3d7bee23c 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -371,7 +371,7 @@ │--airflow-extrasExtras to install by default.                                                   │ │(TEXT)                                                                          │ │[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│aiobotocore,amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,g…│ │--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ │file. It could be full remote URL to the location file, or local file placed in │ │`docker-context-files` (in this case it has to start with                       │ diff --git a/setup.py b/setup.py index 2ac40e2c6abcc..8416f3db3f755 100644 --- a/setup.py +++ b/setup.py @@ -399,6 +399,9 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve "wheel", "yamllint", "aioresponses", +] + +aiobotocore = [ # This required for AWS deferrable operators. # There is conflict between boto3 and aiobotocore dependency botocore. # TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or @@ -421,6 +424,7 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]): devel = get_unique_dependency_list( [ + aiobotocore, cgroups, devel_only, doc, @@ -458,6 +462,7 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]): # Those are extras that are extensions of the 'core' Airflow. They provide additional features # To airflow core. They do not have separate providers because they do not have any operators/hooks etc. CORE_EXTRAS_DEPENDENCIES: dict[str, list[str]] = { + "aiobotocore": aiobotocore, "async": async_packages, "celery": celery, "cgroups": cgroups,