From 5d96f08402006ac697c79eb198f3e100d1004bce Mon Sep 17 00:00:00 2001 From: pankajastro Date: Fri, 25 Apr 2025 13:59:44 +0530 Subject: [PATCH 1/4] Enable running setup/teardown task --- scripts/airflow3/env.sh | 8 ++++++-- scripts/airflow3/requirements.txt | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/airflow3/env.sh b/scripts/airflow3/env.sh index ded745c13d..5da800387f 100644 --- a/scripts/airflow3/env.sh +++ b/scripts/airflow3/env.sh @@ -20,5 +20,9 @@ export AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT=300 export AIRFLOW_REPO_DIR="$PWD/../airflow" export AIRFLOW_CONN_EXAMPLE_CONN=postgresql+psycopg2://postgres:postgres@localhost:5432/airflow_db # Until we fix the ExecutionMode.VIRTUALENV https://github.com/astronomer/astronomer-cosmos/issues/1633 -export AIRFLOW__COSMOS__ENABLE_SETUP_ASYNC_TASK=False -export AIRFLOW__COSMOS__ENABLE_TEARDOWN_ASYNC_TASK=False +export AIRFLOW__COSMOS__ENABLE_SETUP_ASYNC_TASK=True +export AIRFLOW__COSMOS__ENABLE_TEARDOWN_ASYNC_TASK=True +export AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID=gcp_gs_conn +export AIRFLOW__COSMOS__REMOTE_TARGET_PATH=gs://cosmos_remote_target +#export AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID=s3_conn +#export AIRFLOW__COSMOS__REMOTE_TARGET_PATH="s3://cosmos-remote-target" diff --git a/scripts/airflow3/requirements.txt b/scripts/airflow3/requirements.txt index 21dbaf8e80..5fd12753be 100644 --- a/scripts/airflow3/requirements.txt +++ b/scripts/airflow3/requirements.txt @@ -3,8 +3,9 @@ apache-airflow-task-sdk apache-airflow-providers-standard apache-airflow-providers-fab apache-airflow-providers-cncf-kubernetes +# apache-airflow-providers-amazon[s3fs] +apache-airflow-providers-google psycopg2 asyncpg dbt-postgres dbt-bigquery -apache-airflow-providers-google From cf559839d8acede6265be60b7ad7392fcfc5c503 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Fri, 25 Apr 2025 16:17:18 +0530 Subject: [PATCH 2/4] Update scripts/airflow3/requirements.txt --- scripts/airflow3/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/airflow3/requirements.txt b/scripts/airflow3/requirements.txt index 5fd12753be..194c931069 100644 --- a/scripts/airflow3/requirements.txt +++ b/scripts/airflow3/requirements.txt @@ -3,7 +3,9 @@ apache-airflow-task-sdk apache-airflow-providers-standard apache-airflow-providers-fab apache-airflow-providers-cncf-kubernetes -# apache-airflow-providers-amazon[s3fs] +# Installing apache-airflow-providers-amazon[s3fs] and apache-airflow-providers-google together causes a +# dependency resolution conflict for package fsspec. Please install this manually as an independent pip command # if you want to keep remote storage as S3. +apache-airflow-providers-amazon[s3fs] apache-airflow-providers-google psycopg2 asyncpg From 7e1a8e9649252253da317c9d19677035e9968789 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 25 Apr 2025 10:47:41 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20for?= =?UTF-8?q?mat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/airflow3/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/airflow3/requirements.txt b/scripts/airflow3/requirements.txt index 194c931069..19dbbceeac 100644 --- a/scripts/airflow3/requirements.txt +++ b/scripts/airflow3/requirements.txt @@ -3,7 +3,7 @@ apache-airflow-task-sdk apache-airflow-providers-standard apache-airflow-providers-fab apache-airflow-providers-cncf-kubernetes -# Installing apache-airflow-providers-amazon[s3fs] and apache-airflow-providers-google together causes a +# Installing apache-airflow-providers-amazon[s3fs] and apache-airflow-providers-google together causes a # dependency resolution conflict for package fsspec. Please install this manually as an independent pip command # if you want to keep remote storage as S3. apache-airflow-providers-amazon[s3fs] apache-airflow-providers-google From 4dd3cf2d13c586b05449c749f524d49fc8e9dafc Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Fri, 25 Apr 2025 16:17:53 +0530 Subject: [PATCH 4/4] Update scripts/airflow3/requirements.txt --- scripts/airflow3/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/airflow3/requirements.txt b/scripts/airflow3/requirements.txt index 19dbbceeac..4c9f3c962b 100644 --- a/scripts/airflow3/requirements.txt +++ b/scripts/airflow3/requirements.txt @@ -4,7 +4,8 @@ apache-airflow-providers-standard apache-airflow-providers-fab apache-airflow-providers-cncf-kubernetes # Installing apache-airflow-providers-amazon[s3fs] and apache-airflow-providers-google together causes a -# dependency resolution conflict for package fsspec. Please install this manually as an independent pip command # if you want to keep remote storage as S3. +# dependency resolution conflict for package fsspec. Please install this manually as an independent pip command +# if you want to keep remote storage as S3. apache-airflow-providers-amazon[s3fs] apache-airflow-providers-google psycopg2