Skip to content

Fix dataset tests failing in Airflow 3 #1716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tatiana merged 14 commits into from
Apr 29, 2025
Merged

Fix dataset tests failing in Airflow 3 #1716

Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
459601d
Fix AF3 generate_run_id() takes 0 positional arguments
tatiana Apr 29, 2025
640bc68
Restablish conn_file_path argument
tatiana Apr 29, 2025
824b183
Add k8s test that had the same issue
tatiana Apr 29, 2025
f8a610d
Fix unpacking issue
tatiana Apr 29, 2025
39473d1
Fix unpacking issue
tatiana Apr 29, 2025
b486dfc
Fix unraised exception in test
tatiana Apr 29, 2025
e77c8de
Ensure virtualenv_dir path exists (#1724)
pankajkoti Apr 29, 2025
e8df092
Fix dataset test failing in Airflow 3
tatiana Apr 28, 2025
d7cfedc
Fix dataset-related AF3 error
tatiana Apr 28, 2025
5d016d0
Fix test import
tatiana Apr 28, 2025
fda1bcd
🎨 [pre-commit.ci] Auto format from pre-commit.com hooks
pre-commit-ci[bot] Apr 29, 2025
7f1a17f
Merge branch 'main' into issue-1704
tatiana Apr 29, 2025
3807041
🎨 [pre-commit.ci] Auto format from pre-commit.com hooks
pre-commit-ci[bot] Apr 29, 2025
3168623
Disable test that is being solve din another PR
tatiana Apr 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions tests/operators/test_local.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,8 +469,11 @@ def test_run_operator_dataset_inlets_and_outlets(caplog):
reason="From Airflow 2.10 onwards, we started using DatasetAlias, which changed this behaviour.",
)
@pytest.mark.integration
def test_run_operator_dataset_inlets_and_outlets_airflow_210_onwards(caplog):
from airflow.models.dataset import DatasetAliasModel
def test_run_operator_dataset_inlets_and_outlets_airflow_210(caplog):
try:
from airflow.models.asset import AssetAliasModel
except ModuleNotFoundError:
from airflow.models.dataset import DatasetAliasModel as AssetAliasModel
Comment thread
tatiana marked this conversation as resolved.

with DAG("test_id_1", start_date=datetime(2022, 1, 1)) as dag:
seed_operator = DbtSeedLocalOperator(
Expand Down Expand Up @@ -504,8 +507,8 @@ def test_run_operator_dataset_inlets_and_outlets_airflow_210_onwards(caplog):
seed_operator >> run_operator >> test_operator

assert seed_operator.outlets == [] # because emit_datasets=False,
assert run_operator.outlets == [DatasetAliasModel(name="test_id_1__run")]
assert test_operator.outlets == [DatasetAliasModel(name="test_id_1__test")]
assert run_operator.outlets == [AssetAliasModel(name="test_id_1__run")]
assert test_operator.outlets == [AssetAliasModel(name="test_id_1__test")]


@patch("cosmos.settings.enable_dataset_alias", 0)
Expand Down Expand Up @@ -564,7 +567,7 @@ def test_run_operator_dataset_emission_is_skipped(caplog):
assert run_operator.outlets == []


# TODO: Make test compatible with Airflow 3.0. Issue:https://github.com/astronomer/astronomer-cosmos/issues/1705
# TODO: Make test compatible with Airflow 3.0. Issue:https://github.com/astronomer/astronomer-cosmos/issues/1713
@pytest.mark.skipif(version.parse(airflow_version).major == 3, reason="Test need to be updated for Airflow 3.0")
@pytest.mark.skipif(
version.parse(airflow_version) < version.parse("2.4")
Expand All @@ -574,7 +577,10 @@ def test_run_operator_dataset_emission_is_skipped(caplog):
@pytest.mark.integration
@patch("cosmos.settings.enable_dataset_alias", 0)
def test_run_operator_dataset_url_encoded_names(caplog):
from airflow.datasets import Dataset
try:
from airflow.sdk.definitions.asset import Dataset
except ImportError:
from airflow.datasets import Dataset

with DAG("test-id-1", start_date=datetime(2022, 1, 1)) as dag:
run_operator = DbtRunLocalOperator(
Expand Down