diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 23e99e2407..3e0c8b23e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: hooks: - id: pyupgrade args: - - --py37-plus + - --py310-plus - --keep-runtime-typing - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.14.7 diff --git a/cosmos/_triggers/watcher.py b/cosmos/_triggers/watcher.py index 891015c58e..64236b53bb 100644 --- a/cosmos/_triggers/watcher.py +++ b/cosmos/_triggers/watcher.py @@ -4,7 +4,8 @@ import base64 import json import zlib -from typing import Any, AsyncIterator +from collections.abc import AsyncIterator +from typing import Any from airflow.triggers.base import BaseTrigger, TriggerEvent from asgiref.sync import sync_to_async diff --git a/cosmos/_utils/importer.py b/cosmos/_utils/importer.py index 0f8f2b9b2d..7716a17357 100644 --- a/cosmos/_utils/importer.py +++ b/cosmos/_utils/importer.py @@ -1,5 +1,6 @@ import importlib -from typing import Any, Callable +from collections.abc import Callable +from typing import Any def load_method_from_module(module_path: str, method_name: str) -> Callable[..., Any]: diff --git a/cosmos/airflow/graph.py b/cosmos/airflow/graph.py index b8a6507350..95c9d242e4 100644 --- a/cosmos/airflow/graph.py +++ b/cosmos/airflow/graph.py @@ -1,8 +1,9 @@ from __future__ import annotations from collections import OrderedDict, defaultdict +from collections.abc import Callable from copy import deepcopy -from typing import Any, Callable +from typing import Any try: # Airflow 3 from airflow.sdk.bases.operator import BaseOperator diff --git a/cosmos/config.py b/cosmos/config.py index c849d7745f..3d46ccb9b1 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -6,9 +6,10 @@ import shutil import tempfile import warnings +from collections.abc import Callable, Iterator from dataclasses import InitVar, dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterator +from typing import TYPE_CHECKING, Any import yaml from airflow.version import version as airflow_version diff --git a/cosmos/constants.py b/cosmos/constants.py index bdf38ebf22..65378aefcb 100644 --- a/cosmos/constants.py +++ b/cosmos/constants.py @@ -1,7 +1,7 @@ import os +from collections.abc import Callable from enum import Enum from pathlib import Path -from typing import Callable import aenum import airflow diff --git a/cosmos/converter.py b/cosmos/converter.py index b56c91c668..854d4f2710 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -8,7 +8,8 @@ import os import platform import time -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from warnings import warn from airflow.models.dag import DAG diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py index 8da6eeb8d1..8d965bba0b 100644 --- a/cosmos/dbt/project.py +++ b/cosmos/dbt/project.py @@ -2,9 +2,9 @@ import os import shutil +from collections.abc import Generator from contextlib import contextmanager from pathlib import Path -from typing import Generator import yaml diff --git a/cosmos/dbt/runner.py b/cosmos/dbt/runner.py index 00bc7c24e9..345701ff31 100644 --- a/cosmos/dbt/runner.py +++ b/cosmos/dbt/runner.py @@ -1,8 +1,9 @@ from __future__ import annotations import sys +from collections.abc import Callable from functools import lru_cache -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any from cosmos.dbt.project import change_working_directory, environ from cosmos.exceptions import CosmosDbtRunError diff --git a/cosmos/hooks/subprocess.py b/cosmos/hooks/subprocess.py index 1ee47cd267..e2a82e19be 100644 --- a/cosmos/hooks/subprocess.py +++ b/cosmos/hooks/subprocess.py @@ -7,9 +7,10 @@ import contextlib import os import signal +from collections.abc import Callable from subprocess import PIPE, STDOUT, Popen from tempfile import TemporaryDirectory, gettempdir -from typing import Any, Callable, NamedTuple +from typing import Any, NamedTuple try: # Airflow 3.1 onwards diff --git a/cosmos/operators/_asynchronous/bigquery.py b/cosmos/operators/_asynchronous/bigquery.py index 6ec48b1a2a..5fd33c4fde 100644 --- a/cosmos/operators/_asynchronous/bigquery.py +++ b/cosmos/operators/_asynchronous/bigquery.py @@ -3,8 +3,9 @@ import base64 import time import zlib +from collections.abc import Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any from cosmos.operators.base import _sanitize_xcom_key diff --git a/cosmos/operators/_watcher/state.py b/cosmos/operators/_watcher/state.py index 534b81aa36..e565c81bf3 100644 --- a/cosmos/operators/_watcher/state.py +++ b/cosmos/operators/_watcher/state.py @@ -1,8 +1,9 @@ from __future__ import annotations import logging +from collections.abc import Callable from threading import Lock -from typing import Any, Callable +from typing import Any try: from airflow.sdk.types import RuntimeTaskInstanceProtocol as TaskInstance diff --git a/cosmos/operators/aws_ecs.py b/cosmos/operators/aws_ecs.py index 51ff474102..4927e828ce 100644 --- a/cosmos/operators/aws_ecs.py +++ b/cosmos/operators/aws_ecs.py @@ -1,7 +1,8 @@ from __future__ import annotations import inspect -from typing import TYPE_CHECKING, Any, Callable, Sequence +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: # pragma: no cover try: diff --git a/cosmos/operators/aws_eks.py b/cosmos/operators/aws_eks.py index f48833120e..a576f0ea68 100644 --- a/cosmos/operators/aws_eks.py +++ b/cosmos/operators/aws_eks.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Sequence +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.eks import EksHook diff --git a/cosmos/operators/azure_container_instance.py b/cosmos/operators/azure_container_instance.py index ea5348a663..463f942a58 100644 --- a/cosmos/operators/azure_container_instance.py +++ b/cosmos/operators/azure_container_instance.py @@ -1,7 +1,8 @@ from __future__ import annotations import inspect -from typing import TYPE_CHECKING, Any, Callable, Sequence +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any try: from airflow.sdk.bases.operator import BaseOperator # Airflow 3 diff --git a/cosmos/operators/base.py b/cosmos/operators/base.py index cb409482bf..7ebab3ccf8 100644 --- a/cosmos/operators/base.py +++ b/cosmos/operators/base.py @@ -4,8 +4,9 @@ import logging import os from abc import ABCMeta, abstractmethod +from collections.abc import Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any import yaml from airflow.utils.context import context_merge diff --git a/cosmos/operators/docker.py b/cosmos/operators/docker.py index f41529a213..9af92e472c 100644 --- a/cosmos/operators/docker.py +++ b/cosmos/operators/docker.py @@ -1,7 +1,8 @@ from __future__ import annotations import inspect -from typing import TYPE_CHECKING, Any, Callable, Sequence +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any try: from airflow.sdk.bases.operator import BaseOperator # Airflow 3 diff --git a/cosmos/operators/gcp_cloud_run_job.py b/cosmos/operators/gcp_cloud_run_job.py index cd2936b9a9..db915ad6a2 100644 --- a/cosmos/operators/gcp_cloud_run_job.py +++ b/cosmos/operators/gcp_cloud_run_job.py @@ -1,7 +1,8 @@ from __future__ import annotations import inspect -from typing import TYPE_CHECKING, Any, Callable, Sequence +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: # pragma: no cover try: diff --git a/cosmos/operators/kubernetes.py b/cosmos/operators/kubernetes.py index 1eeb01d2bd..2e3d04bd66 100644 --- a/cosmos/operators/kubernetes.py +++ b/cosmos/operators/kubernetes.py @@ -3,8 +3,9 @@ import inspect import re from abc import ABC +from collections.abc import Callable, Sequence from os import PathLike -from typing import TYPE_CHECKING, Any, Callable, Sequence +from typing import TYPE_CHECKING, Any import kubernetes.client as k8s from airflow.providers.cncf.kubernetes.backcompat.backwards_compat_converters import ( diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 90c7fa769b..d8c0a90260 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -10,9 +10,10 @@ import warnings import zlib from abc import ABC, abstractmethod +from collections.abc import Callable, Sequence from functools import cached_property from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Literal, Sequence +from typing import TYPE_CHECKING, Any, Literal from urllib.parse import urlparse import jinja2 diff --git a/cosmos/operators/virtualenv.py b/cosmos/operators/virtualenv.py index 6b0e3785dd..c41f9d3773 100644 --- a/cosmos/operators/virtualenv.py +++ b/cosmos/operators/virtualenv.py @@ -3,9 +3,10 @@ import os import shutil import time +from collections.abc import Callable, Sequence from pathlib import Path from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING, Any, Callable, Sequence +from typing import TYPE_CHECKING, Any import psutil diff --git a/cosmos/operators/watcher.py b/cosmos/operators/watcher.py index 0489da0c17..f8a3c1b5ca 100644 --- a/cosmos/operators/watcher.py +++ b/cosmos/operators/watcher.py @@ -4,9 +4,10 @@ import json import logging import zlib +from collections.abc import Callable from datetime import timedelta from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any from cosmos._triggers.watcher import WatcherTrigger, _parse_compressed_xcom from cosmos.operators._watcher.state import get_xcom_val, safe_xcom_push diff --git a/cosmos/plugin/airflow3.py b/cosmos/plugin/airflow3.py index d3659455c7..3fe73f58e4 100644 --- a/cosmos/plugin/airflow3.py +++ b/cosmos/plugin/airflow3.py @@ -5,8 +5,9 @@ import logging import os import os.path as op +from collections.abc import Generator from contextlib import contextmanager -from typing import Any, Generator +from typing import Any from unittest.mock import patch from urllib.parse import urlsplit diff --git a/cosmos/profiles/__init__.py b/cosmos/profiles/__init__.py index a635f974ef..ecf69e7ba0 100644 --- a/cosmos/profiles/__init__.py +++ b/cosmos/profiles/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Type +from typing import Any from .athena import AthenaAccessKeyProfileMapping from .base import BaseProfileMapping, DbtProfileConfigVars @@ -30,7 +30,7 @@ from .trino.ldap import TrinoLDAPProfileMapping from .vertica.user_pass import VerticaUserPasswordProfileMapping -profile_mappings: list[Type[BaseProfileMapping]] = [ +profile_mappings: list[type[BaseProfileMapping]] = [ AthenaAccessKeyProfileMapping, ClickhouseUserPasswordProfileMapping, GoogleCloudServiceAccountFileProfileMapping, diff --git a/docs/generate_mappings.py b/docs/generate_mappings.py index 049eb8b4e1..f04f776478 100644 --- a/docs/generate_mappings.py +++ b/docs/generate_mappings.py @@ -6,7 +6,6 @@ import os from dataclasses import dataclass -from typing import Type from jinja2 import Environment, FileSystemLoader @@ -22,7 +21,7 @@ class Field: airflow_name: str | list[str] | None = None -def get_fields_from_mapping(mapping: Type[BaseProfileMapping]) -> list[Field]: +def get_fields_from_mapping(mapping: type[BaseProfileMapping]) -> list[Field]: """ Generates Field objects from a profile mapping. """ diff --git a/tests/dbt/parser/test_project.py b/tests/dbt/parser/test_project.py index f17b4b0e59..6a52de135f 100644 --- a/tests/dbt/parser/test_project.py +++ b/tests/dbt/parser/test_project.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import List from unittest.mock import patch import pytest @@ -155,7 +154,7 @@ class KeywordArgValueStr(KeywordArgValue, str): @dataclass class KeywordArg: key: str - value: KeywordArgValue | List + value: KeywordArgValue | list def test_dbtmodelconfig_extract_config_non_kwarg(): diff --git a/tests/perf/test_performance.py b/tests/perf/test_performance.py index 48cabfc74e..ec6096fb60 100644 --- a/tests/perf/test_performance.py +++ b/tests/perf/test_performance.py @@ -2,10 +2,10 @@ import os import time +from collections.abc import Generator from contextlib import contextmanager from functools import cache from pathlib import Path -from typing import Generator import pytest from airflow.models.dagbag import DagBag