diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aaa2231562da4..e64c31465edd1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -376,7 +376,7 @@ repos: name: Update cross-dependencies for providers packages entry: ./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py language: python - files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|$airflow/providers/.*/provider.yaml$ + files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|^airflow/providers/.*/provider.yaml$ pass_filenames: false additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml'] - id: update-extras diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index e87a117350734..243aaa19e6278 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -314,7 +314,21 @@ def get_install_requirements(provider_package_id: str, version_suffix: str) -> s :return: install requirements of the package """ - install_requires = ALL_DEPENDENCIES[provider_package_id][DEPS] + + def apply_version_suffix(install_clause: str) -> str: + if install_clause.startswith("apache-airflow") and ">=" in install_clause and version_suffix != "": + # This is workaround for `pip` way of handling `--pre` installation switch. It apparently does + # not modify the meaning of `install_requires` to include also pre-releases, so we need to + # modify our internal provider and airflow package version references to include all pre-releases + # including all development releases. When you specify dependency as >= X.Y.Z, and you + # have packages X.Y.Zdev0 or X.Y.Zrc1 in a local file, such package is not considered + # as fulfilling the requirement even if `--pre` switch is used. + return install_clause + ".*" + return install_clause + + install_requires = [ + apply_version_suffix(clause) for clause in ALL_DEPENDENCIES[provider_package_id][DEPS] + ] prefix = "\n " return prefix + prefix.join(install_requires) diff --git a/setup.py b/setup.py index 744934e5915d4..c591578935a8e 100644 --- a/setup.py +++ b/setup.py @@ -622,12 +622,27 @@ def is_package_excluded(package: str, exclusion_list: List[str]) -> bool: return any(package.startswith(excluded_package) for excluded_package in exclusion_list) +def remove_provider_limits(package: str) -> str: + """ + Removes the limit for providers in devel_all to account for pre-release and development packages. + + :param package: package name (beginning of it) + :return: true if package should be excluded + """ + return ( + package.split(">=")[0] + if package.startswith("apache-airflow-providers") and ">=" in package + else package + ) + + +devel = [remove_provider_limits(package) for package in devel] devel_all = [ - package + remove_provider_limits(package) for package in devel_all if not is_package_excluded(package=package, exclusion_list=PACKAGES_EXCLUDED_FOR_ALL) ] - +devel_hadoop = [remove_provider_limits(package) for package in devel_hadoop] devel_ci = devel_all