Adjust limits when constructing cross-provider dependencies

.pre-commit-config.yaml

@@ -376,7 +376,7 @@ repos:
         name: Update cross-dependencies for providers packages
         entry: ./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
         language: python
-        files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|$airflow/providers/.*/provider.yaml$
+        files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|^airflow/providers/.*/provider.yaml$
         pass_filenames: false
         additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml']
       - id: update-extras

dev/provider_packages/prepare_provider_packages.py

@@ -314,7 +314,21 @@ def get_install_requirements(provider_package_id: str, version_suffix: str) -> s
 
     :return: install requirements of the package
     """
-    install_requires = ALL_DEPENDENCIES[provider_package_id][DEPS]
+
+    def apply_version_suffix(install_clause: str) -> str:
+        if install_clause.startswith("apache-airflow") and ">=" in install_clause and version_suffix != "":
+            # This is workaround for `pip` way of handling `--pre` installation switch. It apparently does
+            # not modify the meaning of `install_requires` to include also pre-releases, so we need to
+            # modify our internal provider and airflow package version references to include all pre-releases
+            # including all development releases. When you specify dependency as >= X.Y.Z, and you
+            # have packages X.Y.Zdev0 or X.Y.Zrc1 in a local file, such package is not considered
+            # as fulfilling the requirement even if `--pre` switch is used.
+            return install_clause + ".*"
+        return install_clause
+
+    install_requires = [
+        apply_version_suffix(clause) for clause in ALL_DEPENDENCIES[provider_package_id][DEPS]
+    ]
     prefix = "\n    "
     return prefix + prefix.join(install_requires)
 

setup.py

@@ -622,12 +622,27 @@ def is_package_excluded(package: str, exclusion_list: List[str]) -> bool:
     return any(package.startswith(excluded_package) for excluded_package in exclusion_list)
 
 
+def remove_provider_limits(package: str) -> str:
+    """
+    Removes the limit for providers in devel_all to account for pre-release and development packages.
+
+    :param package: package name (beginning of it)
+    :return: true if package should be excluded
+    """
+    return (
+        package.split(">=")[0]
+        if package.startswith("apache-airflow-providers") and ">=" in package
+        else package
+    )
+
+
+devel = [remove_provider_limits(package) for package in devel]
 devel_all = [
-    package
+    remove_provider_limits(package)
     for package in devel_all
     if not is_package_excluded(package=package, exclusion_list=PACKAGES_EXCLUDED_FOR_ALL)
 ]
-
+devel_hadoop = [remove_provider_limits(package) for package in devel_hadoop]
 devel_ci = devel_all