From 5605182d105bf01fc3a585581655f42ed0b712ed Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Tue, 26 Jul 2022 13:29:27 +0200
Subject: [PATCH] Adjust limits when constructing cross-provider dependencies

When we are constructing cross-provider dependencies, we should adjust
the limits to account for some apache-airflow packages that are not yet
released.

The adjustment is to add ".*" after the version number when
dependency is lower-bound with `>=`. It's not explicitly mentioned in
PEP 440 - it is only mentioned there that it works for
equality, but since `>=` is also part of `equal` it also
works there.

Example is a common-sql package that might be limited to `>=1.1.0`
in google provider as part of the change, but it might not yet be released
as it is being released together with the package it is needed by.

We  need to adjust the version whenever in our providers we
refer to other providers with >= install clause because  `--pre`
flag in `pip` only allows to install direct pre-release (and
development) dependencies but it does not modify requirements of
those package to also include pre-releases as transitive dependency.

Also, we need to remove the limits in `devel` dependencies because
the packages are not yet released at the moment we use those
dependencies, so the limits in `devel` should be removed, allowing
the developers to install Airflow without those devel packages.

Also a bug was found that would prevent to generate
the dependencies in case provider.yaml file only changed (bad
specification of .pre-commit include)
---
 .pre-commit-config.yaml                       |  2 +-
 .../prepare_provider_packages.py              | 16 +++++++++++++++-
 setup.py                                      | 19 +++++++++++++++++--
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index aaa2231562da4..e64c31465edd1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -376,7 +376,7 @@ repos:
         name: Update cross-dependencies for providers packages
         entry: ./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
         language: python
-        files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|$airflow/providers/.*/provider.yaml$
+        files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|^airflow/providers/.*/provider.yaml$
         pass_filenames: false
         additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml']
       - id: update-extras
diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index e87a117350734..243aaa19e6278 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -314,7 +314,21 @@ def get_install_requirements(provider_package_id: str, version_suffix: str) -> s
 
     :return: install requirements of the package
     """
-    install_requires = ALL_DEPENDENCIES[provider_package_id][DEPS]
+
+    def apply_version_suffix(install_clause: str) -> str:
+        if install_clause.startswith("apache-airflow") and ">=" in install_clause and version_suffix != "":
+            # This is workaround for `pip` way of handling `--pre` installation switch. It apparently does
+            # not modify the meaning of `install_requires` to include also pre-releases, so we need to
+            # modify our internal provider and airflow package version references to include all pre-releases
+            # including all development releases. When you specify dependency as >= X.Y.Z, and you
+            # have packages X.Y.Zdev0 or X.Y.Zrc1 in a local file, such package is not considered
+            # as fulfilling the requirement even if `--pre` switch is used.
+            return install_clause + ".*"
+        return install_clause
+
+    install_requires = [
+        apply_version_suffix(clause) for clause in ALL_DEPENDENCIES[provider_package_id][DEPS]
+    ]
     prefix = "\n    "
     return prefix + prefix.join(install_requires)
 
diff --git a/setup.py b/setup.py
index 744934e5915d4..c591578935a8e 100644
--- a/setup.py
+++ b/setup.py
@@ -622,12 +622,27 @@ def is_package_excluded(package: str, exclusion_list: List[str]) -> bool:
     return any(package.startswith(excluded_package) for excluded_package in exclusion_list)
 
 
+def remove_provider_limits(package: str) -> str:
+    """
+    Removes the limit for providers in devel_all to account for pre-release and development packages.
+
+    :param package: package name (beginning of it)
+    :return: true if package should be excluded
+    """
+    return (
+        package.split(">=")[0]
+        if package.startswith("apache-airflow-providers") and ">=" in package
+        else package
+    )
+
+
+devel = [remove_provider_limits(package) for package in devel]
 devel_all = [
-    package
+    remove_provider_limits(package)
     for package in devel_all
     if not is_package_excluded(package=package, exclusion_list=PACKAGES_EXCLUDED_FOR_ALL)
 ]
-
+devel_hadoop = [remove_provider_limits(package) for package in devel_hadoop]
 devel_ci = devel_all