Run separate CI job for latest boto/botocore version when needed

.github/workflows/ci.yml

@@ -151,6 +151,7 @@ jobs:
       providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
       skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
       run-tests: ${{ steps.selective-checks.outputs.run-tests }}
+      run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
       run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
       run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
       basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
@@ -906,6 +907,50 @@ jobs:
       - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
         uses: ./.github/actions/post_tests
 
+  tests-postgres-boto:
+    timeout-minutes: 130
+    name: >
+      LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
+      Py${{needs.build-info.outputs.default-python-version}}:
+      ${{needs.build-info.outputs.test-types}}
+    runs-on: "${{needs.build-info.outputs.runs-on}}"
+    needs: [build-info, test-pytest-collection]
+    env:
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
+      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
+      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
+      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
+      BACKEND: "postgres"
+      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+      PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
+      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
+      BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
+      UPGRADE_BOTO: "true"
+      JOB_ID: >
+        postgres-${{needs.build-info.outputs.default-python-version}}-
+        ${{needs.build-info.outputs.default-postgres-version}}
+      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
+    steps:
+      - name: Cleanup repo
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: >
+          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/prepare_breeze_and_image
+      - name: >
+          Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
+        run: breeze testing tests --run-in-parallel
+      - name: >
+          Post Tests: ${{needs.build-info.outputs.default-python-version}}:
+          ${{needs.build-info.outputs.test-types}}
+        uses: ./.github/actions/post_tests
+
   tests-mysql:
     timeout-minutes: 130
     name: >

Dockerfile.ci

@@ -1016,6 +1016,13 @@ else
         exit 1
     fi
 fi
+if [[ ${UPGRADE_BOTO=} == "true" ]]; then
+    echo
+    echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
+    echo
+    pip uninstall aiobotocore -y || true
+    pip install --upgrade boto3 botocore
+fi
 readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
     ALL_TESTS ALL_PRESELECTED_TESTS
 

dev/breeze/src/airflow_breeze/commands/testing_commands.py

@@ -357,6 +357,12 @@ def run_tests_in_parallel(
     is_flag=True,
     envvar="FULL_TESTS_NEEDED",
 )
+@click.option(
+    "--upgrade-boto",
+    help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
+    is_flag=True,
+    envvar="UPGRADE_BOTO",
+)
 @option_verbose
 @option_dry_run
 @click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
@@ -380,6 +386,7 @@ def tests(
     full_tests_needed: bool,
     mount_sources: str,
     extra_pytest_args: tuple,
+    upgrade_boto: bool,
 ):
     docker_filesystem = get_filesystem_type("/var/lib/docker")
     get_console().print(f"Docker filesystem: {docker_filesystem}")
@@ -394,6 +401,7 @@ def tests(
         mount_sources=mount_sources,
         forward_ports=False,
         test_type=test_type,
+        upgrade_boto=upgrade_boto,
     )
     rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
     cleanup_python_generated_files()

dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

@@ -53,6 +53,7 @@
             "options": [
                 "--image-tag",
                 "--mount-sources",
+                "--upgrade-boto",
             ],
         },
     ],

dev/breeze/src/airflow_breeze/params/shell_params.py

@@ -112,6 +112,7 @@ class ShellParams:
     version_suffix_for_pypi: str = ""
     dry_run: bool = False
     verbose: bool = False
+    upgrade_boto: bool = False
 
     def clone_with_test(self, test_type: str) -> ShellParams:
         new_params = deepcopy(self)

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
     set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
     set_value_to_default_if_not_set(env, "TEST_TYPE", "")
     set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
+    set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
     set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
     set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
     set_value_to_default_if_not_set(env, "VERBOSE", "false")
@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
     "SKIP_PROVIDER_TESTS": "skip_provider_tests",
     "SQLITE_URL": "sqlite_url",
     "START_AIRFLOW": "start_airflow",
+    "UPGRADE_BOTO": "upgrade_boto",
     "USE_AIRFLOW_VERSION": "use_airflow_version",
     "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
     "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -154,14 +154,14 @@ def __hash__(self):
             r"\.py$",
         ],
         FileGroupForCi.ALL_SOURCE_FILES: [
-            "^.pre-commit-config.yaml$",
-            "^airflow",
-            "^chart",
-            "^tests",
-            "^kubernetes_tests",
+            r"^.pre-commit-config.yaml$",
+            r"^airflow",
+            r"^chart",
+            r"^tests",
+            r"^kubernetes_tests",
         ],
         FileGroupForCi.SYSTEM_TEST_FILES: [
-            "^tests/system/",
+            r"^tests/system/",
         ],
     }
 )
@@ -180,10 +180,11 @@ def __hash__(self):
             r"^tests/cli",
         ],
         SelectiveUnitTestTypes.PROVIDERS: [
-            "^airflow/providers/",
-            "^tests/providers/",
+            r"^airflow/providers/",
+            r"^tests/system/providers/",
+            r"^tests/providers/",
         ],
-        SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
+        SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
     }
 )
 
@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
     def run_www_tests(self) -> bool:
         return self._should_be_run(FileGroupForCi.WWW_FILES)
 
+    @cached_property
+    def run_amazon_tests(self) -> bool:
+        return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")
+
     @cached_property
     def run_kubernetes_tests(self) -> bool:
         return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)

dev/breeze/tests/test_selective_checks.py

@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "false",
                     "needs-helm-tests": "false",
                     "run-tests": "false",
+                    "run-amazon-tests": "false",
                     "docs-build": "false",
                     "upgrade-to-newer-dependencies": "false",
                     "test-types": "",
@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "false",
                     "run-tests": "true",
+                    "run-amazon-tests": "false",
                     "docs-build": "true",
                     "upgrade-to-newer-dependencies": "false",
                     "test-types": "API Always",
@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "false",
                     "run-tests": "true",
+                    "run-amazon-tests": "true",
                     "docs-build": "true",
                     "upgrade-to-newer-dependencies": "false",
                     "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "false",
                     "run-tests": "true",
+                    "run-amazon-tests": "false",
                     "docs-build": "false",
                     "run-kubernetes-tests": "false",
                     "upgrade-to-newer-dependencies": "false",
@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "false",
                     "run-tests": "false",
+                    "run-amazon-tests": "false",
                     "docs-build": "true",
                     "run-kubernetes-tests": "false",
                     "upgrade-to-newer-dependencies": "false",
@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "true",
                     "run-tests": "true",
+                    "run-amazon-tests": "true",
                     "docs-build": "true",
                     "run-kubernetes-tests": "true",
                     "upgrade-to-newer-dependencies": "false",
@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "true",
                     "run-tests": "true",
+                    "run-amazon-tests": "false",
                     "docs-build": "true",
                     "run-kubernetes-tests": "true",
                     "upgrade-to-newer-dependencies": "false",
@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "true",
                     "run-tests": "true",
+                    "run-amazon-tests": "false",
                     "docs-build": "true",
                     "run-kubernetes-tests": "true",
                     "upgrade-to-newer-dependencies": "false",
@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "needs-helm-tests": "true",
                     "run-tests": "true",
                     "docs-build": "true",
+                    "run-amazon-tests": "false",
                     "run-kubernetes-tests": "true",
                     "upgrade-to-newer-dependencies": "false",
                     "test-types": "Always",
@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "true",
                     "run-tests": "true",
+                    "run-amazon-tests": "true",
                     "docs-build": "true",
                     "upgrade-to-newer-dependencies": "true",
                     "test-types": "API Always CLI Core Other Providers WWW",
@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "image-build": "true",
                     "needs-helm-tests": "true",
                     "run-tests": "true",
+                    "run-amazon-tests": "true",
                     "docs-build": "true",
                     "upgrade-to-newer-dependencies": "true",
                     "test-types": "API Always CLI Core Other Providers WWW",
                 },
                 id="Everything should run and upgrading to newer requirements as dependencies change",
             )
         ),
+        pytest.param(
+            ("airflow/providers/amazon/__init__.py",),
+            {
+                "all-python-versions": "['3.7']",
+                "all-python-versions-list-as-string": "3.7",
+                "python-versions": "['3.7']",
+                "python-versions-list-as-string": "3.7",
+                "image-build": "true",
+                "needs-helm-tests": "false",
+                "run-tests": "true",
+                "docs-build": "true",
+                "run-kubernetes-tests": "false",
+                "upgrade-to-newer-dependencies": "false",
+                "run-amazon-tests": "true",
+                "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
+                "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
+            },
+            id="Providers tests run including amazon tests if amazon provider files changed",
+        ),
+        pytest.param(
+            ("tests/providers/airbyte/__init__.py",),
+            {
+                "all-python-versions": "['3.7']",
+                "all-python-versions-list-as-string": "3.7",
+                "python-versions": "['3.7']",
+                "python-versions-list-as-string": "3.7",
+                "image-build": "true",
+                "needs-helm-tests": "false",
+                "run-tests": "true",
+                "run-amazon-tests": "false",
+                "docs-build": "false",
+                "run-kubernetes-tests": "false",
+                "upgrade-to-newer-dependencies": "false",
+                "test-types": "Always Providers[airbyte,http]",
+            },
+            id="Providers tests run including amazon tests if amazon tests provider files changed",
+        ),
+        pytest.param(
+            ("airflow/providers/amazon/file.py",),
+            {
+                "all-python-versions": "['3.7']",
+                "all-python-versions-list-as-string": "3.7",
+                "python-versions": "['3.7']",
+                "python-versions-list-as-string": "3.7",
+                "image-build": "true",
+                "needs-helm-tests": "false",
+                "run-tests": "true",
+                "run-amazon-tests": "true",
+                "docs-build": "true",
+                "run-kubernetes-tests": "false",
+                "upgrade-to-newer-dependencies": "false",
+                "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
+                "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
+            },
+            id="Providers tests run without amazon tests if no amazon file changed",
+        ),
     ],
 )
 def test_expected_output_pull_request_main(

images/breeze/output-commands-hash.txt

@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
 testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
 testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
-testing:tests:ce6d80b3073bafd1f10c50812432885a
-testing:0f292a29b2a97fbd95778a72249ee997
+testing:tests:ddda3dc31e0002d67fad26b682ff689a
+testing:0b3792aed114983d6c9d12741561ec1c