From 02c07525a0c27e1267d00186f0dcbb5bec92b2e7 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 20 Aug 2024 22:24:51 +0200 Subject: [PATCH] Make latest botocore tests green The latest botocore tests are conflicting with a few requirements and until apache-beam upcoming version is released we need to do some manual exclusions. Those exclusions should make latest botocore test green again. --- Dockerfile.ci | 9 +++++---- pyproject.toml | 3 +++ scripts/docker/entrypoint_ci.sh | 9 +++++---- tests/always/test_example_dags.py | 20 ++++++------------- tests/providers/opensearch/conftest.py | 11 ++++++++-- .../opensearch/hooks/test_opensearch.py | 3 ++- .../opensearch/operators/test_opensearch.py | 3 +++ 7 files changed, 33 insertions(+), 25 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 2ebe82c6ef074..25151df8e983e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1013,16 +1013,17 @@ function check_boto_upgrade() { echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true # We need to include few dependencies to pass pip check with other dependencies: # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) - # * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider) + # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) # * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider) # * yandexcloud requirements for requests does not match those of apache.beam and latest botocore # Both requests and yandexcloud exclusion above might be removed after # https://github.com/apache/beam/issues/32080 is addressed - # When you remove yandexcloud from the above list, also remove it from "test_example_dags.py" - # in "tests/always". + # This is already addressed and planned for 2.59.0 release. + # When you remove yandexcloud and opensearch from the above list, you can also remove the + # optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always". set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ diff --git a/pyproject.toml b/pyproject.toml index 8562a137d2d35..3c09126751cc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -406,6 +406,9 @@ combine-as-imports = true "tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py" = ["E402"] "tests/providers/google/cloud/triggers/test_vertex_ai.py" = ["E402"] "tests/providers/openai/hooks/test_openai.py" = ["E402"] +"tests/providers/opensearch/conftest.py" = ["E402"] +"tests/providers/opensearch/hooks/test_opensearch.py" = ["E402"] +"tests/providers/opensearch/operators/test_opensearch.py" = ["E402"] "tests/providers/openai/operators/test_openai.py" = ["E402"] "tests/providers/qdrant/hooks/test_qdrant.py" = ["E402"] "tests/providers/qdrant/operators/test_qdrant.py" = ["E402"] diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 06b1437c2f8c9..b1d6f4cf708e4 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -250,16 +250,17 @@ function check_boto_upgrade() { echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true # We need to include few dependencies to pass pip check with other dependencies: # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) - # * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider) + # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) # * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider) # * yandexcloud requirements for requests does not match those of apache.beam and latest botocore # Both requests and yandexcloud exclusion above might be removed after # https://github.com/apache/beam/issues/32080 is addressed - # When you remove yandexcloud from the above list, also remove it from "test_example_dags.py" - # in "tests/always". + # This is already addressed and planned for 2.59.0 release. + # When you remove yandexcloud and opensearch from the above list, you can also remove the + # optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always". set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index f81bbf82caacd..b9e45647eca03 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -17,6 +17,7 @@ from __future__ import annotations import os +import re import sys from glob import glob from importlib import metadata as importlib_metadata @@ -38,8 +39,11 @@ # Some examples or system tests may depend on additional packages # that are not included in certain CI checks. # The format of the dictionary is as follows: - # key: the prefix of the file to be excluded, + # key: the regexp matching the file to be excluded, # value: a dictionary containing package distributions with an optional version specifier, e.g., >=2.3.4 + ".*example_bedrock_retrieve_and_generate.py": {"opensearch-py": None}, + ".*example_opensearch.py": {"opensearch-py": None}, + r".*example_yandexcloud.*\.py": {"yandexcloud": None}, } IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = ( # Certain examples or system tests may trigger AirflowProviderDeprecationWarnings. @@ -123,13 +127,6 @@ def example_not_excluded_dags(xfail_db_exception: bool = False): for prefix in PROVIDERS_PREFIXES for provider in suspended_providers_folders ] - temporary_excluded_upgrade_boto_providers_folders = [ - AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix() - for prefix in PROVIDERS_PREFIXES - # TODO - remove me when https://github.com/apache/beam/issues/32080 is addressed - # and we bring back yandex to be run in case of upgrade boto - for provider in ["yandex"] - ] current_python_excluded_providers_folders = [ AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix() for prefix in PROVIDERS_PREFIXES @@ -145,18 +142,13 @@ def example_not_excluded_dags(xfail_db_exception: bool = False): if candidate.startswith(tuple(suspended_providers_folders)): param_marks.append(pytest.mark.skip(reason="Suspended provider")) - if os.environ.get("UPGRADE_BOTO", "false") == "true" and candidate.startswith( - tuple(temporary_excluded_upgrade_boto_providers_folders) - ): - param_marks.append(pytest.mark.skip(reason="Temporary excluded upgrade boto provider")) - if candidate.startswith(tuple(current_python_excluded_providers_folders)): param_marks.append( pytest.mark.skip(reason=f"Not supported for Python {CURRENT_PYTHON_VERSION}") ) for optional, dependencies in OPTIONAL_PROVIDERS_DEPENDENCIES.items(): - if candidate.endswith(optional): + if re.match(optional, candidate): for distribution_name, specifier in dependencies.items(): result, reason = match_optional_dependencies(distribution_name, specifier) if not result: diff --git a/tests/providers/opensearch/conftest.py b/tests/providers/opensearch/conftest.py index 47a447188ecdd..934bbd642ad58 100644 --- a/tests/providers/opensearch/conftest.py +++ b/tests/providers/opensearch/conftest.py @@ -19,12 +19,19 @@ from typing import Any import pytest -from opensearchpy import OpenSearch +from airflow.hooks.base import BaseHook from airflow.models import Connection -from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook from airflow.utils import db +try: + from opensearchpy import OpenSearch + + from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook +except ImportError: + OpenSearch = None # type: ignore[assignment, misc] + OpenSearchHook = BaseHook # type: ignore[assignment,misc] + # TODO: FIXME - those Mocks have overrides that are not used but they also do not make Mypy Happy # mypy: disable-error-code="override" diff --git a/tests/providers/opensearch/hooks/test_opensearch.py b/tests/providers/opensearch/hooks/test_opensearch.py index 84360ae73f46a..43075e8532210 100644 --- a/tests/providers/opensearch/hooks/test_opensearch.py +++ b/tests/providers/opensearch/hooks/test_opensearch.py @@ -18,8 +18,9 @@ from unittest import mock -import opensearchpy import pytest + +opensearchpy = pytest.importorskip("opensearchpy") from opensearchpy import Urllib3HttpConnection from airflow.exceptions import AirflowException diff --git a/tests/providers/opensearch/operators/test_opensearch.py b/tests/providers/opensearch/operators/test_opensearch.py index 706112fef65b3..63ad7eafe48de 100644 --- a/tests/providers/opensearch/operators/test_opensearch.py +++ b/tests/providers/opensearch/operators/test_opensearch.py @@ -17,6 +17,9 @@ from __future__ import annotations import pytest + +opensearchpy = pytest.importorskip("opensearchpy") + from opensearchpy import Document, Keyword, Text from airflow.models import DAG