diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46d7537fa5f0b..bbee4e92a2e12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,7 @@ jobs: providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }} skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} + run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} @@ -906,6 +907,50 @@ jobs: - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}" uses: ./.github/actions/post_tests + tests-postgres-boto: + timeout-minutes: 130 + name: > + LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}}, + Py${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.test-types}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, test-pytest-collection] + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + TEST_TYPES: "${{needs.build-info.outputs.test-types}}" + PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" + FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" + DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" + BACKEND: "postgres" + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" + POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + UPGRADE_BOTO: "true" + JOB_ID: > + postgres-${{needs.build-info.outputs.default-python-version}}- + ${{needs.build-info.outputs.default-postgres-version}} + COVERAGE: "${{needs.build-info.outputs.run-coverage}}" + if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' + steps: + - name: Cleanup repo + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: > + Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} + uses: ./.github/actions/prepare_breeze_and_image + - name: > + Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}} + run: breeze testing tests --run-in-parallel + - name: > + Post Tests: ${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.test-types}} + uses: ./.github/actions/post_tests + tests-mysql: timeout-minutes: 130 name: > diff --git a/Dockerfile.ci b/Dockerfile.ci index 22a5e95faa957..abde6eefb9868 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1016,6 +1016,13 @@ else exit 1 fi fi +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall aiobotocore -y || true + pip install --upgrade boto3 botocore +fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index def292fa443cb..a0033b9075e1c 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -357,6 +357,12 @@ def run_tests_in_parallel( is_flag=True, envvar="FULL_TESTS_NEEDED", ) +@click.option( + "--upgrade-boto", + help="Remove aiobotocore and upgrade botocore and boto to the latest version.", + is_flag=True, + envvar="UPGRADE_BOTO", +) @option_verbose @option_dry_run @click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED) @@ -380,6 +386,7 @@ def tests( full_tests_needed: bool, mount_sources: str, extra_pytest_args: tuple, + upgrade_boto: bool, ): docker_filesystem = get_filesystem_type("/var/lib/docker") get_console().print(f"Docker filesystem: {docker_filesystem}") @@ -394,6 +401,7 @@ def tests( mount_sources=mount_sources, forward_ports=False, test_type=test_type, + upgrade_boto=upgrade_boto, ) rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params) cleanup_python_generated_files() diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index c9d7f86fab367..7cd81a33d9b7d 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -53,6 +53,7 @@ "options": [ "--image-tag", "--mount-sources", + "--upgrade-boto", ], }, ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index a28d62bcad553..10b820617f595 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -112,6 +112,7 @@ class ShellParams: version_suffix_for_pypi: str = "" dry_run: bool = False verbose: bool = False + upgrade_boto: bool = False def clone_with_test(self, test_type: str) -> ShellParams: new_params = deepcopy(self) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 9f5b7a2f1140a..98d341ce19c9e 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false") set_value_to_default_if_not_set(env, "TEST_TYPE", "") set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60") + set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false") set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false") set_value_to_default_if_not_set(env, "VERBOSE", "false") @@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: "SKIP_PROVIDER_TESTS": "skip_provider_tests", "SQLITE_URL": "sqlite_url", "START_AIRFLOW": "start_airflow", + "UPGRADE_BOTO": "upgrade_boto", "USE_AIRFLOW_VERSION": "use_airflow_version", "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 75fb5f0faa387..e8911b8eb1cf1 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -154,14 +154,14 @@ def __hash__(self): r"\.py$", ], FileGroupForCi.ALL_SOURCE_FILES: [ - "^.pre-commit-config.yaml$", - "^airflow", - "^chart", - "^tests", - "^kubernetes_tests", + r"^.pre-commit-config.yaml$", + r"^airflow", + r"^chart", + r"^tests", + r"^kubernetes_tests", ], FileGroupForCi.SYSTEM_TEST_FILES: [ - "^tests/system/", + r"^tests/system/", ], } ) @@ -180,10 +180,11 @@ def __hash__(self): r"^tests/cli", ], SelectiveUnitTestTypes.PROVIDERS: [ - "^airflow/providers/", - "^tests/providers/", + r"^airflow/providers/", + r"^tests/system/providers/", + r"^tests/providers/", ], - SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"], + SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"], } ) @@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool: def run_www_tests(self) -> bool: return self._should_be_run(FileGroupForCi.WWW_FILES) + @cached_property + def run_amazon_tests(self) -> bool: + return "amazon" in self.test_types or "Providers" in self.test_types.split(" ") + @cached_property def run_kubernetes_tests(self) -> bool: return self._should_be_run(FileGroupForCi.KUBERNETES_FILES) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index c67f0c39c02b7..c468e5a78abd3 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "false", "needs-helm-tests": "false", "run-tests": "false", + "run-amazon-tests": "false", "docs-build": "false", "upgrade-to-newer-dependencies": "false", "test-types": "", @@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "upgrade-to-newer-dependencies": "false", "test-types": "API Always", @@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "false", "test-types": "API Always Providers[amazon,common.sql,google,postgres]", @@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "false", + "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", + "run-amazon-tests": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "test-types": "Always", @@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", "test-types": "API Always CLI Core Other Providers WWW", @@ -249,6 +259,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", "test-types": "API Always CLI Core Other Providers WWW", @@ -256,6 +267,62 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Everything should run and upgrading to newer requirements as dependencies change", ) ), + pytest.param( + ("airflow/providers/amazon/__init__.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "python-versions": "['3.7']", + "python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "run-amazon-tests": "true", + "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes," + "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]", + }, + id="Providers tests run including amazon tests if amazon provider files changed", + ), + pytest.param( + ("tests/providers/airbyte/__init__.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "python-versions": "['3.7']", + "python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "false", + "docs-build": "false", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers[airbyte,http]", + }, + id="Providers tests run including amazon tests if amazon tests provider files changed", + ), + pytest.param( + ("airflow/providers/amazon/file.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "python-versions": "['3.7']", + "python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes," + "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]", + }, + id="Providers tests run without amazon tests if no amazon file changed", + ), ], ) def test_expected_output_pull_request_main( diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index b56c3f38668a5..3f197d9eabcce 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c testing:helm-tests:94a442e7f3f63b34c4831a84d165690a testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98 -testing:tests:ce6d80b3073bafd1f10c50812432885a -testing:0f292a29b2a97fbd95778a72249ee997 +testing:tests:ddda3dc31e0002d67fad26b682ff689a +testing:0b3792aed114983d6c9d12741561ec1c diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index db5690e0ad3e1..691a330582364 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-r5 { fill: #868887 } +.breeze-testing-r4 { fill: #868887 } +.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -96,19 +96,19 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│integration-tests                        Run the specified integratio tests.                                      │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│integration-tests                        Run the specified integratio tests.                                      │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index 2dbcc1c42ead7..754c0811e21af 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -1,4 +1,4 @@ - + - + @@ -186,9 +186,12 @@ + + + - Command: testing tests + Command: testing tests @@ -199,53 +202,54 @@ -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit test targets. -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | API | Always | CLI | Core | Other | Providers | WWW | Helm | Postgres | MySQL |      │ -│Quarantine)                                                                                 │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                               │ -│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd | statsd│ -│| trino)                                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Other Providers WWW]              │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--full-tests-neededWhether full set of tests is run.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ -│(selected | all | skip | remove)                                                               │ -│[default: selected]                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | API | Always | CLI | Core | Other | Providers | WWW | Helm | Postgres | MySQL |      │ +│Quarantine)                                                                                 │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                               │ +│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd | statsd│ +│| trino)                                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Other Providers WWW]              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--full-tests-neededWhether full set of tests is run.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ +│(selected | all | skip | remove)                                                               │ +│[default: selected]                                                                            │ +│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index dbea935eeae90..737b7e46ef4c1 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -63,6 +63,7 @@ SKIP_PROVIDER_TESTS SKIP_SSH_SETUP TEST_TIMEOUT TEST_TYPE +UPGRADE_BOTO UPGRADE_TO_NEWER_DEPENDENCIES VERBOSE VERBOSE_COMMANDS diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 7f11966b30702..a35df1963026f 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -76,6 +76,7 @@ services: - SKIP_SSH_SETUP=${SKIP_SSH_SETUP} - TEST_TYPE=${TEST_TYPE} - TEST_TIMEOUT=${TEST_TIMEOUT} + - UPGRADE_BOTO=${UPGRADE_BOTO} - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} - VERBOSE=${VERBOSE} - VERBOSE_COMMANDS=${VERBOSE_COMMANDS} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index c3ca6e842029e..86dd3e562996d 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -61,6 +61,7 @@ SKIP_SSH_SETUP="true" SKIP_ENVIRONMENT_INITIALIZATION="false" SKIP_PROVIDER_TESTS="false" TEST_TYPE= +UPGRADE_BOTO="false" UPGRADE_TO_NEWER_DEPENDENCIES="false" VERBOSE="false" VERBOSE_COMMANDS="false" diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 1ea9307623a5c..98bacb01e4072 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -443,6 +443,13 @@ else exit 1 fi fi +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall aiobotocore -y || true + pip install --upgrade boto3 botocore +fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py b/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py index eef118ce6794e..969729efa30cd 100644 --- a/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py +++ b/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py @@ -24,6 +24,8 @@ from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseAsyncHook from tests.providers.amazon.aws.utils.compat import async_mock +pytest.importorskip("aiobotocore") + try: from aiobotocore.credentials import AioCredentials except ImportError: diff --git a/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py b/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py index b9552837d6347..3cbc39cedabc9 100644 --- a/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py +++ b/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py @@ -24,6 +24,8 @@ from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook from tests.providers.amazon.aws.utils.compat import async_mock +pytest.importorskip("aiobotocore") + class TestRedshiftAsyncHook: @pytest.mark.asyncio diff --git a/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py b/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py index 0bfd40065620d..b933cae91707b 100644 --- a/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py +++ b/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py @@ -24,6 +24,8 @@ from airflow.triggers.base import TriggerEvent from tests.providers.amazon.aws.utils.compat import async_mock +pytest.importorskip("aiobotocore") + TASK_ID = "redshift_trigger_check" POLLING_PERIOD_SECONDS = 1.0