diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 357cadb04cec4..a3aa172b4f134 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1055,10 +1055,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Test Offline SQL generation" run: ./scripts/ci/testing/run_offline_sql_test.sh - name: "Tests: ${{needs.build-info.outputs.test-types}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh + run: breeze testing tests + --run-parallel-test env: PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + TEST_TYPES: ${{needs.build-info.outputs.test-types}} - name: "Upload airflow logs" uses: actions/upload-artifact@v3 if: failure() @@ -1126,10 +1128,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Test downgrade" run: ./scripts/ci/testing/run_downgrade_test.sh - name: "Tests: ${{needs.build-info.outputs.test-types}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh + run: breeze testing tests + --run-parallel-test env: PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + TEST_TYPES: ${{needs.build-info.outputs.test-types}} - name: "Upload airflow logs" uses: actions/upload-artifact@v3 if: failure() @@ -1197,10 +1201,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Test downgrade" run: ./scripts/ci/testing/run_downgrade_test.sh - name: "Tests: ${{needs.build-info.outputs.test-types}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh + run: breeze testing tests + --run-parallel-test env: PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + TEST_TYPES: ${{needs.build-info.outputs.test-types}} - name: "Upload airflow logs" uses: actions/upload-artifact@v3 if: failure() @@ -1266,10 +1272,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Test downgrade" run: ./scripts/ci/testing/run_downgrade_test.sh - name: "Tests: ${{needs.build-info.outputs.test-types}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh + run: breeze testing tests + --run-parallel-test env: PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + TEST_TYPES: ${{needs.build-info.outputs.test-types}} - name: "Upload airflow logs" uses: actions/upload-artifact@v3 if: failure() diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 6cba47a6ef950..9b7394966e5a5 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -260,7 +260,6 @@ def run_build(ci_image_params: BuildCiParams) -> None: @option_verify @option_wait_for_image @option_image_tag_for_pulling -@option_include_success_outputs @option_tag_as_latest @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) def pull( diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 2cd51b4dae9be..0b793b24f3aa7 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -40,12 +40,16 @@ option_github_repository, option_image_name, option_image_tag_for_running, + option_include_success_outputs, option_integration, option_mount_sources, option_mssql_version, option_mysql_version, + option_parallelism, option_postgres_version, option_python, + option_run_parallel_test, + option_skip_cleanup, option_verbose, ) from airflow_breeze.utils.console import get_console, message_type_from_return_code @@ -55,6 +59,7 @@ get_env_variables_for_docker_commands, perform_environment_checks, ) +from airflow_breeze.utils.parallel import check_async_run_results, run_with_pool from airflow_breeze.utils.run_tests import run_docker_compose_tests from airflow_breeze.utils.run_utils import RunCommandResult, run_command @@ -189,6 +194,68 @@ def run_with_progress( return result +def run_tests( + env_variables: Dict[str, str], dry_run: bool, verbose: bool, test_type: str, extra_pytest_args: Tuple +) -> Tuple[int, str]: + env_variables['TEST_TYPE'] = test_type + perform_environment_checks(verbose=verbose) + cmd = ['docker-compose', 'run', '--service-ports', '--rm', 'airflow'] + cmd.extend(list(extra_pytest_args)) + test_result = run_command( + cmd, + verbose=verbose, + dry_run=dry_run, + env=env_variables, + ) + return ( + test_result.returncode, + f"Running tests {test_type}", + ) + + +def run_tests_in_parallel( + env_variables: Dict[str, str], + test_type_list: List[str], + include_success_outputs: bool, + skip_cleanup: bool, + parallelism: int, + dry_run: bool, + verbose: bool, + extra_pytest_args: Tuple, +): + """Run tests in parallel""" + with ci_group(f"Running tests for the type {test_type_list}"): + # get_console().print( + # f"\n[info]Running tests with parallelism = {parallelism} " + # f"for the test type: {test_type_list}[/]" + # ) + all_params = [f"Tests {test_type}" for test_type in test_type_list] + with run_with_pool( + parallelism=parallelism, + all_params=all_params, + ) as (pool, outputs): + results = [ + pool.apply_async( + run_tests, + kwds={ + "env_variables": env_variables, + "dry_run": dry_run, + "verbose": verbose, + "test_type": test_type, + "extra_pytest_args": extra_pytest_args, + }, + ) + for test_type in test_type_list + ] + check_async_run_results( + results=results, + success="All tests run correctly", + outputs=outputs, + include_success_outputs=include_success_outputs, + skip_cleanup=skip_cleanup, + ) + + @testing.command( name='tests', help="Run the specified unit test targets.", @@ -218,6 +285,7 @@ def run_with_progress( "tests should be run - for example --test-type \"Providers[airbyte,http]\"", default="All", type=NotVerifiedBetterChoice(ALLOWED_TEST_TYPE_CHOICES), + envvar='TEST_TYPES', ) @click.option( "--test-timeout", @@ -226,6 +294,10 @@ def run_with_progress( show_default=True, ) @option_db_reset +@option_run_parallel_test +@option_parallelism +@option_include_success_outputs +@option_skip_cleanup @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) def tests( dry_run: bool, @@ -241,8 +313,12 @@ def tests( test_type: str, test_timeout: str, db_reset: bool, - image_tag: str | None, + run_parallel_test: bool, + parallelism: int, + image_tag: Optional[str], mount_sources: str, + skip_cleanup: bool, + include_success_outputs: bool, ): exec_shell_params = ShellParams( verbose=verbose, @@ -293,6 +369,17 @@ def tests( verbose=verbose, dry_run=dry_run, ) + elif run_parallel_test: + run_tests_in_parallel( + env_variables=env_variables, + test_type_list=ALLOWED_TEST_TYPE_CHOICES, + include_success_outputs=include_success_outputs, + skip_cleanup=skip_cleanup, + parallelism=parallelism, + dry_run=dry_run, + verbose=verbose, + extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (), + ) else: result = run_command(cmd, verbose=verbose, dry_run=dry_run, env=env_variables, check=False) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index c79ab558203a9..dc8ab2c28a6ff 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -91,13 +91,10 @@ class SelectiveUnitTestTypes(Enum): ALLOWED_TEST_TYPE_CHOICES = [ "All", - "Always", *all_selective_test_types(), "Helm", "Postgres", "MySQL", - "Integration", - "Other", "Quarantine", ] diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index faec848dacd18..bed884b84e0e8 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -428,6 +428,12 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, is_flag=True, envvar='RUN_IN_PARALLEL', ) +option_run_parallel_test = click.option( + '--run-parallel-test', + help="Run the operation in parallel on all of Test types", + is_flag=True, + envvar='RUN_PARALLEL_TEST', +) option_parallelism = click.option( '--parallelism', help="Maximum number of processes to use while running the operation in parallel.", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 64f1ec89ab4cd..5fdaccbc9ba77 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -11,9 +11,9 @@ ci:resource-check:0fb929ac3496dbbe97acfe99e35accd7 ci:selective-check:d4e3c250cd6f2b0040fbe6557fa423f6 ci:31566cdcdde216086f559215223b2378 ci-image:build:d39a25675e6b74af9bbb1fc2582aacc5 -ci-image:pull:8aca8679e6030ad0d6e59216af40c0b3 +ci-image:pull:fdde25906ad4c666db606201be44c240 ci-image:verify:a2daeaa820c0baca31da2737929b38b9 -ci-image:b1c7a3c6dfa72b127fac559dcfdbb0d3 +ci-image:310bac16f2c61789453569a4f7901595 cleanup:9bf46a1dfd9db4fe13a1c233ad1bb96b compile-www-assets:23675c1862d0968cbff6ab6f1d93d488 exec:89b81bc34d45b0fe6653a6db5482258c @@ -53,5 +53,5 @@ static-checks:425cd78507278494e345fb7648260c24 stop:8ebd8a42f1003495d37b884de5ac7ce6 testing:docker-compose-tests:3e07be65e30219930d3c62a593dd8c6a testing:helm-tests:403231f0a94b261f9c7aae8aea03ec50 -testing:tests:32deda30f3899e8ae6e241238f990d68 -testing:e747ece268ba502c106924eb2f46c550 +testing:tests:418233cc2313a34da99fb41a841fe0cf +testing:416c94fc5fce116f7f79d296484dfa1b diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index e8c33d0cf094a..91edb4c6a9b03 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + - Command: testing tests + Command: testing tests - + - - -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run the specified unit test targets. - -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | Always | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm |  │ -│Postgres | MySQL | Integration | Other | Quarantine)                                        │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value(TEXT)│ -│[default: 60]                                                                    │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(sqlite | mysql | >postgres< | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(10 | 11 | >12< | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ -│[default: latest]                                                             │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ -│selected).                                                                             │ -│(selected | all | skip | remove)                                                       │ -│[default: selected]                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run the specified unit test targets. + +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ +│MySQL | Quarantine)                                                                         │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value(TEXT)│ +│[default: 60]                                                                    │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip)(TEXT)│ +│[default: latest]                                                             │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ +│selected).                                                                             │ +│(selected | all | skip | remove)                                                       │ +│[default: selected]                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--run-parallel-testRun the operation in parallel on all of Test types│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--include-success-outputsWhether to include outputs of successful parallel runs (by default they are not      │ +│printed).                                                                            │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯