Remove requirement for test_ prefix for pytest test modules

.dockerignore

@@ -46,7 +46,6 @@
 !.coveragerc
 !.rat-excludes
 !.dockerignore
-!pytest.ini
 !RELEASE_NOTES.rst
 !LICENSE
 !MANIFEST.in
@@ -71,6 +70,7 @@
 # Setup/version configuration
 !setup.cfg
 !setup.py
+!pyproject.toml
 !manifests
 !generated
 # Now - ignore unnecessary files inside allowed directories

.github/workflows/ci.yml

@@ -499,8 +499,9 @@ jobs:
           cache-dependency-path: ./dev/requirements.txt
       - name: "Test examples of PROD image building"
         run: >
-          python -m pip install -r ./docker_tests/requirements.txt &&
-          python -m pytest docker_tests/test_examples_of_prod_image_building.py -n auto --color=yes
+          cd ./docker_tests &&
+          python -m pip install -r requirements.txt &&
+          python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
 
   test-git-clone-on-windows:
     timeout-minutes: 5

BREEZE.rst

@@ -786,6 +786,16 @@ For example this will run API and WWW tests in parallel:
 
     breeze testing tests --test-types "API WWW" --run-in-parallel
 
+There are few special types of tests that you can run:
+
+* ``All`` - all tests are run in single pytest run.
+* ``PlainAsserts`` - some tests of ours fail when ``--assert=rewrite`` feature of pytest is used. This
+  is in order to get better output of ``assert`` statements This is a special test type that runs those
+  select tests tests with ``--assert=plain`` flag.
+* ``Postgres`` - runs all tests that require Postgres database
+* ``MySQL`` - runs all tests that require MySQL database
+* ``Quarantine`` - runs all tests that are in quarantine (marked with ``@pytest.mark.quarantined``
+  decorator)
 
 Here is the detailed set of options for the ``breeze testing tests`` command.
 
@@ -1017,7 +1027,7 @@ Run selected tests:
 
 .. code-block::bash
 
-    breeze k8s tests kubernetes_tests/test_kubernetes_executor.py
+    breeze k8s tests test_kubernetes_executor.py
 
 All parameters of the command are here:
 
@@ -1042,7 +1052,7 @@ output during test execution.
 
 .. code-block::bash
 
-    breeze k8s tests -- kubernetes_tests/test_kubernetes_executor.py -s
+    breeze k8s tests -- test_kubernetes_executor.py -s
 
 Running k8s complete tests
 ..........................
@@ -1061,7 +1071,7 @@ Run selected tests:
 
 .. code-block::bash
 
-    breeze k8s run-complete-tests kubernetes_tests/test_kubernetes_executor.py
+    breeze k8s run-complete-tests test_kubernetes_executor.py
 
 All parameters of the command are here:
 
@@ -1086,7 +1096,7 @@ output during test execution.
 
 .. code-block::bash
 
-    breeze k8s run-complete-tests -- kubernetes_tests/test_kubernetes_executor.py -s
+    breeze k8s run-complete-tests -- test_kubernetes_executor.py -s
 
 
 Entering k8s shell
@@ -1110,16 +1120,16 @@ be created and airflow deployed to it before running the tests):
 
 .. code-block::bash
 
-    (kind-airflow-python-3.9-v1.24.0:KubernetesExecutor)> pytest kubernetes_tests/test_kubernetes_executor.py
+    (kind-airflow-python-3.9-v1.24.0:KubernetesExecutor)> pytest test_kubernetes_executor.py
     ================================================= test session starts =================================================
     platform linux -- Python 3.10.6, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /home/jarek/code/airflow/.build/.k8s-env/bin/python
     cachedir: .pytest_cache
     rootdir: /home/jarek/code/airflow, configfile: pytest.ini
     plugins: anyio-3.6.1
     collected 2 items
 
-    kubernetes_tests/test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag PASSED           [ 50%]
-    kubernetes_tests/test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure PASSED [100%]
+    test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag PASSED           [ 50%]
+    test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure PASSED [100%]
 
     ================================================== warnings summary ===================================================
     .build/.k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233

Dockerfile.ci

@@ -852,6 +852,8 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
     fi
 fi
 
+rm -f "${AIRFLOW_SOURCES}/pytest.ini"
+
 set +u
 if [[ "${RUN_TESTS}" != "true" ]]; then
     exec /bin/bash "${@}"
@@ -1009,6 +1011,16 @@ else
                 echo "${COLOR_YELLOW}Skip ${providers_dir} as the directory does not exist.${COLOR_RESET}"
             fi
         done
+    elif [[ ${TEST_TYPE} =~ PlainAsserts ]]; then
+        # Those tests fail when --asert=rewrite is set, therefore we run them separately
+        # with --assert=plain to make sure they pass.
+        SELECTED_TESTS=(
+            # this on is mysteriously failing dill serialization. It could be removed once
+            # https://github.com/pytest-dev/pytest/issues/10845 is fixed
+            "tests/operators/test_python.py::TestPythonVirtualenvOperator::test_airflow_context"
+        )
+        EXTRA_PYTEST_ARGS+=("--assert=plain")
+        export PYTEST_PLAIN_ASSERTS="true"
     else
         echo
         echo  "${COLOR_RED}ERROR: Wrong test type ${TEST_TYPE}  ${COLOR_RESET}"
@@ -1073,6 +1085,32 @@ COPY <<"EOF" /entrypoint_exec.sh
 exec /bin/bash "${@}"
 EOF
 
+# The content below is automatically copied from scripts/docker/patch_cassandra_type_code.py
+COPY <<"EOF" /patch_cassandra_type_code.py
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import cassandra.type_codes as cassandra_type_codes
+
+if __name__ == "__main__":
+    print()
+    path_to_patch = cassandra_type_codes.__file__
+    with open(path_to_patch, "r+") as f:
+        content = f.read()
+        if "PYTEST_DONT_REWRITE" in content:
+            print(f"The {path_to_patch} is already patched with PYTEST_DONT_REWRITE")
+            print()
+            exit(0)
+        f.seek(0)
+        content = content.replace('"""', '"""\nPYTEST_DONT_REWRITE', 1)
+        f.write(content)
+        f.truncate()
+        print(f"Patched {path_to_patch} with PYTEST_DONT_REWRITE")
+        print()
+        exit(0)
+EOF
+
 FROM ${PYTHON_BASE_IMAGE} as main
 
 # Nolog bash flag is currently ignored - but you can replace it with other flags (for example
@@ -1301,6 +1339,15 @@ RUN bash /scripts/docker/install_pip_version.sh; \
         bash /scripts/docker/install_additional_dependencies.sh; \
     fi
 
+COPY --from=scripts patch_cassandra_type_code.py /patch_cassandra_type_code.py
+
+# Patch cassandra type_code to avoide accidental type_code assert rewriting breaking pytest
+# test discovery and execution.
+# This one can be fixed once https://github.com/pytest-dev/pytest/issues/10844
+# is fixed and released or once the workaround is merged in cassandra-driver
+# https://github.com/datastax/python-driver/pull/1142
+RUN python /patch_cassandra_type_code.py
+
 # Install autocomplete for airflow
 RUN if command -v airflow; then \
         register-python-argcomplete airflow >> ~/.bashrc ; \
@@ -1309,6 +1356,7 @@ RUN if command -v airflow; then \
 # Install autocomplete for Kubectl
 RUN echo "source /etc/bash_completion" >> ~/.bashrc
 
+
 # We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
 # copying over stuff that is accidentally generated or that we do not need (such as egg-info)
 # if you want to add something that is missing and you expect to see it in the image you can

TESTING.rst

@@ -1154,15 +1154,15 @@ The virtualenv required will be created automatically when the scripts are run.
     ========================================================================================= test session starts ==========================================================================================
     platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin/python
     cachedir: .pytest_cache
-    rootdir: /Users/jarek/IdeaProjects/airflow, configfile: pytest.ini
+    rootdir: /Users/jarek/IdeaProjects/airflow/kubernetes_tests
     plugins: anyio-3.6.1, instafail-0.4.2, xdist-2.5.0, forked-1.4.0, timeouts-1.2.1, cov-3.0.0
     setup timeout: 0.0s, execution timeout: 0.0s, teardown timeout: 0.0s
     collected 55 items
 
-    kubernetes_tests/test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag PASSED                                                                                            [  1%]
-    kubernetes_tests/test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure PASSED                                                                     [  3%]
-    kubernetes_tests/test_kubernetes_pod_operator.py::TestKubernetesPodOperatorSystem::test_already_checked_on_failure PASSED                                                                        [  5%]
-    kubernetes_tests/test_kubernetes_pod_operator.py::TestKubernetesPodOperatorSystem::test_already_checked_on_success   ...
+    test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag PASSED                                                                                            [  1%]
+    test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure PASSED                                                                     [  3%]
+    test_kubernetes_pod_operator.py::TestKubernetesPodOperatorSystem::test_already_checked_on_failure PASSED                                                                        [  5%]
+    test_kubernetes_pod_operator.py::TestKubernetesPodOperatorSystem::test_already_checked_on_success   ...
 
 8b) You can enter an interactive shell to run tests one-by-one
 
@@ -1239,11 +1239,12 @@ and this is where KUBECONFIG env should point to.
 
 You can iterate with tests while you are in the virtualenv. All the tests requiring Kubernetes cluster
 are in "kubernetes_tests" folder. You can add extra ``pytest`` parameters then (for example ``-s`` will
-print output generated test logs and print statements to the terminal immediately.
+print output generated test logs and print statements to the terminal immediately. You should have
+kubernetes_tests as your working directory.
 
 .. code-block:: bash
 
-    pytest kubernetes_tests/test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure -s
+    pytest test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure -s
 
 You can modify the tests or KubernetesPodOperator and re-run them without re-deploying
 Airflow to KinD cluster.

dev/airflow-github

@@ -146,7 +146,6 @@ def is_core_commit(files: list[str]) -> bool:
         "images/",
         "TESTING.rst",
         "codecov.yml",
-        "pytest.ini",
         "kubernetes_tests/",
         ".github/",
         ".pre-commit-config.yaml",

dev/breeze/README.md

@@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
 
 ---------------------------------------------------------------------------------------------------------
 
-Package config hash: a98c6dc9a12523e5aaacaade98c721e869377ab59eead92344006f7fd71fe2978398b87703dfc5efa61fac372a4e798cef9de26d6f4283a47f919cd7e0dc4dfa
+Package config hash: 279584a15bd0cec7a270b8fb85c3a6f853abca893fd4ecb419636fc2319a75d3644bd76e773008d6e3f77440fb2651a307f974a13df32191bea2c2f471c368f2
 
 ---------------------------------------------------------------------------------------------------------

dev/breeze/pyproject.toml

@@ -17,3 +17,19 @@
 [tool.black]
 line-length = 110
 target-version = ['py37', 'py38', 'py39', 'py310']
+
+[tool.pytest.ini_options]
+addopts = "-rasl --verbosity=2 -p no:flaky -p no:nose"
+norecursedirs = [
+    ".eggs",
+]
+log_level = "INFO"
+filterwarnings = [
+    "error::pytest.PytestCollectionWarning",
+]
+python_files = [
+    "*.py",
+]
+testpaths = [
+    "tests",
+]

dev/breeze/src/airflow_breeze/breeze.py

@@ -35,9 +35,9 @@
 from airflow_breeze.commands.production_image_commands import prod_image  # noqa
 from airflow_breeze.commands.release_management_commands import release_management  # noqa
 from airflow_breeze.commands.setup_commands import setup  # noqa
-from airflow_breeze.commands.testing_commands import testing  # noqa
+from airflow_breeze.commands.testing_commands import group_for_testing  # noqa
 
-main.add_command(testing)
+main.add_command(group_for_testing)
 main.add_command(kubernetes_group)
 main.add_command(ci_group)
 main.add_command(ci_image)

dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py

@@ -1266,7 +1266,9 @@ def shell(
         extra_args.append("--no-rcs")
     elif shell_binary.endswith("bash"):
         extra_args.extend(["--norc", "--noprofile"])
-    result = run_command([shell_binary, *extra_args, *shell_args], env=env, check=False)
+    result = run_command(
+        [shell_binary, *extra_args, *shell_args], env=env, check=False, cwd="kubernetes_tests"
+    )
     if result.returncode != 0:
         sys.exit(result.returncode)
 
@@ -1300,17 +1302,15 @@ def _run_tests(
         extra_shell_args.append("--no-rcs")
     elif shell_binary.endswith("bash"):
         extra_shell_args.extend(["--norc", "--noprofile"])
-    the_tests = []
-    if not any(arg.startswith("kubernetes_tests") for arg in test_args):
-        # if no tests specified - use args
-        the_tests.append("kubernetes_tests")
+    the_tests: list[str] = []
     command_to_run = " ".join([quote(arg) for arg in ["pytest", *the_tests, *test_args]])
     get_console(output).print(f"[info] Command to run:[/] {command_to_run}")
     result = run_command(
         [shell_binary, *extra_shell_args, "-c", command_to_run],
         output=output,
         env=env,
         check=False,
+        cwd="kubernetes_tests",
     )
     return result.returncode, f"Tests {kubectl_cluster_name}"
 
@@ -1336,7 +1336,7 @@ def _run_tests(
 @option_verbose
 @option_dry_run
 @click.argument("test_args", nargs=-1, type=click.Path())
-def tests(
+def kubernetes_tests_command(
     python: str,
     kubernetes_version: str,
     executor: str,

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -26,7 +26,7 @@
 
 from airflow_breeze.commands.ci_image_commands import ci_image
 from airflow_breeze.commands.production_image_commands import prod_image
-from airflow_breeze.commands.testing_commands import testing
+from airflow_breeze.commands.testing_commands import group_for_testing
 from airflow_breeze.configure_rich_click import click
 from airflow_breeze.utils.click_utils import BreezeGroup
 from airflow_breeze.utils.common_options import (
@@ -74,7 +74,7 @@ def get_command(self, ctx: Context, cmd_name: str):
             return prod_image.get_command(ctx, "build")
         if cmd_name == "tests":
             print_deprecated("tests", "testing tests")
-            return testing.get_command(ctx, "tests")
+            return group_for_testing.get_command(ctx, "tests")
         if cmd_name == "config":
             print_deprecated("config", "setup config")
             return setup.get_command(ctx, "config")

dev/breeze/src/airflow_breeze/commands/testing_commands.py

@@ -72,11 +72,11 @@
 
 
 @click.group(cls=BreezeGroup, name="testing", help="Tools that developers can use to run tests")
-def testing():
+def group_for_testing():
     pass
 
 
-@testing.command(
+@group_for_testing.command(
     name="docker-compose-tests",
     context_settings=dict(
         ignore_unknown_options=True,
@@ -308,7 +308,7 @@ def run_tests_in_parallel(
     )
 
 
-@testing.command(
+@group_for_testing.command(
     name="tests",
     help="Run the specified unit test targets.",
     context_settings=dict(
@@ -347,7 +347,7 @@ def run_tests_in_parallel(
 @click.option(
     "--test-types",
     help="Space separated list of test types used for testing in parallel.",
-    default=" ".join(all_selective_test_types()),
+    default=" ".join(all_selective_test_types()) + " PlainAsserts",
     show_default=True,
     envvar="TEST_TYPES",
 )
@@ -366,7 +366,7 @@ def run_tests_in_parallel(
 @option_verbose
 @option_dry_run
 @click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
-def tests(
+def command_for_tests(
     python: str,
     backend: str,
     postgres_version: str,
@@ -432,7 +432,7 @@ def tests(
         sys.exit(returncode)
 
 
-@testing.command(
+@group_for_testing.command(
     name="integration-tests",
     help="Run the specified integratio tests.",
     context_settings=dict(
@@ -506,7 +506,7 @@ def integration_tests(
     sys.exit(returncode)
 
 
-@testing.command(
+@group_for_testing.command(
     name="helm-tests",
     help="Run Helm chart tests.",
     context_settings=dict(

dev/breeze/src/airflow_breeze/global_constants.py

@@ -109,7 +109,7 @@ class SelectiveUnitTestTypes(Enum):
 ALLOWED_TEST_TYPE_CHOICES = [
     "All",
     *all_selective_test_types(),
-    "Helm",
+    "PlainAsserts",
     "Postgres",
     "MySQL",
     "Quarantine",