diff --git a/Dockerfile.ci b/Dockerfile.ci
index e30f1a9a7567b..88e4316559856 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1184,32 +1184,6 @@ COPY <<"EOF" /entrypoint_exec.sh
 exec /bin/bash "${@}"
 EOF
 
-# The content below is automatically copied from scripts/docker/patch_cassandra_type_code.py
-COPY <<"EOF" /patch_cassandra_type_code.py
-#!/usr/bin/env python
-
-from __future__ import annotations
-
-import cassandra.type_codes as cassandra_type_codes
-
-if __name__ == "__main__":
-    print()
-    path_to_patch = cassandra_type_codes.__file__
-    with open(path_to_patch, "r+") as f:
-        content = f.read()
-        if "PYTEST_DONT_REWRITE" in content:
-            print(f"The {path_to_patch} is already patched with PYTEST_DONT_REWRITE")
-            print()
-            exit(0)
-        f.seek(0)
-        content = content.replace('"""', '"""\nPYTEST_DONT_REWRITE', 1)
-        f.write(content)
-        f.truncate()
-        print(f"Patched {path_to_patch} with PYTEST_DONT_REWRITE")
-        print()
-        exit(0)
-EOF
-
 FROM ${PYTHON_BASE_IMAGE} as main
 
 # Nolog bash flag is currently ignored - but you can replace it with other flags (for example
@@ -1424,15 +1398,6 @@ RUN bash /scripts/docker/install_pip_version.sh; \
         bash /scripts/docker/install_additional_dependencies.sh; \
     fi
 
-COPY --from=scripts patch_cassandra_type_code.py /patch_cassandra_type_code.py
-
-# Patch cassandra type_code to avoide accidental type_code assert rewriting breaking pytest
-# test discovery and execution.
-# This one can be fixed once https://github.com/pytest-dev/pytest/issues/10844
-# is fixed and released or once the workaround is merged in cassandra-driver
-# https://github.com/datastax/python-driver/pull/1142
-RUN python /patch_cassandra_type_code.py
-
 # Install autocomplete for airflow
 RUN if command -v airflow; then \
         register-python-argcomplete airflow >> ~/.bashrc ; \
diff --git a/pyproject.toml b/pyproject.toml
index bdddc5610c7bc..bad61144d76e0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -99,7 +99,7 @@ filterwarnings = [
     "ignore::DeprecationWarning:apispec.utils",
 ]
 python_files = [
-    "*.py",
+    "test_*.py",
 ]
 testpaths = [
     "tests",
diff --git a/scripts/docker/patch_cassandra_type_code.py b/scripts/docker/patch_cassandra_type_code.py
deleted file mode 100644
index eba955d31c8a5..0000000000000
--- a/scripts/docker/patch_cassandra_type_code.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import annotations
-
-import cassandra.type_codes as cassandra_type_codes
-
-if __name__ == "__main__":
-    print()
-    path_to_patch = cassandra_type_codes.__file__
-    with open(path_to_patch, "r+") as f:
-        content = f.read()
-        if "PYTEST_DONT_REWRITE" in content:
-            print(f"The {path_to_patch} is already patched with PYTEST_DONT_REWRITE")
-            print()
-            exit(0)
-        f.seek(0)
-        content = content.replace('"""', '"""\nPYTEST_DONT_REWRITE', 1)
-        f.write(content)
-        f.truncate()
-        print(f"Patched {path_to_patch} with PYTEST_DONT_REWRITE")
-        print()
-        exit(0)
diff --git a/tests/conftest.py b/tests/conftest.py
index 4c44fd98107fc..98dcb3be960b1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -905,3 +905,14 @@ def clear_lru_cache():
 
     ExecutorLoader.validate_database_executor_compatibility.cache_clear()
     _get_grouped_entry_points.cache_clear()
+
+
+@pytest.fixture(autouse=True)
+def refuse_to_run_test_from_wrongly_named_files(request):
+    filename: str = request.node.fspath.basename
+    if not request.node.fspath.basename.startswith("test_"):
+        raise Exception(
+            f"All test method files in tests/ must start with 'test_'. Seems that {filename} "
+            f"contains {request.function} that looks like a test case. Please rename the file to "
+            f"follow the test_* pattern if you want to run the tests in it."
+        )
diff --git a/tests/providers/databricks/utils/databricks.py b/tests/providers/databricks/utils/test_databricks.py
similarity index 100%
rename from tests/providers/databricks/utils/databricks.py
rename to tests/providers/databricks/utils/test_databricks.py
diff --git a/tests/providers/smtp/hooks/smtp.py b/tests/providers/smtp/hooks/test_smtp.py
similarity index 100%
rename from tests/providers/smtp/hooks/smtp.py
rename to tests/providers/smtp/hooks/test_smtp.py