Merge pull request #5452 from jenshnielsen/src_layout_move_tests

Move tests outside of src layout

.github/workflows/pytest.yaml

@@ -82,11 +82,11 @@ jobs:
       if: ${{ !matrix.min-version }}
     - name: Run parallel tests
       run: |
-        pytest -m "not serial" --cov=qcodes --cov-report xml --hypothesis-profile ci  --durations=20 src/qcodes
+        pytest -m "not serial" --cov=qcodes --cov-report xml --hypothesis-profile ci  --durations=20 tests
 # a subset of the tests fails when run in parallel on Windows so run those in serial here
     - name: Run serial tests
       run: |
-        pytest -m "serial" -n 0 --dist no --cov=qcodes --cov-report xml --cov-append --hypothesis-profile ci src/qcodes
+        pytest -m "serial" -n 0 --dist no --cov=qcodes --cov-report xml --cov-append --hypothesis-profile ci tests
     - name: Upload coverage to Codecov
       uses: codecov/[email protected]
       with:

.gitmodules

@@ -1,5 +1,5 @@
 [submodule "qcodes/tests/dataset/fixtures/db_files"]
-	path = src/qcodes/tests/dataset/fixtures/db_files
+	path = tests/dataset/fixtures/db_files
 	url = https://github.com/QCoDeS/qcodes_db_fixtures.git
 	branch = main
 [submodule "typings"]

docs/changes/newsfragments/5452.breaking

@@ -0,0 +1,5 @@
+Tests are no longer shipped as part of the qcodes package. The qcodes.tests
+namespace still exists but will be deprecated in QCoDeS 0.43.0.
+`qcodes.test` is deprecated and will be removed in a future release.
+To run the tests against an installed version clone git repo to matching tag and
+run `pytest tests` from the root of the repo.

pyproject.toml

@@ -205,7 +205,7 @@ stubPath = "typings/stubs"
 [tool.pytest.ini_options]
 minversion = "6.0"
 junit_family = "legacy"
-
+testpaths = "tests"
 addopts = "-n auto --dist=loadfile"
 
 markers = "serial"
@@ -284,6 +284,7 @@ known-first-party = ["qcodes"]
 # in tests and examples
 "docs/*" = ["TID253"]
 "src/qcodes/tests/*" = ["TID253"]
+"tests/*" = ["TID253"]
 
 [tool.ruff.flake8-tidy-imports]
 # There modules are relatively slow to import

src/qcodes/__init__.py

@@ -75,6 +75,7 @@
     combine,
 )
 from qcodes.station import Station
+from qcodes.utils import deprecate
 
 # ensure to close all instruments when interpreter is closed
 atexit.register(Instrument.close_all)
@@ -89,22 +90,18 @@
         QCoDeSDeprecationWarning,
     )
 
+
+@deprecate(
+    reason="tests are no longer shipped as part of QCoDeS",
+    alternative="Clone git repo to matching tag and run `pytest tests` from the root of the repo.",
+)
 def test(**kwargs: Any) -> int:
     """
-    Run QCoDeS tests. This requires the test requirements given
-    in test_requirements.txt to be installed.
-    All arguments are forwarded to pytest.main
+    Deprecated
     """
-    try:
-        import pytest
-        from hypothesis import settings
-        settings(deadline=1000)
-    except ImportError:
-        print("Need pytest and hypothesis to run tests")
-        return 1
-    args = ['--pyargs', 'qcodes.tests']
-    retcode = pytest.main(args, **kwargs)
-    return retcode
-
-
-test.__test__ = False  # type: ignore[attr-defined] # Don't try to run this method as a test
+    return 0
+
+
+del deprecate
+
+test.__test__ = False  # Don't try to run this method as a test

tests/common.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import cProfile
+import os
+from functools import wraps
+from pathlib import Path
+from time import sleep
+from typing import TYPE_CHECKING, Any, Callable, TypeVar
+
+import pytest
+from typing_extensions import ParamSpec
+
+from qcodes.metadatable import MetadatableWithName
+
+if TYPE_CHECKING:
+    from pytest import ExceptionInfo
+
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+def retry_until_does_not_throw(
+    exception_class_to_expect: type[Exception] = AssertionError,
+    tries: int = 5,
+    delay: float = 0.1,
+) -> Callable[[Callable[P, T]], Callable[P, T]]:
+    """
+    Call the decorated function given number of times with given delay between
+    the calls until it does not throw an exception of a given class.
+
+    If the function throws an exception of a different class, it gets propagated
+    outside (i.e. the function is not called anymore).
+
+    Usage:
+        >>  x = False  # let's assume that another thread has access to "x",
+                       # and it is going to change "x" to "True" very soon
+        >>  @retry_until_does_not_throw() ...
+            def assert_x_is_true(): ...
+                assert x, "x is still False..." ...
+        >>  assert_x_is_true()  # depending on the settings of
+                                # "retry_until_does_not_throw", it will keep
+                                # calling the function (with breaks in between)
+                                # until either it does not throw or
+                                # the number of tries is exceeded.
+
+    Args:
+        exception_class_to_expect
+            Only in case of this exception the function will be called again
+        tries
+            Number of times to retry calling the function before giving up
+        delay
+            Delay between retries of the function call, in seconds
+
+    Returns:
+        A callable that runs the decorated function until it does not throw
+        a given exception
+    """
+
+    def retry_until_passes_decorator(func: Callable[P, T]) -> Callable[P, T]:
+
+        @wraps(func)
+        def func_retry(*args: P.args, **kwargs: P.kwargs) -> T:
+            tries_left = tries - 1
+            while tries_left > 0:
+                try:
+                    return func(*args, **kwargs)
+                except exception_class_to_expect:
+                    tries_left -= 1
+                    sleep(delay)
+            # the very last attempt to call the function is outside
+            # the "try-except" clause, so that the exception can propagate
+            # up the call stack
+            return func(*args, **kwargs)
+
+        return func_retry
+
+    return retry_until_passes_decorator
+
+
+def profile(func: Callable[P, T]) -> Callable[P, T]:
+    """
+    Decorator that profiles the wrapped function with cProfile.
+
+    It produces a '.prof' file in the current working directory
+    that has the name of the executed function.
+
+    Use the 'Stats' class from the 'pstats' module to read the file,
+    analyze the profile data (for example, 'p.sort_stats('tottime')'
+    where 'p' is an instance of the 'Stats' class), and print the data
+    (for example, 'p.print_stats()').
+    """
+
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        profile_filename = func.__name__ + '.prof'
+        profiler = cProfile.Profile()
+        result = profiler.runcall(func, *args, **kwargs)
+        profiler.dump_stats(profile_filename)
+        return result
+    return wrapper
+
+
+def error_caused_by(excinfo: ExceptionInfo[Any], cause: str) -> bool:
+    """
+    Helper function to figure out whether an exception was caused by another
+    exception with the message provided.
+
+    Args:
+        excinfo: the output of with pytest.raises() as excinfo
+        cause: the error message or a substring of it
+    """
+
+    exc_repr = excinfo.getrepr()
+
+    chain = getattr(exc_repr, "chain", None)
+
+    if chain is not None:
+        # first element of the chain is info about the root exception
+        error_location = chain[0][1]
+        root_traceback = chain[0][0]
+        # the error location is the most reliable data since
+        # it only contains the location and the error raised.
+        # however there are cases where this is empty
+        # in such cases fall back to the traceback
+        if error_location is not None:
+            return cause in str(error_location)
+        else:
+            return cause in str(root_traceback)
+    else:
+        return False
+
+
+def skip_if_no_fixtures(dbname: str | Path) -> None:
+    if not os.path.exists(dbname):
+        pytest.skip(
+            "No db-file fixtures found. "
+            "Make sure that your git clone of qcodes has submodules "
+            "This can be done by executing: `git submodule update --init`"
+        )
+
+
+class DummyComponent(MetadatableWithName):
+
+    """Docstring for DummyComponent."""
+
+    def __init__(self, name: str):
+        super().__init__()
+        self.name = name
+
+    def __str__(self) -> str:
+        return self.full_name
+
+    def set(self, value: float) -> float:
+        value = value * 2
+        return value
+
+    @property
+    def short_name(self) -> str:
+        return self.name
+
+    @property
+    def full_name(self) -> str:
+        return self.full_name