Move tests to pytest

* custom pytest collector and item to surface pyyaml data-driven tests as individual pytest unit tests
* moved some true unit tests to pytest-native tests
* deprecated `setup.py test` custom command

.github/workflows/ci.yaml

@@ -176,7 +176,8 @@ jobs:
           LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
         python3 -V
@@ -296,9 +297,12 @@ jobs:
         C_INCLUDE_PATH: libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
         LIBRARY_PATH: libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
+        PYYAML_FORCE_CYTHON: 1
+        PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         python3 -V
@@ -458,8 +462,8 @@ jobs:
         build bdist_wheel
 
         # run tests on built wheel
-        python -m pip install dist/*.whl
-        python tests/lib/test_all.py
+        python -m pip install dist/*.whl pytest
+        python -m pytest
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

.github/workflows/manual_artifact_build.yaml

@@ -175,7 +175,8 @@ jobs:
           LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
         python3 -V
@@ -295,9 +296,12 @@ jobs:
         C_INCLUDE_PATH: libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
         LIBRARY_PATH: libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
+        PYYAML_FORCE_CYTHON: 1
+        PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         python3 -V
@@ -457,8 +461,8 @@ jobs:
         build bdist_wheel
 
         # run tests on built wheel
-        python -m pip install dist/*.whl
-        python tests/lib/test_all.py
+        python -m pip install dist/*.whl pytest
+        python -m pytest
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

MANIFEST.in

@@ -2,6 +2,6 @@ include CHANGES README LICENSE Makefile pyproject.toml setup.py
 recursive-include lib/yaml *.py
 recursive-include lib/_yaml *.py
 recursive-include examples *.py *.cfg *.yaml
-recursive-include tests/data *
-recursive-include tests/lib *.py
+recursive-include tests/legacy_tests/ *.py
+recursive-include tests/legacy_tests/data *
 recursive-include yaml *

Makefile

@@ -46,6 +46,6 @@ clean:
 	    dist/ \
 	    lib/PyYAML.egg-info/ \
 	    lib/yaml/__pycache__/ \
-	    tests/lib/__pycache__/ \
-	    yaml/_yaml.c \
-
+	    tests/__pycache__/ \
+	    tests/legacy_tests/__pycache__/ \
+	    yaml/_yaml.c

setup.py

@@ -250,6 +250,7 @@ def finalize_options(self):
         pass
 
     def run(self):
+        warnings.warn('Running tests via `setup.py test` is deprecated and will be removed in a future release. Use `pytest` instead to ensure that the complete test suite is run.', DeprecationWarning)
         build_cmd = self.get_finalized_command('build')
         build_cmd.run()
 
@@ -263,7 +264,7 @@ def run(self):
             temp_test_path = pathlib.Path(tempdir.name) / 'pyyaml'
             shutil.copytree(build_cmd.build_lib, temp_test_path)
             sys.path.insert(0, str(temp_test_path))
-            sys.path.insert(0, 'tests/lib')
+            sys.path.insert(0, 'tests/legacy_tests')
 
             import test_all
             if not test_all.main([]):

tests/legacy_tests/conftest.py

@@ -0,0 +1,130 @@
+# pytest custom collection adapter for legacy pyyaml unit tests/data files; surfaces each
+# legacy test case as a pyyaml item
+
+import os
+import pytest
+import warnings
+
+from test_appliance import find_test_filenames, DATA
+
+try:
+    from yaml import _yaml
+    HAS_LIBYAML_EXT = True
+    del _yaml
+except ImportError:
+    HAS_LIBYAML_EXT = False
+
+
+_test_filenames = find_test_filenames(DATA)
+
+# ignore all datafiles
+collect_ignore_glob = ['data/*']
+
+
+class PyYAMLItem(pytest.Item):
+    def __init__(self, parent=None, config=None, session=None, nodeid=None, function=None, filenames=None, **kwargs):
+        self._function = function
+        self._fargs = filenames or []
+
+        super().__init__(os.path.basename(filenames[0]) if filenames else parent.name, parent, config, session, nodeid)
+        # this is gnarly since the type of fspath is private; fixed in pytest 7 to use pathlib on the `path` attr
+        if filenames:  # pass the data file location as the test path
+            self.fspath = parent.fspath.__class__(filenames[0])
+            self.lineno = 1
+        else:  # pass the function location in the code
+            self.fspath = parent.fspath.__class__(function.__code__.co_filename)
+            self.lineno = function.__code__.co_firstlineno
+
+    def runtest(self):
+        self._function(verbose=True, *self._fargs)
+
+    def reportinfo(self):
+        return self.fspath, self.lineno, ''
+
+
+class PyYAMLCollector(pytest.Collector):
+    def __init__(self, name, parent=None, function=None, **kwargs):
+        self._function = function
+        self.fspath = parent.fspath.__class__(function.__code__.co_filename)
+        self.lineno = function.__code__.co_firstlineno
+
+        # avoid fspath deprecation warnings on pytest < 7
+        if hasattr(self, 'path') and 'fspath' in kwargs:
+            del kwargs['fspath']
+
+        super().__init__(name=name, parent=parent, **kwargs)
+
+    def collect(self):
+        items = []
+
+        unittest = getattr(self._function, 'unittest', None)
+
+        if unittest is True:  # no filenames
+            items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=None))
+        else:
+            for base, exts in _test_filenames:
+                filenames = []
+                for ext in unittest:
+                    if ext not in exts:
+                        break
+                    filenames.append(os.path.join(DATA, base + ext))
+                else:
+                    skip_exts = getattr(self._function, 'skip', [])
+                    for skip_ext in skip_exts:
+                        if skip_ext in exts:
+                            break
+                    else:
+                        items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=filenames))
+
+        return items or None
+
+    def reportinfo(self):
+        return self.fspath, self.lineno, ''
+
+    @classmethod
+    def from_parent(cls, parent, fspath, **kwargs):
+        return super().from_parent(parent=parent, fspath=fspath, **kwargs)
+
+
+@pytest.hookimpl(hookwrapper=True, trylast=True)
+def pytest_pycollect_makeitem(collector, name: str, obj: object):
+    outcome = yield
+    outcome.get_result()
+    if not callable(obj):
+        outcome.force_result(None)
+        return
+    unittest = getattr(obj, 'unittest', None)
+
+    if not unittest:
+        outcome.force_result(None)
+        return
+
+    if unittest is True:  # no file list to run against, just return a test item instead of a collector
+        outcome.force_result(PyYAMLItem.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj))
+        return
+
+    # there's a file list; return a collector to create individual items for each
+    outcome.force_result(PyYAMLCollector.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj))
+    return
+
+
+def pytest_collection_modifyitems(session, config, items):
+    pass
+
+def pytest_ignore_collect(path, config):
+    basename = path.basename
+    # ignore all Python files in this subtree for normal pytest collection
+    if basename not in ['test_yaml.py', 'test_yaml_ext.py']:
+        return True
+
+    # ignore extension tests (depending on config)
+    if basename == 'test_yaml_ext.py':
+        require_libyaml = os.environ.get('PYYAML_FORCE_LIBYAML', None)
+        if require_libyaml == '1' and not HAS_LIBYAML_EXT:
+            raise RuntimeError('PYYAML_FORCE_LIBYAML envvar is set, but libyaml extension is not available')
+        if require_libyaml == '0':
+            return True
+        if not HAS_LIBYAML_EXT:
+            warnings.warn('libyaml extension is not available, skipping libyaml tests')
+            return True
+