Move tests to pytest

* custom pytest collector and item to surface pyyaml data-driven tests as individual pytest unit tests
* moved some true unit tests to pytest-native tests
* deprecated `setup.py test` custom command
* updated Makefile to use pytest
* align test matrix with planned 7.x Python support

.github/workflows/ci.yaml

@@ -120,22 +120,16 @@ jobs:
     strategy:
       matrix:
         include:
-        - { platform: manylinux1, arch: x86_64, spec: cp36 }
-#        - { platform: manylinux1, arch: x86_64, spec: cp37 }
-#        - { platform: manylinux1, arch: x86_64, spec: cp38 }
+        - { platform: manylinux1, arch: x86_64, spec: cp38 }
 #        - { platform: manylinux1, arch: x86_64, spec: cp39 }
 #        - { platform: manylinux2014, arch: x86_64, spec: cp310 }
 #        - { platform: manylinux2014, arch: x86_64, spec: cp311 }
         - { platform: manylinux2014, arch: x86_64, spec: cp312 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp36 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp37 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp38 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp39 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp310 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp311 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp312 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp36 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp37 }
 #        - { platform: manylinux2014, arch: s390x, spec: cp38 }
 #        - { platform: manylinux2014, arch: s390x, spec: cp39 }
 #        - { platform: manylinux2014, arch: s390x, spec: cp310 }
@@ -176,7 +170,8 @@ jobs:
           LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
         python3 -V
@@ -239,10 +234,6 @@ jobs:
     strategy:
       matrix:
         include:
-#        - spec: cp36-macosx_x86_64
-#          cibw_version: cibuildwheel==2.11.1
-#        - spec: cp37-macosx_x86_64
-#          cibw_version: cibuildwheel==2.11.1
 #        - spec: cp38-macosx_x86_64
 #          cibw_version: cibuildwheel==2.11.1
 #        - spec: cp39-macosx_x86_64
@@ -296,9 +287,12 @@ jobs:
         C_INCLUDE_PATH: libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
         LIBRARY_PATH: libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
+        PYYAML_FORCE_CYTHON: 1
+        PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         python3 -V
@@ -360,17 +354,6 @@ jobs:
     strategy:
       matrix:
         include:
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: 3.6
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: 3.7
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
 #          spec: 3.8
 #        - platform: windows-2019
 #          build_arch: x64
@@ -387,15 +370,7 @@ jobs:
         - platform: windows-2019
           build_arch: x64
           python_arch: x64
-          spec: '3.12.0-rc.1'
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: 3.6
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: 3.7
+          spec: '3.12'
 #        - platform: windows-2019
 #          build_arch: win32
 #          python_arch: x86
@@ -415,7 +390,7 @@ jobs:
         - platform: windows-2019
           build_arch: win32
           python_arch: x86
-          spec: '3.12.0-rc.1'
+          spec: '3.12'
     steps:
     # autocrlf screws up tests under Windows
     - name: Set git to use LF
@@ -458,8 +433,8 @@ jobs:
         build bdist_wheel
 
         # run tests on built wheel
-        python -m pip install dist/*.whl
-        python tests/lib/test_all.py
+        python -m pip install dist/*.whl pytest
+        python -I -m pytest
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

.github/workflows/manual_artifact_build.yaml

@@ -1,7 +1,7 @@
 ---
 # This is the actual artifact build/release workflow. This workflow exists temporarily
 # because GHA doesn't support a dynamic/conditional matrix. Ensure changes are synced with ci.yaml.
-name: Manual Artifact Build
+name: PyYAML CI
 
 on:
 #  push:
@@ -76,7 +76,6 @@ jobs:
         - { platform: manylinux2014, arch: aarch64 }
         - { platform: manylinux2014, arch: s390x }
         - { platform: musllinux_1_1, arch: x86_64 }
-
     env:
       DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}}
     steps:
@@ -119,22 +118,16 @@ jobs:
     strategy:
       matrix:
         include:
-        - { platform: manylinux1, arch: x86_64, spec: cp36 }
-        - { platform: manylinux1, arch: x86_64, spec: cp37 }
         - { platform: manylinux1, arch: x86_64, spec: cp38 }
         - { platform: manylinux1, arch: x86_64, spec: cp39 }
         - { platform: manylinux2014, arch: x86_64, spec: cp310 }
         - { platform: manylinux2014, arch: x86_64, spec: cp311 }
         - { platform: manylinux2014, arch: x86_64, spec: cp312 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp36 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp37 }
         - { platform: manylinux2014, arch: aarch64, spec: cp38 }
         - { platform: manylinux2014, arch: aarch64, spec: cp39 }
         - { platform: manylinux2014, arch: aarch64, spec: cp310 }
         - { platform: manylinux2014, arch: aarch64, spec: cp311 }
         - { platform: manylinux2014, arch: aarch64, spec: cp312 }
-        - { platform: manylinux2014, arch: s390x, spec: cp36 }
-        - { platform: manylinux2014, arch: s390x, spec: cp37 }
         - { platform: manylinux2014, arch: s390x, spec: cp38 }
         - { platform: manylinux2014, arch: s390x, spec: cp39 }
         - { platform: manylinux2014, arch: s390x, spec: cp310 }
@@ -175,7 +168,8 @@ jobs:
           LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
         python3 -V
@@ -238,10 +232,6 @@ jobs:
     strategy:
       matrix:
         include:
-        - spec: cp36-macosx_x86_64
-          cibw_version: cibuildwheel==2.11.1
-        - spec: cp37-macosx_x86_64
-          cibw_version: cibuildwheel==2.11.1
         - spec: cp38-macosx_x86_64
           cibw_version: cibuildwheel==2.11.1
         - spec: cp39-macosx_x86_64
@@ -295,9 +285,12 @@ jobs:
         C_INCLUDE_PATH: libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
         LIBRARY_PATH: libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
+        PYYAML_FORCE_CYTHON: 1
+        PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         python3 -V
@@ -359,17 +352,6 @@ jobs:
     strategy:
       matrix:
         include:
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: 3.6
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: 3.7
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
           spec: 3.8
         - platform: windows-2019
           build_arch: x64
@@ -386,15 +368,7 @@ jobs:
         - platform: windows-2019
           build_arch: x64
           python_arch: x64
-          spec: '3.12.0-rc.1'
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: 3.6
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: 3.7
+          spec: '3.12'
         - platform: windows-2019
           build_arch: win32
           python_arch: x86
@@ -414,7 +388,7 @@ jobs:
         - platform: windows-2019
           build_arch: win32
           python_arch: x86
-          spec: '3.12.0-rc.1'
+          spec: '3.12'
     steps:
     # autocrlf screws up tests under Windows
     - name: Set git to use LF
@@ -457,8 +431,8 @@ jobs:
         build bdist_wheel
 
         # run tests on built wheel
-        python -m pip install dist/*.whl
-        python tests/lib/test_all.py
+        python -m pip install dist/*.whl pytest
+        python -I -m pytest
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

.gitignore

@@ -16,4 +16,4 @@ _pycache_/*
 
 # local IDE state
 /.idea/*
-
+/.tox/*

MANIFEST.in

@@ -2,6 +2,6 @@ include CHANGES README LICENSE Makefile pyproject.toml setup.py
 recursive-include lib/yaml *.py
 recursive-include lib/_yaml *.py
 recursive-include examples *.py *.cfg *.yaml
-recursive-include tests/data *
-recursive-include tests/lib *.py
+recursive-include tests/legacy_tests/ *.py
+recursive-include tests/legacy_tests/data *
 recursive-include yaml *

Makefile

@@ -24,28 +24,25 @@ installext:
 	${PYTHON} setup.py --with-libyaml install ${PARAMETERS}
 
 test: build
-	${PYTHON} tests/lib/test_build.py ${TEST}
+	PYYAML_FORCE_LIBYAML=0 ${PYTHON} -I -m pytest
 
 testext: buildext
-	${PYTHON} tests/lib/test_build_ext.py ${TEST}
+	PYYAML_FORCE_LIBYAML=1 ${PYTHON} -I -m pytest
 
 testall:
-	${PYTHON} setup.py test
+	${PYTHON} -m pytest
 
 dist:
 	@# No longer uploading a zip file to pypi
 	@# ${PYTHON} setup.py --with-libyaml sdist --formats=zip,gztar
 	${PYTHON} setup.py --with-libyaml sdist --formats=gztar
 
-windist:
-	${PYTHON} setup.py --with-libyaml bdist_wininst
-
 clean:
 	${PYTHON} setup.py --with-libyaml clean -a
 	rm -fr \
 	    dist/ \
 	    lib/PyYAML.egg-info/ \
 	    lib/yaml/__pycache__/ \
-	    tests/lib/__pycache__/ \
-	    yaml/_yaml.c \
-
+	    tests/__pycache__/ \
+	    tests/legacy_tests/__pycache__/ \
+	    yaml/_yaml.c

setup.py

@@ -250,6 +250,7 @@ class test(Command):
         pass
 
     def run(self):
+        warnings.warn('Running tests via `setup.py test` is deprecated and will be removed in a future release. Use `pytest` instead to ensure that the complete test suite is run.', DeprecationWarning)
         build_cmd = self.get_finalized_command('build')
         build_cmd.run()
 
@@ -263,7 +264,7 @@ class test(Command):
             temp_test_path = pathlib.Path(tempdir.name) / 'pyyaml'
             shutil.copytree(build_cmd.build_lib, temp_test_path)
             sys.path.insert(0, str(temp_test_path))
-            sys.path.insert(0, 'tests/lib')
+            sys.path.insert(0, 'tests/legacy_tests')
 
             import test_all
             if not test_all.main([]):

tests/legacy_tests/conftest.py

@@ -0,0 +1,130 @@
+# pytest custom collection adapter for legacy pyyaml unit tests/data files; surfaces each
+# legacy test case as a pyyaml item
+
+import os
+import pytest
+import warnings
+
+from test_appliance import find_test_filenames, DATA
+
+try:
+    from yaml import _yaml
+    HAS_LIBYAML_EXT = True
+    del _yaml
+except ImportError:
+    HAS_LIBYAML_EXT = False
+
+
+_test_filenames = find_test_filenames(DATA)
+
+# ignore all datafiles
+collect_ignore_glob = ['data/*']
+
+
+class PyYAMLItem(pytest.Item):
+    def __init__(self, parent=None, config=None, session=None, nodeid=None, function=None, filenames=None, **kwargs):
+        self._function = function
+        self._fargs = filenames or []
+
+        super().__init__(os.path.basename(filenames[0]) if filenames else parent.name, parent, config, session, nodeid)
+        # this is gnarly since the type of fspath is private; fixed in pytest 7 to use pathlib on the `path` attr
+        if filenames:  # pass the data file location as the test path
+            self.fspath = parent.fspath.__class__(filenames[0])
+            self.lineno = 1
+        else:  # pass the function location in the code
+            self.fspath = parent.fspath.__class__(function.__code__.co_filename)
+            self.lineno = function.__code__.co_firstlineno
+
+    def runtest(self):
+        self._function(verbose=True, *self._fargs)
+
+    def reportinfo(self):
+        return self.fspath, self.lineno, ''
+
+
+class PyYAMLCollector(pytest.Collector):
+    def __init__(self, name, parent=None, function=None, **kwargs):
+        self._function = function
+        self.fspath = parent.fspath.__class__(function.__code__.co_filename)
+        self.lineno = function.__code__.co_firstlineno
+
+        # avoid fspath deprecation warnings on pytest < 7
+        if hasattr(self, 'path') and 'fspath' in kwargs:
+            del kwargs['fspath']
+
+        super().__init__(name=name, parent=parent, **kwargs)
+
+    def collect(self):
+        items = []
+
+        unittest = getattr(self._function, 'unittest', None)
+
+        if unittest is True:  # no filenames
+            items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=None))
+        else:
+            for base, exts in _test_filenames:
+                filenames = []
+                for ext in unittest:
+                    if ext not in exts:
+                        break
+                    filenames.append(os.path.join(DATA, base + ext))
+                else:
+                    skip_exts = getattr(self._function, 'skip', [])
+                    for skip_ext in skip_exts:
+                        if skip_ext in exts:
+                            break
+                    else:
+                        items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=filenames))
+
+        return items or None
+
+    def reportinfo(self):
+        return self.fspath, self.lineno, ''
+
+    @classmethod
+    def from_parent(cls, parent, fspath, **kwargs):
+        return super().from_parent(parent=parent, fspath=fspath, **kwargs)
+
+
+@pytest.hookimpl(hookwrapper=True, trylast=True)
+def pytest_pycollect_makeitem(collector, name: str, obj: object):
+    outcome = yield
+    outcome.get_result()
+    if not callable(obj):
+        outcome.force_result(None)
+        return
+    unittest = getattr(obj, 'unittest', None)
+
+    if not unittest:
+        outcome.force_result(None)
+        return
+
+    if unittest is True:  # no file list to run against, just return a test item instead of a collector
+        outcome.force_result(PyYAMLItem.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj))
+        return
+
+    # there's a file list; return a collector to create individual items for each
+    outcome.force_result(PyYAMLCollector.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj))
+    return
+
+
+def pytest_collection_modifyitems(session, config, items):
+    pass
+
+def pytest_ignore_collect(path, config):
+    basename = path.basename
+    # ignore all Python files in this subtree for normal pytest collection
+    if basename not in ['test_yaml.py', 'test_yaml_ext.py']:
+        return True
+
+    # ignore extension tests (depending on config)
+    if basename == 'test_yaml_ext.py':
+        require_libyaml = os.environ.get('PYYAML_FORCE_LIBYAML', None)
+        if require_libyaml == '1' and not HAS_LIBYAML_EXT:
+            raise RuntimeError('PYYAML_FORCE_LIBYAML envvar is set, but libyaml extension is not available')
+        if require_libyaml == '0':
+            return True
+        if not HAS_LIBYAML_EXT:
+            warnings.warn('libyaml extension is not available, skipping libyaml tests')
+            return True
+