Move tests to pytest

* custom pytest collector and item to surface pyyaml data-driven tests as individual pytest unit tests
* moved some true unit tests to pytest-native tests
* deprecated `setup.py test` custom command
* updated Makefile to use pytest
* align test matrix with planned 7.x Python support
This commit is contained in:
Matt Davis 2021-11-24 16:37:09 -08:00
parent 155ec463f6
commit a98fd6088e
No known key found for this signature in database
GPG Key ID: 270F53F3C6D92108
606 changed files with 184 additions and 117 deletions

View File

@ -120,22 +120,16 @@ jobs:
strategy:
matrix:
include:
- { platform: manylinux1, arch: x86_64, spec: cp36 }
# - { platform: manylinux1, arch: x86_64, spec: cp37 }
# - { platform: manylinux1, arch: x86_64, spec: cp38 }
- { platform: manylinux1, arch: x86_64, spec: cp38 }
# - { platform: manylinux1, arch: x86_64, spec: cp39 }
# - { platform: manylinux2014, arch: x86_64, spec: cp310 }
# - { platform: manylinux2014, arch: x86_64, spec: cp311 }
- { platform: manylinux2014, arch: x86_64, spec: cp312 }
# - { platform: manylinux2014, arch: aarch64, spec: cp36 }
# - { platform: manylinux2014, arch: aarch64, spec: cp37 }
# - { platform: manylinux2014, arch: aarch64, spec: cp38 }
# - { platform: manylinux2014, arch: aarch64, spec: cp39 }
# - { platform: manylinux2014, arch: aarch64, spec: cp310 }
# - { platform: manylinux2014, arch: aarch64, spec: cp311 }
# - { platform: manylinux2014, arch: aarch64, spec: cp312 }
# - { platform: manylinux2014, arch: s390x, spec: cp36 }
# - { platform: manylinux2014, arch: s390x, spec: cp37 }
# - { platform: manylinux2014, arch: s390x, spec: cp38 }
# - { platform: manylinux2014, arch: s390x, spec: cp39 }
# - { platform: manylinux2014, arch: s390x, spec: cp310 }
@ -176,7 +170,8 @@ jobs:
LD_LIBRARY_PATH=libyaml/src/.libs
PYYAML_FORCE_CYTHON=1
PYYAML_FORCE_LIBYAML=1
CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
CIBW_TEST_COMMAND: cd {project}; pytest
CIBW_TEST_REQUIRES: pytest
run: |
set -eux
python3 -V
@ -239,10 +234,6 @@ jobs:
strategy:
matrix:
include:
# - spec: cp36-macosx_x86_64
# cibw_version: cibuildwheel==2.11.1
# - spec: cp37-macosx_x86_64
# cibw_version: cibuildwheel==2.11.1
# - spec: cp38-macosx_x86_64
# cibw_version: cibuildwheel==2.11.1
# - spec: cp39-macosx_x86_64
@ -296,9 +287,12 @@ jobs:
C_INCLUDE_PATH: libyaml/include
CIBW_BUILD: ${{matrix.spec}}
CIBW_BUILD_VERBOSITY: 1
CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
CIBW_TEST_COMMAND: cd {project}; pytest
CIBW_TEST_REQUIRES: pytest
LIBRARY_PATH: libyaml/src/.libs
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
PYYAML_FORCE_CYTHON: 1
PYYAML_FORCE_LIBYAML: 1
SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
run: |
python3 -V
@ -360,17 +354,6 @@ jobs:
strategy:
matrix:
include:
# - platform: windows-2019
# build_arch: x64
# python_arch: x64
# spec: 3.6
# - platform: windows-2019
# build_arch: x64
# python_arch: x64
# spec: 3.7
# - platform: windows-2019
# build_arch: x64
# python_arch: x64
# spec: 3.8
# - platform: windows-2019
# build_arch: x64
@ -387,15 +370,7 @@ jobs:
- platform: windows-2019
build_arch: x64
python_arch: x64
spec: '3.12.0-rc.1'
# - platform: windows-2019
# build_arch: win32
# python_arch: x86
# spec: 3.6
# - platform: windows-2019
# build_arch: win32
# python_arch: x86
# spec: 3.7
spec: '3.12'
# - platform: windows-2019
# build_arch: win32
# python_arch: x86
@ -415,7 +390,7 @@ jobs:
- platform: windows-2019
build_arch: win32
python_arch: x86
spec: '3.12.0-rc.1'
spec: '3.12'
steps:
# autocrlf screws up tests under Windows
- name: Set git to use LF
@ -458,8 +433,8 @@ jobs:
build bdist_wheel
# run tests on built wheel
python -m pip install dist/*.whl
python tests/lib/test_all.py
python -m pip install dist/*.whl pytest
python -I -m pytest
- name: Upload artifacts
uses: actions/upload-artifact@v3

View File

@ -1,7 +1,7 @@
---
# This is the actual artifact build/release workflow. This workflow exists temporarily
# because GHA doesn't support a dynamic/conditional matrix. Ensure changes are synced with ci.yaml.
name: Manual Artifact Build
name: PyYAML CI
on:
# push:
@ -76,7 +76,6 @@ jobs:
- { platform: manylinux2014, arch: aarch64 }
- { platform: manylinux2014, arch: s390x }
- { platform: musllinux_1_1, arch: x86_64 }
env:
DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}}
steps:
@ -119,22 +118,16 @@ jobs:
strategy:
matrix:
include:
- { platform: manylinux1, arch: x86_64, spec: cp36 }
- { platform: manylinux1, arch: x86_64, spec: cp37 }
- { platform: manylinux1, arch: x86_64, spec: cp38 }
- { platform: manylinux1, arch: x86_64, spec: cp39 }
- { platform: manylinux2014, arch: x86_64, spec: cp310 }
- { platform: manylinux2014, arch: x86_64, spec: cp311 }
- { platform: manylinux2014, arch: x86_64, spec: cp312 }
- { platform: manylinux2014, arch: aarch64, spec: cp36 }
- { platform: manylinux2014, arch: aarch64, spec: cp37 }
- { platform: manylinux2014, arch: aarch64, spec: cp38 }
- { platform: manylinux2014, arch: aarch64, spec: cp39 }
- { platform: manylinux2014, arch: aarch64, spec: cp310 }
- { platform: manylinux2014, arch: aarch64, spec: cp311 }
- { platform: manylinux2014, arch: aarch64, spec: cp312 }
- { platform: manylinux2014, arch: s390x, spec: cp36 }
- { platform: manylinux2014, arch: s390x, spec: cp37 }
- { platform: manylinux2014, arch: s390x, spec: cp38 }
- { platform: manylinux2014, arch: s390x, spec: cp39 }
- { platform: manylinux2014, arch: s390x, spec: cp310 }
@ -175,7 +168,8 @@ jobs:
LD_LIBRARY_PATH=libyaml/src/.libs
PYYAML_FORCE_CYTHON=1
PYYAML_FORCE_LIBYAML=1
CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
CIBW_TEST_COMMAND: cd {project}; pytest
CIBW_TEST_REQUIRES: pytest
run: |
set -eux
python3 -V
@ -238,10 +232,6 @@ jobs:
strategy:
matrix:
include:
- spec: cp36-macosx_x86_64
cibw_version: cibuildwheel==2.11.1
- spec: cp37-macosx_x86_64
cibw_version: cibuildwheel==2.11.1
- spec: cp38-macosx_x86_64
cibw_version: cibuildwheel==2.11.1
- spec: cp39-macosx_x86_64
@ -295,9 +285,12 @@ jobs:
C_INCLUDE_PATH: libyaml/include
CIBW_BUILD: ${{matrix.spec}}
CIBW_BUILD_VERBOSITY: 1
CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
CIBW_TEST_COMMAND: cd {project}; pytest
CIBW_TEST_REQUIRES: pytest
LIBRARY_PATH: libyaml/src/.libs
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
PYYAML_FORCE_CYTHON: 1
PYYAML_FORCE_LIBYAML: 1
SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
run: |
python3 -V
@ -359,17 +352,6 @@ jobs:
strategy:
matrix:
include:
- platform: windows-2019
build_arch: x64
python_arch: x64
spec: 3.6
- platform: windows-2019
build_arch: x64
python_arch: x64
spec: 3.7
- platform: windows-2019
build_arch: x64
python_arch: x64
spec: 3.8
- platform: windows-2019
build_arch: x64
@ -386,15 +368,7 @@ jobs:
- platform: windows-2019
build_arch: x64
python_arch: x64
spec: '3.12.0-rc.1'
- platform: windows-2019
build_arch: win32
python_arch: x86
spec: 3.6
- platform: windows-2019
build_arch: win32
python_arch: x86
spec: 3.7
spec: '3.12'
- platform: windows-2019
build_arch: win32
python_arch: x86
@ -414,7 +388,7 @@ jobs:
- platform: windows-2019
build_arch: win32
python_arch: x86
spec: '3.12.0-rc.1'
spec: '3.12'
steps:
# autocrlf screws up tests under Windows
- name: Set git to use LF
@ -457,8 +431,8 @@ jobs:
build bdist_wheel
# run tests on built wheel
python -m pip install dist/*.whl
python tests/lib/test_all.py
python -m pip install dist/*.whl pytest
python -I -m pytest
- name: Upload artifacts
uses: actions/upload-artifact@v3

2
.gitignore vendored
View File

@ -16,4 +16,4 @@ _pycache_/*
# local IDE state
/.idea/*
/.tox/*

View File

@ -2,6 +2,6 @@ include CHANGES README LICENSE Makefile pyproject.toml setup.py
recursive-include lib/yaml *.py
recursive-include lib/_yaml *.py
recursive-include examples *.py *.cfg *.yaml
recursive-include tests/data *
recursive-include tests/lib *.py
recursive-include tests/legacy_tests/ *.py
recursive-include tests/legacy_tests/data *
recursive-include yaml *

View File

@ -24,28 +24,25 @@ installext:
${PYTHON} setup.py --with-libyaml install ${PARAMETERS}
test: build
${PYTHON} tests/lib/test_build.py ${TEST}
PYYAML_FORCE_LIBYAML=0 ${PYTHON} -I -m pytest
testext: buildext
${PYTHON} tests/lib/test_build_ext.py ${TEST}
PYYAML_FORCE_LIBYAML=1 ${PYTHON} -I -m pytest
testall:
${PYTHON} setup.py test
${PYTHON} -m pytest
dist:
@# No longer uploading a zip file to pypi
@# ${PYTHON} setup.py --with-libyaml sdist --formats=zip,gztar
${PYTHON} setup.py --with-libyaml sdist --formats=gztar
windist:
${PYTHON} setup.py --with-libyaml bdist_wininst
clean:
${PYTHON} setup.py --with-libyaml clean -a
rm -fr \
dist/ \
lib/PyYAML.egg-info/ \
lib/yaml/__pycache__/ \
tests/lib/__pycache__/ \
yaml/_yaml.c \
tests/__pycache__/ \
tests/legacy_tests/__pycache__/ \
yaml/_yaml.c

View File

@ -250,6 +250,7 @@ class test(Command):
pass
def run(self):
warnings.warn('Running tests via `setup.py test` is deprecated and will be removed in a future release. Use `pytest` instead to ensure that the complete test suite is run.', DeprecationWarning)
build_cmd = self.get_finalized_command('build')
build_cmd.run()
@ -263,7 +264,7 @@ class test(Command):
temp_test_path = pathlib.Path(tempdir.name) / 'pyyaml'
shutil.copytree(build_cmd.build_lib, temp_test_path)
sys.path.insert(0, str(temp_test_path))
sys.path.insert(0, 'tests/lib')
sys.path.insert(0, 'tests/legacy_tests')
import test_all
if not test_all.main([]):

View File

@ -0,0 +1,130 @@
# pytest custom collection adapter for legacy pyyaml unit tests/data files; surfaces each
# legacy test case as a pyyaml item
import os
import pytest
import warnings
from test_appliance import find_test_filenames, DATA
try:
from yaml import _yaml
HAS_LIBYAML_EXT = True
del _yaml
except ImportError:
HAS_LIBYAML_EXT = False
_test_filenames = find_test_filenames(DATA)
# ignore all datafiles
collect_ignore_glob = ['data/*']
class PyYAMLItem(pytest.Item):
def __init__(self, parent=None, config=None, session=None, nodeid=None, function=None, filenames=None, **kwargs):
self._function = function
self._fargs = filenames or []
super().__init__(os.path.basename(filenames[0]) if filenames else parent.name, parent, config, session, nodeid)
# this is gnarly since the type of fspath is private; fixed in pytest 7 to use pathlib on the `path` attr
if filenames: # pass the data file location as the test path
self.fspath = parent.fspath.__class__(filenames[0])
self.lineno = 1
else: # pass the function location in the code
self.fspath = parent.fspath.__class__(function.__code__.co_filename)
self.lineno = function.__code__.co_firstlineno
def runtest(self):
self._function(verbose=True, *self._fargs)
def reportinfo(self):
return self.fspath, self.lineno, ''
class PyYAMLCollector(pytest.Collector):
def __init__(self, name, parent=None, function=None, **kwargs):
self._function = function
self.fspath = parent.fspath.__class__(function.__code__.co_filename)
self.lineno = function.__code__.co_firstlineno
# avoid fspath deprecation warnings on pytest < 7
if hasattr(self, 'path') and 'fspath' in kwargs:
del kwargs['fspath']
super().__init__(name=name, parent=parent, **kwargs)
def collect(self):
items = []
unittest = getattr(self._function, 'unittest', None)
if unittest is True: # no filenames
items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=None))
else:
for base, exts in _test_filenames:
filenames = []
for ext in unittest:
if ext not in exts:
break
filenames.append(os.path.join(DATA, base + ext))
else:
skip_exts = getattr(self._function, 'skip', [])
for skip_ext in skip_exts:
if skip_ext in exts:
break
else:
items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=filenames))
return items or None
def reportinfo(self):
return self.fspath, self.lineno, ''
@classmethod
def from_parent(cls, parent, fspath, **kwargs):
return super().from_parent(parent=parent, fspath=fspath, **kwargs)
@pytest.hookimpl(hookwrapper=True, trylast=True)
def pytest_pycollect_makeitem(collector, name: str, obj: object):
outcome = yield
outcome.get_result()
if not callable(obj):
outcome.force_result(None)
return
unittest = getattr(obj, 'unittest', None)
if not unittest:
outcome.force_result(None)
return
if unittest is True: # no file list to run against, just return a test item instead of a collector
outcome.force_result(PyYAMLItem.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj))
return
# there's a file list; return a collector to create individual items for each
outcome.force_result(PyYAMLCollector.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj))
return
def pytest_collection_modifyitems(session, config, items):
pass
def pytest_ignore_collect(path, config):
basename = path.basename
# ignore all Python files in this subtree for normal pytest collection
if basename not in ['test_yaml.py', 'test_yaml_ext.py']:
return True
# ignore extension tests (depending on config)
if basename == 'test_yaml_ext.py':
require_libyaml = os.environ.get('PYYAML_FORCE_LIBYAML', None)
if require_libyaml == '1' and not HAS_LIBYAML_EXT:
raise RuntimeError('PYYAML_FORCE_LIBYAML envvar is set, but libyaml extension is not available')
if require_libyaml == '0':
return True
if not HAS_LIBYAML_EXT:
warnings.warn('libyaml extension is not available, skipping libyaml tests')
return True

Some files were not shown because too many files have changed in this diff Show More