diff --git a/.travis.yml b/.travis.yml index 83134951..09b06ede 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,6 +81,10 @@ jobs: python: '3.6' - env: TOXENV=py36-pytest51-xdist29-coverage50 python: '3.6' + - env: TOXENV=py36-pytest52-xdist29-coverage45 + python: '3.6' + - env: TOXENV=py36-pytest52-xdist29-coverage50 + python: '3.6' - env: TOXENV=py37-pytest46-xdist29-coverage45 python: '3.7' - env: TOXENV=py37-pytest46-xdist29-coverage50 @@ -89,6 +93,10 @@ jobs: python: '3.7' - env: TOXENV=py37-pytest51-xdist29-coverage50 python: '3.7' + - env: TOXENV=py37-pytest52-xdist29-coverage45 + python: '3.7' + - env: TOXENV=py37-pytest52-xdist29-coverage50 + python: '3.7' - env: TOXENV=py38-pytest46-xdist29-coverage45 python: '3.8-dev' - env: TOXENV=py38-pytest46-xdist29-coverage50 @@ -97,6 +105,10 @@ jobs: python: '3.8-dev' - env: TOXENV=py38-pytest51-xdist29-coverage50 python: '3.8-dev' + - env: TOXENV=py38-pytest52-xdist29-coverage45 + python: '3.8-dev' + - env: TOXENV=py38-pytest52-xdist29-coverage50 + python: '3.8-dev' - env: TOXENV=pypy3-pytest46-xdist29-coverage45 python: 'pypy3' - env: TOXENV=pypy3-pytest46-xdist29-coverage50 @@ -105,6 +117,10 @@ jobs: python: 'pypy3' - env: TOXENV=pypy3-pytest51-xdist29-coverage50 python: 'pypy3' + - env: TOXENV=pypy3-pytest52-xdist29-coverage45 + python: 'pypy3' + - env: TOXENV=pypy3-pytest52-xdist29-coverage50 + python: 'pypy3' - stage: examples python: '3.6' diff --git a/AUTHORS.rst b/AUTHORS.rst index 206a88bc..03782571 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -30,3 +30,4 @@ Authors * Семён Марьясин - https://github.com/MarSoft * Alexander Shadchin - https://github.com/shadchin * Thomas Grainger - https://graingert.co.uk +* Ned Batchelder - https://nedbatchelder.com diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 01b5ace0..6514982a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,8 @@ Changelog 2.7.2.dev0 (unreleased) ----------------------- +* Added --cov-context option for setting the coverage.py dynamic context for + each test. * Match pytest-xdist master/worker terminology. Contributed in `#321 `_ diff --git a/appveyor.yml b/appveyor.yml index b414a642..bf561d77 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,8 +7,8 @@ environment: - TOXENV: 'py27-pytest310-xdist27-coverage45,py27-pytest46-xdist27-coverage45,py27-pytest310-xdist27-coverage50,py27-pytest46-xdist27-coverage50' - TOXENV: 'py34-pytest310-xdist27-coverage45,py34-pytest46-xdist27-coverage45' - TOXENV: 'py35-pytest310-xdist27-coverage45,py35-pytest46-xdist27-coverage45,py35-pytest310-xdist27-coverage50,py35-pytest46-xdist27-coverage50' - - TOXENV: 'py36-pytest310-xdist27-coverage45,py36-pytest46-xdist27-coverage45,py36-pytest310-xdist27-coverage50,py36-pytest46-xdist27-coverage50,py36-pytest46-xdist29-coverage45,py36-pytest46-xdist29-coverage50,py36-pytest51-xdist29-coverage45,py36-pytest51-xdist29-coverage50' - - TOXENV: 'py37-pytest310-xdist27-coverage45,py37-pytest46-xdist27-coverage45,py37-pytest310-xdist27-coverage50,py37-pytest46-xdist27-coverage50,py37-pytest46-xdist29-coverage45,py37-pytest46-xdist29-coverage50,py37-pytest51-xdist29-coverage45,py37-pytest51-xdist29-coverage50' + - TOXENV: 'py36-pytest310-xdist27-coverage45,py36-pytest46-xdist27-coverage45,py36-pytest310-xdist27-coverage50,py36-pytest46-xdist27-coverage50,py36-pytest46-xdist29-coverage45,py36-pytest46-xdist29-coverage50,py36-pytest51-xdist29-coverage45,py36-pytest51-xdist29-coverage50,py36-pytest52-xdist29-coverage45,py36-pytest52-xdist29-coverage50' + - TOXENV: 'py37-pytest310-xdist27-coverage45,py37-pytest46-xdist27-coverage45,py37-pytest310-xdist27-coverage50,py37-pytest46-xdist27-coverage50,py37-pytest46-xdist29-coverage45,py37-pytest46-xdist29-coverage50,py37-pytest51-xdist29-coverage45,py37-pytest51-xdist29-coverage50,py37-pytest52-xdist29-coverage45,py37-pytest52-xdist29-coverage50' - TOXENV: 'pypy-pytest310-xdist27-coverage45,pypy-pytest46-xdist27-coverage45,pypy-pytest310-xdist27-coverage50,pypy-pytest46-xdist27-coverage50' init: diff --git a/docs/config.rst b/docs/config.rst index 02a82a0e..3e5bf938 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -63,3 +63,4 @@ The complete list of command line options is: --cov-append Do not delete coverage but append to current. Default: False --cov-branch Enable branch coverage. + --cov-context Choose the method for setting the dynamic context. diff --git a/docs/contexts.rst b/docs/contexts.rst new file mode 100644 index 00000000..e5256fcc --- /dev/null +++ b/docs/contexts.rst @@ -0,0 +1,18 @@ +======== +Contexts +======== + +Coverage.py 5.0 can record separate coverage data for different contexts during +one run of a test suite. Pytest-cov can use this feature to record coverage +data for each test individually, with the ``--cov-context=test`` option. + +The context name recorded in the coverage.py database is the pytest test id, +and the phase of execution, one of "setup", "run", or "teardown". These two +are separated with a pipe symbol. You might see contexts like:: + + test_functions.py::test_addition|run + test_fancy.py::test_parametrized[1-101]|setup + test_oldschool.py::RegressionTests::test_error|run + +Note that parameterized tests include the values of the parameters in the test +id, and each set of parameter values is recorded as a separate test. diff --git a/docs/index.rst b/docs/index.rst index 494cd2df..84ff5810 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Contents: debuggers xdist subprocess-support + contexts tox plugins markers-fixtures diff --git a/setup.py b/setup.py index 024d9c36..b3f2115f 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,24 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -from __future__ import absolute_import, print_function +from __future__ import absolute_import +from __future__ import print_function import io -from itertools import chain import re +from distutils.command.build import build from glob import glob +from itertools import chain from os.path import basename from os.path import dirname from os.path import join from os.path import splitext -from distutils.command.build import build from setuptools import Command from setuptools import find_packages from setuptools import setup from setuptools.command.develop import develop -from setuptools.command.install_lib import install_lib from setuptools.command.easy_install import easy_install +from setuptools.command.install_lib import install_lib def read(*names, **kwargs): diff --git a/src/pytest_cov/engine.py b/src/pytest_cov/engine.py index d228d5b5..93a0050e 100644 --- a/src/pytest_cov/engine.py +++ b/src/pytest_cov/engine.py @@ -1,5 +1,4 @@ """Coverage controllers for use by pytest-cov and nose-cov.""" - import contextlib import copy import os @@ -10,8 +9,10 @@ import coverage from coverage.data import CoverageData +from .compat import StringIO +from .compat import workerinput +from .compat import workeroutput from .embed import cleanup -from .compat import StringIO, workeroutput, workerinput class _NullFile(object): @@ -130,8 +131,7 @@ def summary(self, stream): 'file': stream, } skip_covered = isinstance(self.cov_report, dict) and 'skip-covered' in self.cov_report.values() - if hasattr(coverage, 'version_info') and coverage.version_info[0] >= 4: - options.update({'skip_covered': skip_covered or None}) + options.update({'skip_covered': skip_covered or None}) with _backup(self.cov, "config"): total = self.cov.report(**options) diff --git a/src/pytest_cov/plugin.py b/src/pytest_cov/plugin.py index d84a6dbc..fb7a8f54 100644 --- a/src/pytest_cov/plugin.py +++ b/src/pytest_cov/plugin.py @@ -3,6 +3,7 @@ import os import warnings +import coverage import pytest from coverage.misc import CoverageException @@ -48,6 +49,14 @@ def validate_fail_under(num_str): return float(num_str) +def validate_context(arg): + if coverage.version_info <= (5, 0): + raise argparse.ArgumentTypeError('Contexts are only supported with coverage.py >= 5.x') + if arg != "test": + raise argparse.ArgumentTypeError('--cov-context=test is the only supported value') + return arg + + class StoreReport(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): report_type, file = values @@ -88,6 +97,9 @@ def pytest_addoption(parser): 'Default: False') group.addoption('--cov-branch', action='store_true', default=None, help='Enable branch coverage.') + group.addoption('--cov-context', action='store', metavar='CONTEXT', + type=validate_context, + help='Dynamic contexts to use. "test" for now.') def _prepare_cov_source(cov_source): @@ -198,6 +210,9 @@ def pytest_sessionstart(self, session): elif not self._started: self.start(engine.Central) + if self.options.cov_context == 'test': + session.config.pluginmanager.register(TestContextPlugin(self.cov_controller.cov), '_cov_contexts') + def pytest_configure_node(self, node): """Delegate to our implementation. @@ -308,6 +323,24 @@ def pytest_runtest_call(self, item): yield +class TestContextPlugin(object): + def __init__(self, cov): + self.cov = cov + + def pytest_runtest_setup(self, item): + self.switch_context(item, 'setup') + + def pytest_runtest_teardown(self, item): + self.switch_context(item, 'teardown') + + def pytest_runtest_call(self, item): + self.switch_context(item, 'run') + + def switch_context(self, item, when): + context = "{item.nodeid}|{when}".format(item=item, when=when) + self.cov.switch_context(context) + + @pytest.fixture def no_cover(): """A pytest fixture to disable coverage.""" diff --git a/tests/contextful.py b/tests/contextful.py new file mode 100644 index 00000000..3527e499 --- /dev/null +++ b/tests/contextful.py @@ -0,0 +1,105 @@ +# A test file for test_pytest_cov.py:test_contexts + +import unittest + +import pytest + + +def test_01(): + assert 1 == 1 # r1 + + +def test_02(): + assert 2 == 2 # r2 + + +class OldStyleTests(unittest.TestCase): + items = [] + + @classmethod + def setUpClass(cls): + cls.items.append("hello") # s3 + + @classmethod + def tearDownClass(cls): + cls.items.pop() # t4 + + def setUp(self): + self.number = 1 # r3 r4 + + def tearDown(self): + self.number = None # r3 r4 + + def test_03(self): + assert self.number == 1 # r3 + assert self.items[0] == "hello" # r3 + + def test_04(self): + assert self.number == 1 # r4 + assert self.items[0] == "hello" # r4 + + +@pytest.fixture +def some_data(): + return [1, 2, 3] # s5 s6 + + +def test_05(some_data): + assert len(some_data) == 3 # r5 + + +@pytest.fixture +def more_data(some_data): + return [2*x for x in some_data] # s6 + + +def test_06(some_data, more_data): + assert len(some_data) == len(more_data) # r6 + + +@pytest.fixture(scope='session') +def expensive_data(): + return list(range(10)) # s7 + + +def test_07(expensive_data): + assert len(expensive_data) == 10 # r7 + + +def test_08(expensive_data): + assert len(expensive_data) == 10 # r8 + + +@pytest.fixture(params=[1, 2, 3]) +def parametrized_number(request): + return request.param # s9-1 s9-2 s9-3 + + +def test_09(parametrized_number): + assert parametrized_number > 0 # r9-1 r9-2 r9-3 + + +def test_10(): + assert 1 == 1 # r10 + + +@pytest.mark.parametrize("x, ans", [ + (1, 101), + (2, 202), +]) +def test_11(x, ans): + assert 100 * x + x == ans # r11-1 r11-2 + + +@pytest.mark.parametrize("x, ans", [ + (1, 101), + (2, 202), +], ids=['one', 'two']) +def test_12(x, ans): + assert 100 * x + x == ans # r12-1 r12-2 + + +@pytest.mark.parametrize("x", [1, 2]) +@pytest.mark.parametrize("y", [3, 4]) +def test_13(x, y): + assert x + y > 0 # r13-1 r13-2 r13-3 r13-4 diff --git a/tests/test_pytest_cov.py b/tests/test_pytest_cov.py index 6d0e7e78..adc28d54 100644 --- a/tests/test_pytest_cov.py +++ b/tests/test_pytest_cov.py @@ -1,9 +1,10 @@ +import collections import glob import os import platform +import re import subprocess import sys -from distutils.version import StrictVersion from itertools import chain import coverage @@ -17,15 +18,15 @@ from process_tests import wait_for_strings from six import exec_ +import pytest_cov.plugin +from pytest_cov import compat + try: from StringIO import StringIO except ImportError: from io import StringIO -import pytest_cov.plugin -from pytest_cov import compat - -coverage, platform, StrictVersion # required for skipif mark on test_cov_min_from_coveragerc +coverage, platform # required for skipif mark on test_cov_min_from_coveragerc max_worker_restart_0 = "--max-" + compat.worker + "-restart=0" @@ -468,7 +469,6 @@ def test_central_nonspecific(testdir, prop): assert result.ret == 0 -@pytest.mark.skipif('StrictVersion(coverage.__version__) <= StrictVersion("3.8")') def test_cov_min_from_coveragerc(testdir): script = testdir.makepyfile(SCRIPT) testdir.tmpdir.join('.coveragerc').write(""" @@ -1636,7 +1636,6 @@ def test_basic(): SKIP_COVERED_RESULT = '1 file skipped due to complete coverage.' -@pytest.mark.skipif('StrictVersion(coverage.__version__) < StrictVersion("4.0")') @pytest.mark.parametrize('report_option', [ 'term-missing:skip-covered', 'term:skip-covered']) @@ -1651,7 +1650,6 @@ def test_skip_covered_cli(testdir, report_option): result.stdout.fnmatch_lines([SKIP_COVERED_RESULT]) -@pytest.mark.skipif('StrictVersion(coverage.__version__) < StrictVersion("4.0")') def test_skip_covered_coveragerc_config(testdir): testdir.makefile('', coveragerc=SKIP_COVERED_COVERAGERC) script = testdir.makepyfile(SKIP_COVERED_TEST) @@ -1932,3 +1930,95 @@ def test_cov_and_no_cov(testdir): script) assert result.ret == 0 + + +def find_labels(text, pattern): + all_labels = collections.defaultdict(list) + lines = text.splitlines() + for lineno, line in enumerate(lines, start=1): + labels = re.findall(pattern, line) + for label in labels: + all_labels[label].append(lineno) + return all_labels + + +# The contexts and their labels in contextful.py +EXPECTED_CONTEXTS = { + '': 'c0', + 'test_contexts.py::test_01|run': 'r1', + 'test_contexts.py::test_02|run': 'r2', + 'test_contexts.py::OldStyleTests::test_03|setup': 's3', + 'test_contexts.py::OldStyleTests::test_03|run': 'r3', + 'test_contexts.py::OldStyleTests::test_04|run': 'r4', + 'test_contexts.py::OldStyleTests::test_04|teardown': 't4', + 'test_contexts.py::test_05|setup': 's5', + 'test_contexts.py::test_05|run': 'r5', + 'test_contexts.py::test_06|setup': 's6', + 'test_contexts.py::test_06|run': 'r6', + 'test_contexts.py::test_07|setup': 's7', + 'test_contexts.py::test_07|run': 'r7', + 'test_contexts.py::test_08|run': 'r8', + 'test_contexts.py::test_09[1]|setup': 's9-1', + 'test_contexts.py::test_09[1]|run': 'r9-1', + 'test_contexts.py::test_09[2]|setup': 's9-2', + 'test_contexts.py::test_09[2]|run': 'r9-2', + 'test_contexts.py::test_09[3]|setup': 's9-3', + 'test_contexts.py::test_09[3]|run': 'r9-3', + 'test_contexts.py::test_10|run': 'r10', + 'test_contexts.py::test_11[1-101]|run': 'r11-1', + 'test_contexts.py::test_11[2-202]|run': 'r11-2', + 'test_contexts.py::test_12[one]|run': 'r12-1', + 'test_contexts.py::test_12[two]|run': 'r12-2', + 'test_contexts.py::test_13[3-1]|run': 'r13-1', + 'test_contexts.py::test_13[3-2]|run': 'r13-2', + 'test_contexts.py::test_13[4-1]|run': 'r13-3', + 'test_contexts.py::test_13[4-2]|run': 'r13-4', +} + + +@pytest.mark.skipif("coverage.version_info < (5, 0)") +@xdist_params +def test_contexts(testdir, opts): + with open(os.path.join(os.path.dirname(__file__), "contextful.py")) as f: + contextful_tests = f.read() + script = testdir.makepyfile(contextful_tests) + result = testdir.runpytest('-v', + '--cov=%s' % script.dirpath(), + '--cov-context=test', + script, + *opts.split() + ) + assert result.ret == 0 + result.stdout.fnmatch_lines([ + 'test_contexts* 100%*', + ]) + + data = coverage.CoverageData(".coverage") + data.read() + assert data.measured_contexts() == set(EXPECTED_CONTEXTS) + measured = data.measured_files() + assert len(measured) == 1 + test_context_path = list(measured)[0] + assert test_context_path.lower() == os.path.abspath("test_contexts.py").lower() + + line_data = find_labels(contextful_tests, r"[crst]\d+(?:-\d+)?") + for context, label in EXPECTED_CONTEXTS.items(): + if context == '': + continue + data.set_query_context(context) + actual = data.lines(test_context_path) + assert line_data[label] == actual, "Wrong lines for context {!r}".format(context) + + +@pytest.mark.skipif("coverage.version_info >= (5, 0)") +def test_contexts_not_supported(testdir): + script = testdir.makepyfile("a = 1") + result = testdir.runpytest('-v', + '--cov=%s' % script.dirpath(), + '--cov-context=test', + script, + ) + result.stderr.fnmatch_lines([ + '*argument --cov-context: Contexts are only supported with coverage.py >= 5.x', + ]) + assert result.ret != 0 diff --git a/tox.ini b/tox.ini index a546659d..fd5a9f5c 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = check py{27,34,35,36,37,py,py3}-pytest{310,46}-xdist27-coverage45 py{27,35,36,37,py,py3}-pytest{310,46}-xdist27-coverage50 - py{36,37,38,py3}-pytest{46,51}-xdist29-coverage{45,50} + py{36,37,38,py3}-pytest{46,51,52}-xdist29-coverage{45,50} docs [testenv] @@ -21,7 +21,8 @@ setenv = pytest44: _DEP_PYTEST=pytest==4.4.2 pytest45: _DEP_PYTEST=pytest==4.5.0 pytest46: _DEP_PYTEST=pytest==4.6.5 - pytest51: _DEP_PYTEST=pytest==5.1.0 + pytest51: _DEP_PYTEST=pytest==5.1.3 + pytest52: _DEP_PYTEST=pytest==5.2.0 xdist22: _DEP_PYTESTXDIST=pytest-xdist==1.22.0 xdist27: _DEP_PYTESTXDIST=pytest-xdist==1.27.0 @@ -29,7 +30,7 @@ setenv = xdist29: _DEP_PYTESTXDIST=pytest-xdist==1.29.0 coverage45: _DEP_COVERAGE=coverage==4.5.4 - coverage50: _DEP_COVERAGE=coverage==5.0a7 + coverage50: _DEP_COVERAGE=coverage==5.0a8 passenv = * @@ -66,9 +67,11 @@ deps = flake8 readme-renderer pygments + isort skip_install = true usedevelop = false commands = python setup.py check --strict --metadata --restructuredtext check-manifest {toxinidir} flake8 src tests setup.py + isort --check-only --diff --recursive src tests setup.py