Add basic test coverage configuration

This handles:

* Sub-processes within unit tests (thanks pytest-cov)
* Our pytest-fixture-based virtual environments and subprocesses
  therein
* Running with xdist (e.g. `-n auto`)
* Combining results from all of the above using paths rooted with
  `src/pip/*`

This doesn't handle:

* Platform-specific branches
* Python 2
* CI integration

.coveragerc

@@ -1,4 +0,0 @@
-[run]
-branch = True
-omit =
-  src/pip/_vendor/*

setup.cfg

@@ -59,6 +59,44 @@ markers =
     yaml: yaml based tests
     fails_on_new_resolver: Does not yet work on the new resolver
 
+[coverage:run]
+branch = True
+# Do not gather coverage for vendored libraries.
+omit = */_vendor/*
+# Centralized absolute file prefix for coverage files.
+data_file = ${COVERAGE_OUTPUT_DIR}/.coverage
+# By default, each covered process will try to truncate and then write to
+# `data_file`, but with `parallel`, they will write to separate files suffixed
+# with hostname, pid, and a timestamp.
+parallel = True
+# If not set, then at the termination of each worker (when using pytest-xdist),
+# the following is traced: "Coverage.py warning: Module pip was previously
+# imported, but not measured (module-not-measured)"
+disable_warnings = module-not-measured
+
+[coverage:paths]
+# We intentionally use "source0" here because pytest-cov unconditionally sets
+# "source" after loading the config.
+source0 =
+    # The primary source code path which other paths will be combined into.
+    src/pip/
+    # Unit test source directory e.g.
+    # `.tox/coverage-py3/lib/pythonX.Y/site-packages/pip/...`
+    */site-packages/pip/
+    # Functional test virtual environment directories, which look like
+    # `tmpdir/pip0/pip/src/pip/...`
+    */pip/src/pip/
+
+[coverage:report]
+exclude_lines =
+    # We must re-state the default because the `exclude_lines` option overrides
+    # it.
+    pragma: no cover
+    # This excludes typing-specific code, which will be validated by mypy anyway.
+    if MYPY_CHECK_RUNNING
+    # Can be set to exclude e.g. `if PY2:` on Python 3
+    ${PIP_CI_COVERAGE_EXCLUDES}
+
 [bdist_wheel]
 universal = 1
 

tests/conftest.py

@@ -294,6 +294,13 @@ def wheel_install(tmpdir_factory, common_wheels):
                                           'wheel')
 
 
+@pytest.fixture(scope='session')
+def coverage_install(tmpdir_factory, common_wheels):
+    return _common_wheel_editable_install(tmpdir_factory,
+                                          common_wheels,
+                                          'coverage')
+
+
 def install_egg_link(venv, project_name, egg_info_dir):
     with open(venv.site / 'easy-install.pth', 'a') as fp:
         fp.write(str(egg_info_dir.resolve()) + '\n')
@@ -303,7 +310,7 @@ def install_egg_link(venv, project_name, egg_info_dir):
 
 @pytest.fixture(scope='session')
 def virtualenv_template(request, tmpdir_factory, pip_src,
-                        setuptools_install, common_wheels):
+                        setuptools_install, coverage_install):
 
     if six.PY3 and request.config.getoption('--use-venv'):
         venv_type = 'venv'
@@ -327,6 +334,13 @@ def virtualenv_template(request, tmpdir_factory, pip_src,
     subprocess.check_call([venv.bin / 'python', 'setup.py', '-q', 'develop'],
                           cwd=pip_editable)
 
+    # Install coverage and pth file for executing it in any spawned processes
+    # in this virtual environment.
+    install_egg_link(venv, 'coverage', coverage_install)
+    # zz prefix ensures the file is after easy-install.pth.
+    with open(venv.site / 'zz-coverage-helper.pth', 'a') as f:
+        f.write('import coverage; coverage.process_startup()')
+
     # Drop (non-relocatable) launchers.
     for exe in os.listdir(venv.bin):
         if not (

tools/requirements/tests-common_wheels.txt

@@ -7,3 +7,5 @@
 
 setuptools >= 40.8.0
 wheel
+# As required by pytest-cov.
+coverage >= 4.4

tox.ini

@@ -8,6 +8,7 @@ envlist =
 # Wrapper for calls to pip that make sure the version being used is the
 # original virtualenv (stable) version, and not the code being tested.
 pip = python {toxinidir}/tools/tox_pip.py
+mkdirp = python -c 'import os, sys; os.path.exists(sys.argv[1]) or os.mkdir(sys.argv[1])'
 
 [testenv]
 # Remove USERNAME once we drop PY2.
@@ -30,9 +31,20 @@ commands = pytest --timeout 300 []
 install_command = {[helpers]pip} install {opts} {packages}
 list_dependencies_command = {[helpers]pip} freeze --all
 
-[testenv:coverage-py3]
+[testenv:coverage]
 basepython = python3
-commands = pytest --timeout 300 --cov=pip --cov-report=term-missing --cov-report=xml --cov-report=html tests/unit {posargs}
+commands =
+    {[helpers]mkdirp} {toxinidir}/.coverage-output
+    pytest --timeout 300 --cov=pip --cov-config={toxinidir}/setup.cfg []
+
+setenv =
+    # Used in coverage configuration in setup.cfg.
+    COVERAGE_OUTPUT_DIR = {toxinidir}/.coverage-output
+    # Ensure coverage is enabled in child processes in virtual environments
+    # since they won't already have been enabled by pytest-cov.
+    COVERAGE_PROCESS_START = {toxinidir}/setup.cfg
+    # Used in coverage configuration in setup.cfg.
+    PIP_CI_COVERAGE_EXCLUDES = if PY2
 
 [testenv:docs]
 # Don't skip install here since pip_sphinxext uses pip's internals.