tests: speedup testsuite

Speedup virtualenv creation: create one (per session) relocatable
virtual environment, and then just make a copy of the resulting tree
when a new virtualenv is needed.

tests/conftest.py

@@ -8,7 +8,6 @@ import pytest
 import six
 
 import pip._internal
-from pip._internal.utils import appdirs
 from tests.lib import SRC_DIR, TestData
 from tests.lib.path import Path
 from tests.lib.scripttest import PipTestEnvironment
@@ -124,15 +123,9 @@ def isolate(tmpdir):
         )
 
 
-@pytest.fixture
-def virtualenv(tmpdir, isolate):
-    """
-    Return a virtual environment which is unique to each test function
-    invocation created inside of a sub directory of the test function's
-    temporary directory. The returned object is a
-    ``tests.lib.venv.VirtualEnvironment`` object.
-    """
-
+@pytest.yield_fixture(scope='session')
+def virtualenv_template(tmpdir_factory):
+    tmpdir = Path(str(tmpdir_factory.mktemp('virtualenv')))
     # Copy over our source tree so that each virtual environment is self
     # contained
     pip_src = tmpdir.join("pip_src").abspath
@@ -144,18 +137,33 @@ def virtualenv(tmpdir, isolate):
             "tests", "pip.egg-info", "build", "dist", ".tox", ".git",
         ),
     )
-
     # Create the virtual environment
     venv = VirtualEnvironment.create(
-        tmpdir.join("workspace", "venv"),
+        tmpdir.join("venv_orig"),
         pip_source_dir=pip_src,
+        relocatable=True,
     )
+    # Rename original virtualenv directory to make sure
+    # it's not reused by mistake from one of the copies.
+    venv_template = tmpdir / "venv_template"
+    os.rename(venv.location, venv_template)
+    yield venv_template
+    tmpdir.rmtree(noerrors=True)
 
-    # Clean out our cache: creating the venv injects wheels into it.
-    if os.path.exists(appdirs.user_cache_dir("pip")):
-        shutil.rmtree(appdirs.user_cache_dir("pip"))
 
-    return venv
+@pytest.yield_fixture
+def virtualenv(virtualenv_template, tmpdir, isolate):
+    """
+    Return a virtual environment which is unique to each test function
+    invocation created inside of a sub directory of the test function's
+    temporary directory. The returned object is a
+    ``tests.lib.venv.VirtualEnvironment`` object.
+    """
+    venv_location = tmpdir.join("workspace", "venv")
+    shutil.copytree(virtualenv_template, venv_location, symlinks=True)
+    venv = VirtualEnvironment(venv_location)
+    yield venv
+    venv_location.rmtree(noerrors=True)
 
 
 @pytest.fixture

tests/lib/venv.py

@@ -1,19 +1,12 @@
 from __future__ import absolute_import
 
-import os
-import subprocess
+import distutils
 
 import virtualenv as _virtualenv
 
 from . import virtualenv_lib_path
 from .path import Path
 
-# On Python < 3.3 we don't have subprocess.DEVNULL
-try:
-    DEVNULL = subprocess.DEVNULL
-except AttributeError:
-    DEVNULL = open(os.devnull, "wb")
-
 
 class VirtualEnvironment(object):
     """
@@ -21,27 +14,26 @@ class VirtualEnvironment(object):
     virtualenv but in the future it could use pyvenv.
     """
 
-    def __init__(self, location, *args, **kwargs):
+    def __init__(self, location, system_site_packages=False):
         self.location = Path(location)
-        self.pip_source_dir = kwargs.pop("pip_source_dir")
-        self._system_site_packages = kwargs.pop("system_site_packages", False)
-
+        self._system_site_packages = system_site_packages
         home, lib, inc, bin = _virtualenv.path_locations(self.location)
         self.lib = Path(virtualenv_lib_path(home, lib))
         self.bin = Path(bin)
 
-        super(VirtualEnvironment, self).__init__(*args, **kwargs)
-
     def __repr__(self):
         return "<VirtualEnvironment {0}>".format(self.location)
 
     @classmethod
-    def create(cls, location, clear=False, pip_source_dir=None):
-        obj = cls(location, pip_source_dir=pip_source_dir)
-        obj._create(clear=clear)
+    def create(cls, location, clear=False,
+               pip_source_dir=None, relocatable=False):
+        obj = cls(location)
+        obj._create(clear=clear,
+                    pip_source_dir=pip_source_dir,
+                    relocatable=relocatable)
         return obj
 
-    def _create(self, clear=False):
+    def _create(self, clear=False, pip_source_dir=None, relocatable=False):
         # Create the actual virtual environment
         _virtualenv.create_environment(
             self.location,
@@ -50,23 +42,13 @@ class VirtualEnvironment(object):
             no_pip=True,
             no_wheel=True,
         )
-
-        # Install our development version of pip install the virtual
-        # environment
-        cmd = [self.bin.join("python"), "setup.py", "install", "--no-compile"]
-        p = subprocess.Popen(
-            cmd,
-            cwd=self.pip_source_dir,
-            stderr=subprocess.STDOUT,
-            stdout=DEVNULL,
-        )
-        p.communicate()
-        if p.returncode != 0:
-            raise subprocess.CalledProcessError(
-                p.returncode,
-                cmd,
-                output=p.stdout,
-            )
+        _virtualenv.install_wheel([pip_source_dir or '.'],
+                                  self.bin.join("python"))
+        if relocatable:
+            _virtualenv.make_environment_relocatable(self.location)
+        # FIXME: some tests rely on 'easy-install.pth' being already present.
+        site_package = distutils.sysconfig.get_python_lib(prefix=self.location)
+        Path(site_package).join('easy-install.pth').touch()
 
     def clear(self):
         self._create(clear=True)