Issue #2563: Read cached wheels from ~/.cache/pip

This won't put wheels into that directory, but will read them if they
are there. --no-cache-dir will disable reading such wheels.

docs/reference/pip_install.rst

@@ -388,6 +388,23 @@ Windows
   :file:`<CSIDL_LOCAL_APPDATA>\\pip\\Cache`
 
 
+Wheel cache
+***********
+
+Pip will read from the subdirectory ``wheels`` within the pip cache dir and use
+any packages found there. This is disabled via the same ``no-cache-dir`` option
+that disables the HTTP cache. The internal structure of that cache is not part
+of the Pip API. As of 7.0 pip uses a subdirectory per sdist that wheels were
+built from, and wheels within that subdirectory.
+
+Pip attempts to choose the best wheels from those built in preference to
+building a new wheel. Note that this means when a package has both optional
+C extensions and builds `py` tagged wheels when the C extension can't be built
+that pip will not attempt to build a better wheel for Python's that would have
+supported it, once any generic wheel is built. To correct this, make sure that
+the wheel's are built with Python specific tags - e.g. pp on Pypy.
+
+
 Hash Verification
 +++++++++++++++++
 

pip/basecommand.py

@@ -289,6 +289,7 @@ class RequirementCommand(Command):
             requirement_set.add_requirement(
                 InstallRequirement.from_line(
                     name, None, isolated=options.isolated_mode,
+                    cache_root=options.cache_dir
                 )
             )
 
@@ -298,6 +299,7 @@ class RequirementCommand(Command):
                     name,
                     default_vcs=options.default_vcs,
                     isolated=options.isolated_mode,
+                    cache_root=options.cache_dir
                 )
             )
 

pip/commands/freeze.py

@@ -60,7 +60,8 @@ class FreezeCommand(Command):
             local_only=options.local,
             user_only=options.user,
             skip_regex=options.skip_requirements_regex,
-            isolated=options.isolated_mode)
+            isolated=options.isolated_mode,
+            cache_root=options.cache_dir)
 
         for line in freeze(**freeze_kwargs):
             sys.stdout.write(line + '\n')

pip/commands/install.py

@@ -239,6 +239,7 @@ class InstallCommand(RequirementCommand):
                                 delete=build_delete) as build_dir:
                 requirement_set = RequirementSet(
                     build_dir=build_dir,
+                    cache_root=options.cache_dir,
                     src_dir=options.src_dir,
                     download_dir=options.download_dir,
                     upgrade=options.upgrade,

pip/commands/list.py

@@ -131,6 +131,7 @@ class ListCommand(Command):
             for dist in installed_packages:
                 req = InstallRequirement.from_line(
                     dist.key, None, isolated=options.isolated_mode,
+                    cache_root=options.cache_dir,
                 )
                 typ = 'unknown'
                 try:

pip/commands/uninstall.py

@@ -45,6 +45,7 @@ class UninstallCommand(Command):
 
             requirement_set = RequirementSet(
                 build_dir=None,
+                cache_root=options.cache_dir,
                 src_dir=None,
                 download_dir=None,
                 isolated=options.isolated_mode,
@@ -54,13 +55,15 @@ class UninstallCommand(Command):
                 requirement_set.add_requirement(
                     InstallRequirement.from_line(
                         name, isolated=options.isolated_mode,
+                        cache_root=options.cache_dir,
                     )
                 )
             for filename in options.requirements:
                 for req in parse_requirements(
                         filename,
                         options=options,
-                        session=session):
+                        session=session,
+                        cache_root=options.cache_dir):
                     requirement_set.add_requirement(req)
             if not requirement_set.has_requirements:
                 raise InstallationError(

pip/commands/wheel.py

@@ -159,6 +159,7 @@ class WheelCommand(RequirementCommand):
                                 delete=build_delete) as build_dir:
                 requirement_set = RequirementSet(
                     build_dir=build_dir,
+                    cache_root=options.cache_dir,
                     src_dir=options.src_dir,
                     download_dir=None,
                     ignore_dependencies=options.ignore_dependencies,

pip/operations/freeze.py

@@ -21,7 +21,8 @@ def freeze(
         find_links=None, local_only=None, user_only=None, skip_regex=None,
         find_tags=False,
         default_vcs=None,
-        isolated=False):
+        isolated=False,
+        cache_root=None):
     find_links = find_links or []
     skip_match = None
 
@@ -75,11 +76,13 @@ def freeze(
                         line,
                         default_vcs=default_vcs,
                         isolated=isolated,
+                        cache_root=cache_root,
                     )
                 else:
                     line_req = InstallRequirement.from_line(
                         line,
                         isolated=isolated,
+                        cache_root=cache_root,
                     )
 
                 if not line_req.name:

pip/req/req_file.py

@@ -77,7 +77,7 @@ IGNORE = 5
 
 
 def parse_requirements(filename, finder=None, comes_from=None, options=None,
-                       session=None):
+                       session=None, cache_root=None):
     """
     Parse a requirements file and yield InstallRequirement instances.
 
@@ -87,7 +87,6 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
     :param options:    Global options.
     :param session:    Instance of pip.download.PipSession.
     """
-
     if session is None:
         raise TypeError(
             "parse_requirements() missing 1 required keyword argument: "
@@ -99,7 +98,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
     )
 
     parser = parse_content(
-        filename, content, finder, comes_from, options, session
+        filename, content, finder, comes_from, options, session, cache_root
     )
 
     for item in parser:
@@ -107,7 +106,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
 
 
 def parse_content(filename, content, finder=None, comes_from=None,
-                  options=None, session=None):
+                  options=None, session=None, cache_root=None):
 
     # Split, sanitize and join lines with continuations.
     content = content.splitlines()
@@ -129,8 +128,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
             comes_from = '-r %s (line %s)' % (filename, line_number)
             isolated = options.isolated_mode if options else False
             yield InstallRequirement.from_line(
-                req, comes_from, isolated=isolated, options=opts
+                req, comes_from, isolated=isolated, options=opts,
-            )
+                cache_root=cache_root)
 
         # ---------------------------------------------------------------------
         elif linetype == REQUIREMENT_EDITABLE:
@@ -139,8 +138,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
             default_vcs = options.default_vcs if options else None
             yield InstallRequirement.from_editable(
                 value, comes_from=comes_from,
-                default_vcs=default_vcs, isolated=isolated
+                default_vcs=default_vcs, isolated=isolated,
-            )
+                cache_root=cache_root)
 
         # ---------------------------------------------------------------------
         elif linetype == REQUIREMENT_FILE:
@@ -152,8 +151,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
                 req_url = os.path.join(os.path.dirname(filename), value)
             # TODO: Why not use `comes_from='-r {} (line {})'` here as well?
             parser = parse_requirements(
-                req_url, finder, comes_from, options, session
+                req_url, finder, comes_from, options, session,
-            )
+                cache_root=cache_root)
             for req in parser:
                 yield req
 

pip/req/req_install.py

@@ -73,7 +73,8 @@ class InstallRequirement(object):
 
     def __init__(self, req, comes_from, source_dir=None, editable=False,
                  link=None, as_egg=False, update=True, editable_options=None,
-                 pycompile=True, markers=None, isolated=False, options=None):
+                 pycompile=True, markers=None, isolated=False, options=None,
+                 cache_root=None):
         self.extras = ()
         if isinstance(req, six.string_types):
             req = pkg_resources.Requirement.parse(req)
@@ -88,6 +89,7 @@ class InstallRequirement(object):
             editable_options = {}
 
         self.editable_options = editable_options
+        self._cache_root = cache_root
         self.link = link
         self.as_egg = as_egg
         self.markers = markers
@@ -118,7 +120,7 @@ class InstallRequirement(object):
 
     @classmethod
     def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
-                      isolated=False, options=None):
+                      isolated=False, options=None, cache_root=None):
         from pip.index import Link
 
         name, url, extras_override, editable_options = parse_editable(
@@ -133,7 +135,8 @@ class InstallRequirement(object):
                   link=Link(url),
                   editable_options=editable_options,
                   isolated=isolated,
-                  options=options if options else {})
+                  options=options if options else {},
+                  cache_root=cache_root)
 
         if extras_override is not None:
             res.extras = extras_override
@@ -141,7 +144,9 @@ class InstallRequirement(object):
         return res
 
     @classmethod
-    def from_line(cls, name, comes_from=None, isolated=False, options=None):
+    def from_line(
+            cls, name, comes_from=None, isolated=False, options=None,
+            cache_root=None):
         """Creates an InstallRequirement from a name, which might be a
         requirement, directory containing 'setup.py', filename, or URL.
         """
@@ -208,7 +213,7 @@ class InstallRequirement(object):
 
         options = options if options else {}
         return cls(req, comes_from, link=link, markers=markers,
-                   isolated=isolated, options=options)
+                   isolated=isolated, options=options, cache_root=cache_root)
 
     def __str__(self):
         if self.req:
@@ -241,6 +246,16 @@ class InstallRequirement(object):
         if self.link is None:
             self.link = finder.find_requirement(self, upgrade)
 
+    @property
+    def link(self):
+        return self._link
+
+    @link.setter
+    def link(self, link):
+        # Lookup a cached wheel, if possible.
+        link = pip.wheel.cached_wheel(self._cache_root, link)
+        self._link = link
+
     @property
     def specifier(self):
         return self.req.specifier

pip/req/req_set.py

@@ -139,7 +139,8 @@ class RequirementSet(object):
                  ignore_installed=False, as_egg=False, target_dir=None,
                  ignore_dependencies=False, force_reinstall=False,
                  use_user_site=False, session=None, pycompile=True,
-                 isolated=False, wheel_download_dir=None):
+                 isolated=False, wheel_download_dir=None,
+                 cache_root=None):
         """Create a RequirementSet.
 
         :param wheel_download_dir: Where still-packed .whl files should be
@@ -149,6 +150,8 @@ class RequirementSet(object):
         :param download_dir: Where still packed archives should be written to.
             If None they are not saved, and are deleted immediately after
             unpacking.
+        :param cache_root: The root of the pip cache, for passing to
+            InstallRequirement.
         """
         if session is None:
             raise TypeError(
@@ -181,6 +184,7 @@ class RequirementSet(object):
         if wheel_download_dir:
             wheel_download_dir = normalize_path(wheel_download_dir)
         self.wheel_download_dir = wheel_download_dir
+        self._cache_root = cache_root
         # Maps from install_req -> dependencies_of_install_req
         self._dependencies = defaultdict(list)
 
@@ -512,6 +516,7 @@ class RequirementSet(object):
                     str(subreq),
                     req_to_install,
                     isolated=self.isolated,
+                    cache_root=self._cache_root,
                 )
                 more_reqs.extend(self.add_requirement(
                     sub_install_req, req_to_install.name))

pip/wheel.py

@@ -5,6 +5,7 @@ from __future__ import absolute_import
 
 import compileall
 import csv
+import errno
 import functools
 import hashlib
 import logging
@@ -20,6 +21,8 @@ from email.parser import Parser
 
 from pip._vendor.six import StringIO
 
+import pip
+from pip.download import path_to_url
 from pip.exceptions import InvalidWheelFilename, UnsupportedWheel
 from pip.locations import distutils_scheme
 from pip import pep425tags
@@ -39,6 +42,51 @@ VERSION_COMPATIBLE = (1, 0)
 logger = logging.getLogger(__name__)
 
 
+def _cache_for_filename(cache_dir, sdistfilename):
+    """Return a directory to store cached wheels in for sdistfilename.
+
+    Because there are M wheels for any one sdist, we provide a directory
+    to cache them in, and then consult that directory when looking up
+    cache hits.
+
+    :param cache_dir: The cache_dir being used by pip.
+    :param sdistfilename: The filename of the sdist for which this will cache
+        wheels.
+    """
+    return os.path.join(cache_dir, 'wheels', sdistfilename)
+
+
+def cached_wheel(cache_dir, link):
+    if not cache_dir:
+        return link
+    if not link:
+        return link
+    if link.is_wheel:
+        return link
+    root = _cache_for_filename(cache_dir, link.filename)
+    try:
+        wheel_names = os.listdir(root)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return link
+        raise
+    candidates = []
+    for wheel_name in wheel_names:
+        try:
+            wheel = Wheel(wheel_name)
+        except InvalidWheelFilename:
+            continue
+        if not wheel.supported():
+            # Built for a different python/arch/etc
+            continue
+        candidates.append((wheel.support_index_min(), wheel_name))
+    if not candidates:
+        return link
+    candidates.sort()
+    path = os.path.join(root, candidates[0][1])
+    return pip.index.Link(path_to_url(path), trusted=True)
+
+
 def rehash(path, algo='sha256', blocksize=1 << 20):
     """Return (hash, length) for path using hashlib.new(algo)"""
     h = hashlib.new(algo)

tests/unit/test_req_file.py

@@ -182,7 +182,7 @@ class TestParseContent(object):
         import pip.req.req_file
 
         def stub_parse_requirements(req_url, finder, comes_from, options,
-                                    session):
+                                    session, cache_root):
             return [req]
         parse_requirements_stub = stub(call=stub_parse_requirements)
         monkeypatch.setattr(pip.req.req_file, 'parse_requirements',