From 4926409340569b7c2c10cc0373f2b75025b9279d Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 31 Mar 2015 10:44:02 +1300 Subject: [PATCH] Issue #2563: Read cached wheels from ~/.cache/pip This won't put wheels into that directory, but will read them if they are there. --no-cache-dir will disable reading such wheels. --- docs/reference/pip_install.rst | 17 ++++++++++++ pip/basecommand.py | 2 ++ pip/commands/freeze.py | 3 ++- pip/commands/install.py | 1 + pip/commands/list.py | 1 + pip/commands/uninstall.py | 5 +++- pip/commands/wheel.py | 1 + pip/operations/freeze.py | 5 +++- pip/req/req_file.py | 19 +++++++------- pip/req/req_install.py | 25 ++++++++++++++---- pip/req/req_set.py | 7 ++++- pip/wheel.py | 48 ++++++++++++++++++++++++++++++++++ tests/unit/test_req_file.py | 2 +- 13 files changed, 116 insertions(+), 20 deletions(-) diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 5f11cba73..29612593e 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -388,6 +388,23 @@ Windows :file:`\\pip\\Cache` +Wheel cache +*********** + +Pip will read from the subdirectory ``wheels`` within the pip cache dir and use +any packages found there. This is disabled via the same ``no-cache-dir`` option +that disables the HTTP cache. The internal structure of that cache is not part +of the Pip API. As of 7.0 pip uses a subdirectory per sdist that wheels were +built from, and wheels within that subdirectory. + +Pip attempts to choose the best wheels from those built in preference to +building a new wheel. Note that this means when a package has both optional +C extensions and builds `py` tagged wheels when the C extension can't be built +that pip will not attempt to build a better wheel for Python's that would have +supported it, once any generic wheel is built. To correct this, make sure that +the wheel's are built with Python specific tags - e.g. pp on Pypy. + + Hash Verification +++++++++++++++++ diff --git a/pip/basecommand.py b/pip/basecommand.py index 002c25b35..7dffe03d5 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -289,6 +289,7 @@ class RequirementCommand(Command): requirement_set.add_requirement( InstallRequirement.from_line( name, None, isolated=options.isolated_mode, + cache_root=options.cache_dir ) ) @@ -298,6 +299,7 @@ class RequirementCommand(Command): name, default_vcs=options.default_vcs, isolated=options.isolated_mode, + cache_root=options.cache_dir ) ) diff --git a/pip/commands/freeze.py b/pip/commands/freeze.py index 42261b295..7f87c3403 100644 --- a/pip/commands/freeze.py +++ b/pip/commands/freeze.py @@ -60,7 +60,8 @@ class FreezeCommand(Command): local_only=options.local, user_only=options.user, skip_regex=options.skip_requirements_regex, - isolated=options.isolated_mode) + isolated=options.isolated_mode, + cache_root=options.cache_dir) for line in freeze(**freeze_kwargs): sys.stdout.write(line + '\n') diff --git a/pip/commands/install.py b/pip/commands/install.py index 5b4489019..44b1e5351 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -239,6 +239,7 @@ class InstallCommand(RequirementCommand): delete=build_delete) as build_dir: requirement_set = RequirementSet( build_dir=build_dir, + cache_root=options.cache_dir, src_dir=options.src_dir, download_dir=options.download_dir, upgrade=options.upgrade, diff --git a/pip/commands/list.py b/pip/commands/list.py index dbc17b2f6..cd46abd78 100644 --- a/pip/commands/list.py +++ b/pip/commands/list.py @@ -131,6 +131,7 @@ class ListCommand(Command): for dist in installed_packages: req = InstallRequirement.from_line( dist.key, None, isolated=options.isolated_mode, + cache_root=options.cache_dir, ) typ = 'unknown' try: diff --git a/pip/commands/uninstall.py b/pip/commands/uninstall.py index c2af5f730..2194f3221 100644 --- a/pip/commands/uninstall.py +++ b/pip/commands/uninstall.py @@ -45,6 +45,7 @@ class UninstallCommand(Command): requirement_set = RequirementSet( build_dir=None, + cache_root=options.cache_dir, src_dir=None, download_dir=None, isolated=options.isolated_mode, @@ -54,13 +55,15 @@ class UninstallCommand(Command): requirement_set.add_requirement( InstallRequirement.from_line( name, isolated=options.isolated_mode, + cache_root=options.cache_dir, ) ) for filename in options.requirements: for req in parse_requirements( filename, options=options, - session=session): + session=session, + cache_root=options.cache_dir): requirement_set.add_requirement(req) if not requirement_set.has_requirements: raise InstallationError( diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index 98834b645..ccfa1c19c 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -159,6 +159,7 @@ class WheelCommand(RequirementCommand): delete=build_delete) as build_dir: requirement_set = RequirementSet( build_dir=build_dir, + cache_root=options.cache_dir, src_dir=options.src_dir, download_dir=None, ignore_dependencies=options.ignore_dependencies, diff --git a/pip/operations/freeze.py b/pip/operations/freeze.py index d61a84c72..fb18c151b 100644 --- a/pip/operations/freeze.py +++ b/pip/operations/freeze.py @@ -21,7 +21,8 @@ def freeze( find_links=None, local_only=None, user_only=None, skip_regex=None, find_tags=False, default_vcs=None, - isolated=False): + isolated=False, + cache_root=None): find_links = find_links or [] skip_match = None @@ -75,11 +76,13 @@ def freeze( line, default_vcs=default_vcs, isolated=isolated, + cache_root=cache_root, ) else: line_req = InstallRequirement.from_line( line, isolated=isolated, + cache_root=cache_root, ) if not line_req.name: diff --git a/pip/req/req_file.py b/pip/req/req_file.py index f5c6c3309..3d1df1dbb 100644 --- a/pip/req/req_file.py +++ b/pip/req/req_file.py @@ -77,7 +77,7 @@ IGNORE = 5 def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None): + session=None, cache_root=None): """ Parse a requirements file and yield InstallRequirement instances. @@ -87,7 +87,6 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, :param options: Global options. :param session: Instance of pip.download.PipSession. """ - if session is None: raise TypeError( "parse_requirements() missing 1 required keyword argument: " @@ -99,7 +98,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, ) parser = parse_content( - filename, content, finder, comes_from, options, session + filename, content, finder, comes_from, options, session, cache_root ) for item in parser: @@ -107,7 +106,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, def parse_content(filename, content, finder=None, comes_from=None, - options=None, session=None): + options=None, session=None, cache_root=None): # Split, sanitize and join lines with continuations. content = content.splitlines() @@ -129,8 +128,8 @@ def parse_content(filename, content, finder=None, comes_from=None, comes_from = '-r %s (line %s)' % (filename, line_number) isolated = options.isolated_mode if options else False yield InstallRequirement.from_line( - req, comes_from, isolated=isolated, options=opts - ) + req, comes_from, isolated=isolated, options=opts, + cache_root=cache_root) # --------------------------------------------------------------------- elif linetype == REQUIREMENT_EDITABLE: @@ -139,8 +138,8 @@ def parse_content(filename, content, finder=None, comes_from=None, default_vcs = options.default_vcs if options else None yield InstallRequirement.from_editable( value, comes_from=comes_from, - default_vcs=default_vcs, isolated=isolated - ) + default_vcs=default_vcs, isolated=isolated, + cache_root=cache_root) # --------------------------------------------------------------------- elif linetype == REQUIREMENT_FILE: @@ -152,8 +151,8 @@ def parse_content(filename, content, finder=None, comes_from=None, req_url = os.path.join(os.path.dirname(filename), value) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? parser = parse_requirements( - req_url, finder, comes_from, options, session - ) + req_url, finder, comes_from, options, session, + cache_root=cache_root) for req in parser: yield req diff --git a/pip/req/req_install.py b/pip/req/req_install.py index e103c579f..6d910e2bb 100644 --- a/pip/req/req_install.py +++ b/pip/req/req_install.py @@ -73,7 +73,8 @@ class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, link=None, as_egg=False, update=True, editable_options=None, - pycompile=True, markers=None, isolated=False, options=None): + pycompile=True, markers=None, isolated=False, options=None, + cache_root=None): self.extras = () if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse(req) @@ -88,6 +89,7 @@ class InstallRequirement(object): editable_options = {} self.editable_options = editable_options + self._cache_root = cache_root self.link = link self.as_egg = as_egg self.markers = markers @@ -118,7 +120,7 @@ class InstallRequirement(object): @classmethod def from_editable(cls, editable_req, comes_from=None, default_vcs=None, - isolated=False, options=None): + isolated=False, options=None, cache_root=None): from pip.index import Link name, url, extras_override, editable_options = parse_editable( @@ -133,7 +135,8 @@ class InstallRequirement(object): link=Link(url), editable_options=editable_options, isolated=isolated, - options=options if options else {}) + options=options if options else {}, + cache_root=cache_root) if extras_override is not None: res.extras = extras_override @@ -141,7 +144,9 @@ class InstallRequirement(object): return res @classmethod - def from_line(cls, name, comes_from=None, isolated=False, options=None): + def from_line( + cls, name, comes_from=None, isolated=False, options=None, + cache_root=None): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -208,7 +213,7 @@ class InstallRequirement(object): options = options if options else {} return cls(req, comes_from, link=link, markers=markers, - isolated=isolated, options=options) + isolated=isolated, options=options, cache_root=cache_root) def __str__(self): if self.req: @@ -241,6 +246,16 @@ class InstallRequirement(object): if self.link is None: self.link = finder.find_requirement(self, upgrade) + @property + def link(self): + return self._link + + @link.setter + def link(self, link): + # Lookup a cached wheel, if possible. + link = pip.wheel.cached_wheel(self._cache_root, link) + self._link = link + @property def specifier(self): return self.req.specifier diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 660c9eb9f..751c85bd1 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -139,7 +139,8 @@ class RequirementSet(object): ignore_installed=False, as_egg=False, target_dir=None, ignore_dependencies=False, force_reinstall=False, use_user_site=False, session=None, pycompile=True, - isolated=False, wheel_download_dir=None): + isolated=False, wheel_download_dir=None, + cache_root=None): """Create a RequirementSet. :param wheel_download_dir: Where still-packed .whl files should be @@ -149,6 +150,8 @@ class RequirementSet(object): :param download_dir: Where still packed archives should be written to. If None they are not saved, and are deleted immediately after unpacking. + :param cache_root: The root of the pip cache, for passing to + InstallRequirement. """ if session is None: raise TypeError( @@ -181,6 +184,7 @@ class RequirementSet(object): if wheel_download_dir: wheel_download_dir = normalize_path(wheel_download_dir) self.wheel_download_dir = wheel_download_dir + self._cache_root = cache_root # Maps from install_req -> dependencies_of_install_req self._dependencies = defaultdict(list) @@ -512,6 +516,7 @@ class RequirementSet(object): str(subreq), req_to_install, isolated=self.isolated, + cache_root=self._cache_root, ) more_reqs.extend(self.add_requirement( sub_install_req, req_to_install.name)) diff --git a/pip/wheel.py b/pip/wheel.py index d689903fc..6b9129c29 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import compileall import csv +import errno import functools import hashlib import logging @@ -20,6 +21,8 @@ from email.parser import Parser from pip._vendor.six import StringIO +import pip +from pip.download import path_to_url from pip.exceptions import InvalidWheelFilename, UnsupportedWheel from pip.locations import distutils_scheme from pip import pep425tags @@ -39,6 +42,51 @@ VERSION_COMPATIBLE = (1, 0) logger = logging.getLogger(__name__) +def _cache_for_filename(cache_dir, sdistfilename): + """Return a directory to store cached wheels in for sdistfilename. + + Because there are M wheels for any one sdist, we provide a directory + to cache them in, and then consult that directory when looking up + cache hits. + + :param cache_dir: The cache_dir being used by pip. + :param sdistfilename: The filename of the sdist for which this will cache + wheels. + """ + return os.path.join(cache_dir, 'wheels', sdistfilename) + + +def cached_wheel(cache_dir, link): + if not cache_dir: + return link + if not link: + return link + if link.is_wheel: + return link + root = _cache_for_filename(cache_dir, link.filename) + try: + wheel_names = os.listdir(root) + except OSError as e: + if e.errno == errno.ENOENT: + return link + raise + candidates = [] + for wheel_name in wheel_names: + try: + wheel = Wheel(wheel_name) + except InvalidWheelFilename: + continue + if not wheel.supported(): + # Built for a different python/arch/etc + continue + candidates.append((wheel.support_index_min(), wheel_name)) + if not candidates: + return link + candidates.sort() + path = os.path.join(root, candidates[0][1]) + return pip.index.Link(path_to_url(path), trusted=True) + + def rehash(path, algo='sha256', blocksize=1 << 20): """Return (hash, length) for path using hashlib.new(algo)""" h = hashlib.new(algo) diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index e5eebf277..0f9fcd0a5 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -182,7 +182,7 @@ class TestParseContent(object): import pip.req.req_file def stub_parse_requirements(req_url, finder, comes_from, options, - session): + session, cache_root): return [req] parse_requirements_stub = stub(call=stub_parse_requirements) monkeypatch.setattr(pip.req.req_file, 'parse_requirements',