mirror of https://github.com/pypa/pip
Issue #2563: Read cached wheels from ~/.cache/pip
This won't put wheels into that directory, but will read them if they are there. --no-cache-dir will disable reading such wheels.
This commit is contained in:
parent
786ff8c607
commit
4926409340
|
@ -388,6 +388,23 @@ Windows
|
|||
:file:`<CSIDL_LOCAL_APPDATA>\\pip\\Cache`
|
||||
|
||||
|
||||
Wheel cache
|
||||
***********
|
||||
|
||||
Pip will read from the subdirectory ``wheels`` within the pip cache dir and use
|
||||
any packages found there. This is disabled via the same ``no-cache-dir`` option
|
||||
that disables the HTTP cache. The internal structure of that cache is not part
|
||||
of the Pip API. As of 7.0 pip uses a subdirectory per sdist that wheels were
|
||||
built from, and wheels within that subdirectory.
|
||||
|
||||
Pip attempts to choose the best wheels from those built in preference to
|
||||
building a new wheel. Note that this means when a package has both optional
|
||||
C extensions and builds `py` tagged wheels when the C extension can't be built
|
||||
that pip will not attempt to build a better wheel for Python's that would have
|
||||
supported it, once any generic wheel is built. To correct this, make sure that
|
||||
the wheel's are built with Python specific tags - e.g. pp on Pypy.
|
||||
|
||||
|
||||
Hash Verification
|
||||
+++++++++++++++++
|
||||
|
||||
|
|
|
@ -289,6 +289,7 @@ class RequirementCommand(Command):
|
|||
requirement_set.add_requirement(
|
||||
InstallRequirement.from_line(
|
||||
name, None, isolated=options.isolated_mode,
|
||||
cache_root=options.cache_dir
|
||||
)
|
||||
)
|
||||
|
||||
|
@ -298,6 +299,7 @@ class RequirementCommand(Command):
|
|||
name,
|
||||
default_vcs=options.default_vcs,
|
||||
isolated=options.isolated_mode,
|
||||
cache_root=options.cache_dir
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
@ -60,7 +60,8 @@ class FreezeCommand(Command):
|
|||
local_only=options.local,
|
||||
user_only=options.user,
|
||||
skip_regex=options.skip_requirements_regex,
|
||||
isolated=options.isolated_mode)
|
||||
isolated=options.isolated_mode,
|
||||
cache_root=options.cache_dir)
|
||||
|
||||
for line in freeze(**freeze_kwargs):
|
||||
sys.stdout.write(line + '\n')
|
||||
|
|
|
@ -239,6 +239,7 @@ class InstallCommand(RequirementCommand):
|
|||
delete=build_delete) as build_dir:
|
||||
requirement_set = RequirementSet(
|
||||
build_dir=build_dir,
|
||||
cache_root=options.cache_dir,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=options.download_dir,
|
||||
upgrade=options.upgrade,
|
||||
|
|
|
@ -131,6 +131,7 @@ class ListCommand(Command):
|
|||
for dist in installed_packages:
|
||||
req = InstallRequirement.from_line(
|
||||
dist.key, None, isolated=options.isolated_mode,
|
||||
cache_root=options.cache_dir,
|
||||
)
|
||||
typ = 'unknown'
|
||||
try:
|
||||
|
|
|
@ -45,6 +45,7 @@ class UninstallCommand(Command):
|
|||
|
||||
requirement_set = RequirementSet(
|
||||
build_dir=None,
|
||||
cache_root=options.cache_dir,
|
||||
src_dir=None,
|
||||
download_dir=None,
|
||||
isolated=options.isolated_mode,
|
||||
|
@ -54,13 +55,15 @@ class UninstallCommand(Command):
|
|||
requirement_set.add_requirement(
|
||||
InstallRequirement.from_line(
|
||||
name, isolated=options.isolated_mode,
|
||||
cache_root=options.cache_dir,
|
||||
)
|
||||
)
|
||||
for filename in options.requirements:
|
||||
for req in parse_requirements(
|
||||
filename,
|
||||
options=options,
|
||||
session=session):
|
||||
session=session,
|
||||
cache_root=options.cache_dir):
|
||||
requirement_set.add_requirement(req)
|
||||
if not requirement_set.has_requirements:
|
||||
raise InstallationError(
|
||||
|
|
|
@ -159,6 +159,7 @@ class WheelCommand(RequirementCommand):
|
|||
delete=build_delete) as build_dir:
|
||||
requirement_set = RequirementSet(
|
||||
build_dir=build_dir,
|
||||
cache_root=options.cache_dir,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
ignore_dependencies=options.ignore_dependencies,
|
||||
|
|
|
@ -21,7 +21,8 @@ def freeze(
|
|||
find_links=None, local_only=None, user_only=None, skip_regex=None,
|
||||
find_tags=False,
|
||||
default_vcs=None,
|
||||
isolated=False):
|
||||
isolated=False,
|
||||
cache_root=None):
|
||||
find_links = find_links or []
|
||||
skip_match = None
|
||||
|
||||
|
@ -75,11 +76,13 @@ def freeze(
|
|||
line,
|
||||
default_vcs=default_vcs,
|
||||
isolated=isolated,
|
||||
cache_root=cache_root,
|
||||
)
|
||||
else:
|
||||
line_req = InstallRequirement.from_line(
|
||||
line,
|
||||
isolated=isolated,
|
||||
cache_root=cache_root,
|
||||
)
|
||||
|
||||
if not line_req.name:
|
||||
|
|
|
@ -77,7 +77,7 @@ IGNORE = 5
|
|||
|
||||
|
||||
def parse_requirements(filename, finder=None, comes_from=None, options=None,
|
||||
session=None):
|
||||
session=None, cache_root=None):
|
||||
"""
|
||||
Parse a requirements file and yield InstallRequirement instances.
|
||||
|
||||
|
@ -87,7 +87,6 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
|
|||
:param options: Global options.
|
||||
:param session: Instance of pip.download.PipSession.
|
||||
"""
|
||||
|
||||
if session is None:
|
||||
raise TypeError(
|
||||
"parse_requirements() missing 1 required keyword argument: "
|
||||
|
@ -99,7 +98,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
|
|||
)
|
||||
|
||||
parser = parse_content(
|
||||
filename, content, finder, comes_from, options, session
|
||||
filename, content, finder, comes_from, options, session, cache_root
|
||||
)
|
||||
|
||||
for item in parser:
|
||||
|
@ -107,7 +106,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
|
|||
|
||||
|
||||
def parse_content(filename, content, finder=None, comes_from=None,
|
||||
options=None, session=None):
|
||||
options=None, session=None, cache_root=None):
|
||||
|
||||
# Split, sanitize and join lines with continuations.
|
||||
content = content.splitlines()
|
||||
|
@ -129,8 +128,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
|
|||
comes_from = '-r %s (line %s)' % (filename, line_number)
|
||||
isolated = options.isolated_mode if options else False
|
||||
yield InstallRequirement.from_line(
|
||||
req, comes_from, isolated=isolated, options=opts
|
||||
)
|
||||
req, comes_from, isolated=isolated, options=opts,
|
||||
cache_root=cache_root)
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
elif linetype == REQUIREMENT_EDITABLE:
|
||||
|
@ -139,8 +138,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
|
|||
default_vcs = options.default_vcs if options else None
|
||||
yield InstallRequirement.from_editable(
|
||||
value, comes_from=comes_from,
|
||||
default_vcs=default_vcs, isolated=isolated
|
||||
)
|
||||
default_vcs=default_vcs, isolated=isolated,
|
||||
cache_root=cache_root)
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
elif linetype == REQUIREMENT_FILE:
|
||||
|
@ -152,8 +151,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
|
|||
req_url = os.path.join(os.path.dirname(filename), value)
|
||||
# TODO: Why not use `comes_from='-r {} (line {})'` here as well?
|
||||
parser = parse_requirements(
|
||||
req_url, finder, comes_from, options, session
|
||||
)
|
||||
req_url, finder, comes_from, options, session,
|
||||
cache_root=cache_root)
|
||||
for req in parser:
|
||||
yield req
|
||||
|
||||
|
|
|
@ -73,7 +73,8 @@ class InstallRequirement(object):
|
|||
|
||||
def __init__(self, req, comes_from, source_dir=None, editable=False,
|
||||
link=None, as_egg=False, update=True, editable_options=None,
|
||||
pycompile=True, markers=None, isolated=False, options=None):
|
||||
pycompile=True, markers=None, isolated=False, options=None,
|
||||
cache_root=None):
|
||||
self.extras = ()
|
||||
if isinstance(req, six.string_types):
|
||||
req = pkg_resources.Requirement.parse(req)
|
||||
|
@ -88,6 +89,7 @@ class InstallRequirement(object):
|
|||
editable_options = {}
|
||||
|
||||
self.editable_options = editable_options
|
||||
self._cache_root = cache_root
|
||||
self.link = link
|
||||
self.as_egg = as_egg
|
||||
self.markers = markers
|
||||
|
@ -118,7 +120,7 @@ class InstallRequirement(object):
|
|||
|
||||
@classmethod
|
||||
def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
|
||||
isolated=False, options=None):
|
||||
isolated=False, options=None, cache_root=None):
|
||||
from pip.index import Link
|
||||
|
||||
name, url, extras_override, editable_options = parse_editable(
|
||||
|
@ -133,7 +135,8 @@ class InstallRequirement(object):
|
|||
link=Link(url),
|
||||
editable_options=editable_options,
|
||||
isolated=isolated,
|
||||
options=options if options else {})
|
||||
options=options if options else {},
|
||||
cache_root=cache_root)
|
||||
|
||||
if extras_override is not None:
|
||||
res.extras = extras_override
|
||||
|
@ -141,7 +144,9 @@ class InstallRequirement(object):
|
|||
return res
|
||||
|
||||
@classmethod
|
||||
def from_line(cls, name, comes_from=None, isolated=False, options=None):
|
||||
def from_line(
|
||||
cls, name, comes_from=None, isolated=False, options=None,
|
||||
cache_root=None):
|
||||
"""Creates an InstallRequirement from a name, which might be a
|
||||
requirement, directory containing 'setup.py', filename, or URL.
|
||||
"""
|
||||
|
@ -208,7 +213,7 @@ class InstallRequirement(object):
|
|||
|
||||
options = options if options else {}
|
||||
return cls(req, comes_from, link=link, markers=markers,
|
||||
isolated=isolated, options=options)
|
||||
isolated=isolated, options=options, cache_root=cache_root)
|
||||
|
||||
def __str__(self):
|
||||
if self.req:
|
||||
|
@ -241,6 +246,16 @@ class InstallRequirement(object):
|
|||
if self.link is None:
|
||||
self.link = finder.find_requirement(self, upgrade)
|
||||
|
||||
@property
|
||||
def link(self):
|
||||
return self._link
|
||||
|
||||
@link.setter
|
||||
def link(self, link):
|
||||
# Lookup a cached wheel, if possible.
|
||||
link = pip.wheel.cached_wheel(self._cache_root, link)
|
||||
self._link = link
|
||||
|
||||
@property
|
||||
def specifier(self):
|
||||
return self.req.specifier
|
||||
|
|
|
@ -139,7 +139,8 @@ class RequirementSet(object):
|
|||
ignore_installed=False, as_egg=False, target_dir=None,
|
||||
ignore_dependencies=False, force_reinstall=False,
|
||||
use_user_site=False, session=None, pycompile=True,
|
||||
isolated=False, wheel_download_dir=None):
|
||||
isolated=False, wheel_download_dir=None,
|
||||
cache_root=None):
|
||||
"""Create a RequirementSet.
|
||||
|
||||
:param wheel_download_dir: Where still-packed .whl files should be
|
||||
|
@ -149,6 +150,8 @@ class RequirementSet(object):
|
|||
:param download_dir: Where still packed archives should be written to.
|
||||
If None they are not saved, and are deleted immediately after
|
||||
unpacking.
|
||||
:param cache_root: The root of the pip cache, for passing to
|
||||
InstallRequirement.
|
||||
"""
|
||||
if session is None:
|
||||
raise TypeError(
|
||||
|
@ -181,6 +184,7 @@ class RequirementSet(object):
|
|||
if wheel_download_dir:
|
||||
wheel_download_dir = normalize_path(wheel_download_dir)
|
||||
self.wheel_download_dir = wheel_download_dir
|
||||
self._cache_root = cache_root
|
||||
# Maps from install_req -> dependencies_of_install_req
|
||||
self._dependencies = defaultdict(list)
|
||||
|
||||
|
@ -512,6 +516,7 @@ class RequirementSet(object):
|
|||
str(subreq),
|
||||
req_to_install,
|
||||
isolated=self.isolated,
|
||||
cache_root=self._cache_root,
|
||||
)
|
||||
more_reqs.extend(self.add_requirement(
|
||||
sub_install_req, req_to_install.name))
|
||||
|
|
48
pip/wheel.py
48
pip/wheel.py
|
@ -5,6 +5,7 @@ from __future__ import absolute_import
|
|||
|
||||
import compileall
|
||||
import csv
|
||||
import errno
|
||||
import functools
|
||||
import hashlib
|
||||
import logging
|
||||
|
@ -20,6 +21,8 @@ from email.parser import Parser
|
|||
|
||||
from pip._vendor.six import StringIO
|
||||
|
||||
import pip
|
||||
from pip.download import path_to_url
|
||||
from pip.exceptions import InvalidWheelFilename, UnsupportedWheel
|
||||
from pip.locations import distutils_scheme
|
||||
from pip import pep425tags
|
||||
|
@ -39,6 +42,51 @@ VERSION_COMPATIBLE = (1, 0)
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cache_for_filename(cache_dir, sdistfilename):
|
||||
"""Return a directory to store cached wheels in for sdistfilename.
|
||||
|
||||
Because there are M wheels for any one sdist, we provide a directory
|
||||
to cache them in, and then consult that directory when looking up
|
||||
cache hits.
|
||||
|
||||
:param cache_dir: The cache_dir being used by pip.
|
||||
:param sdistfilename: The filename of the sdist for which this will cache
|
||||
wheels.
|
||||
"""
|
||||
return os.path.join(cache_dir, 'wheels', sdistfilename)
|
||||
|
||||
|
||||
def cached_wheel(cache_dir, link):
|
||||
if not cache_dir:
|
||||
return link
|
||||
if not link:
|
||||
return link
|
||||
if link.is_wheel:
|
||||
return link
|
||||
root = _cache_for_filename(cache_dir, link.filename)
|
||||
try:
|
||||
wheel_names = os.listdir(root)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
return link
|
||||
raise
|
||||
candidates = []
|
||||
for wheel_name in wheel_names:
|
||||
try:
|
||||
wheel = Wheel(wheel_name)
|
||||
except InvalidWheelFilename:
|
||||
continue
|
||||
if not wheel.supported():
|
||||
# Built for a different python/arch/etc
|
||||
continue
|
||||
candidates.append((wheel.support_index_min(), wheel_name))
|
||||
if not candidates:
|
||||
return link
|
||||
candidates.sort()
|
||||
path = os.path.join(root, candidates[0][1])
|
||||
return pip.index.Link(path_to_url(path), trusted=True)
|
||||
|
||||
|
||||
def rehash(path, algo='sha256', blocksize=1 << 20):
|
||||
"""Return (hash, length) for path using hashlib.new(algo)"""
|
||||
h = hashlib.new(algo)
|
||||
|
|
|
@ -182,7 +182,7 @@ class TestParseContent(object):
|
|||
import pip.req.req_file
|
||||
|
||||
def stub_parse_requirements(req_url, finder, comes_from, options,
|
||||
session):
|
||||
session, cache_root):
|
||||
return [req]
|
||||
parse_requirements_stub = stub(call=stub_parse_requirements)
|
||||
monkeypatch.setattr(pip.req.req_file, 'parse_requirements',
|
||||
|
|
Loading…
Reference in New Issue