mirror of
https://github.com/pypa/pip
synced 2023-12-13 21:30:23 +01:00
Introduce pip.cache.Cache base class for pip's caches (#4623)
This commit is contained in:
parent
133ec67bed
commit
3dd4a0d1ac
4 changed files with 87 additions and 50 deletions
123
pip/cache.py
123
pip/cache.py
|
@ -16,34 +16,28 @@ from pip.wheel import InvalidWheelFilename, Wheel
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class WheelCache(object):
|
class Cache(object):
|
||||||
"""A cache of wheels for future installs."""
|
"""An abstract class - provides cache directories for data from links
|
||||||
|
|
||||||
def __init__(self, cache_dir, format_control):
|
|
||||||
"""Create a wheel cache.
|
|
||||||
|
|
||||||
:param cache_dir: The root of the cache.
|
:param cache_dir: The root of the cache.
|
||||||
:param format_control: A pip.index.FormatControl object to limit
|
:param format_control: A pip.index.FormatControl object to limit
|
||||||
binaries being read from the cache.
|
binaries being read from the cache.
|
||||||
|
:param allowed_formats: which formats of files the cache should store.
|
||||||
|
('binary' and 'source' are the only allowed values)
|
||||||
"""
|
"""
|
||||||
self._cache_dir = expanduser(cache_dir) if cache_dir else None
|
|
||||||
self._format_control = format_control
|
|
||||||
|
|
||||||
def get_cache_path_for_link(self, link):
|
def __init__(self, cache_dir, format_control, allowed_formats):
|
||||||
"""
|
super(Cache, self).__init__()
|
||||||
Return a directory to store cached wheels in for link.
|
self.cache_dir = expanduser(cache_dir) if cache_dir else None
|
||||||
|
self.format_control = format_control
|
||||||
|
self.allowed_formats = allowed_formats
|
||||||
|
|
||||||
Because there are M wheels for any one sdist, we provide a directory
|
_valid_formats = {"source", "binary"}
|
||||||
to cache them in, and then consult that directory when looking up
|
assert self.allowed_formats.union(_valid_formats) == _valid_formats
|
||||||
cache hits.
|
|
||||||
|
|
||||||
We only insert things into the cache if they have plausible version
|
def _get_cache_path_parts(self, link):
|
||||||
numbers, so that we don't contaminate the cache with things that were
|
"""Get parts of part that must be os.path.joined with cache_dir
|
||||||
not unique. E.g. ./package might have dozens of installs done for it
|
|
||||||
and build a version of 0.0...and if we built and cached a wheel, we'd
|
|
||||||
end up using the same wheel even if the source has been edited.
|
|
||||||
|
|
||||||
:param link: The link of the sdist for which this will cache wheels.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# We want to generate an url to use as our cache key, we don't want to
|
# We want to generate an url to use as our cache key, we don't want to
|
||||||
|
@ -65,37 +59,82 @@ class WheelCache(object):
|
||||||
# FS.
|
# FS.
|
||||||
parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
|
parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
|
||||||
|
|
||||||
# Inside of the base location for cached wheels, expand our parts and
|
return parts
|
||||||
# join them all together.
|
|
||||||
return os.path.join(self._cache_dir, "wheels", *parts)
|
|
||||||
|
|
||||||
def cached_wheel(self, link, package_name):
|
def _get_candidates(self, link, package_name):
|
||||||
not_cached = (
|
can_not_cache = (
|
||||||
not self._cache_dir or
|
not self.cache_dir or
|
||||||
not link or
|
not package_name or
|
||||||
link.is_wheel or
|
not link
|
||||||
not link.is_artifact or
|
|
||||||
not package_name
|
|
||||||
)
|
)
|
||||||
|
if can_not_cache:
|
||||||
if not_cached:
|
return []
|
||||||
return link
|
|
||||||
|
|
||||||
canonical_name = canonicalize_name(package_name)
|
canonical_name = canonicalize_name(package_name)
|
||||||
formats = pip.index.fmt_ctl_formats(
|
formats = pip.index.fmt_ctl_formats(
|
||||||
self._format_control, canonical_name
|
self.format_control, canonical_name
|
||||||
)
|
)
|
||||||
if "binary" not in formats:
|
if not self.allowed_formats.intersection(formats):
|
||||||
return link
|
return []
|
||||||
root = self.get_cache_path_for_link(link)
|
|
||||||
|
root = self.get_path_for_link(link)
|
||||||
try:
|
try:
|
||||||
wheel_names = os.listdir(root)
|
return os.listdir(root)
|
||||||
except OSError as err:
|
except OSError as err:
|
||||||
if err.errno in {errno.ENOENT, errno.ENOTDIR}:
|
if err.errno in {errno.ENOENT, errno.ENOTDIR}:
|
||||||
return link
|
return []
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def get_path_for_link(self, link):
|
||||||
|
"""Return a directory to store cached items in for link.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def get(self, link, package_name):
|
||||||
|
"""Returns a link to a cached item if it exists, otherwise returns the
|
||||||
|
passed link.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def _link_for_candidate(self, link, candidate):
|
||||||
|
root = self.get_path_for_link(link)
|
||||||
|
path = os.path.join(root, candidate)
|
||||||
|
|
||||||
|
return pip.index.Link(path_to_url(path))
|
||||||
|
|
||||||
|
|
||||||
|
class WheelCache(Cache):
|
||||||
|
"""A cache of wheels for future installs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, cache_dir, format_control):
|
||||||
|
super(WheelCache, self).__init__(cache_dir, format_control, {"binary"})
|
||||||
|
|
||||||
|
def get_path_for_link(self, link):
|
||||||
|
"""Return a directory to store cached wheels for link
|
||||||
|
|
||||||
|
Because there are M wheels for any one sdist, we provide a directory
|
||||||
|
to cache them in, and then consult that directory when looking up
|
||||||
|
cache hits.
|
||||||
|
|
||||||
|
We only insert things into the cache if they have plausible version
|
||||||
|
numbers, so that we don't contaminate the cache with things that were
|
||||||
|
not unique. E.g. ./package might have dozens of installs done for it
|
||||||
|
and build a version of 0.0...and if we built and cached a wheel, we'd
|
||||||
|
end up using the same wheel even if the source has been edited.
|
||||||
|
|
||||||
|
:param link: The link of the sdist for which this will cache wheels.
|
||||||
|
"""
|
||||||
|
parts = self._get_cache_path_parts(link)
|
||||||
|
|
||||||
|
# Inside of the base location for cached wheels, expand our parts and
|
||||||
|
# join them all together.
|
||||||
|
return os.path.join(self.cache_dir, "wheels", *parts)
|
||||||
|
|
||||||
|
def get(self, link, package_name):
|
||||||
candidates = []
|
candidates = []
|
||||||
for wheel_name in wheel_names:
|
|
||||||
|
for wheel_name in self._get_candidates(link, package_name):
|
||||||
try:
|
try:
|
||||||
wheel = Wheel(wheel_name)
|
wheel = Wheel(wheel_name)
|
||||||
except InvalidWheelFilename:
|
except InvalidWheelFilename:
|
||||||
|
@ -104,8 +143,8 @@ class WheelCache(object):
|
||||||
# Built for a different python/arch/etc
|
# Built for a different python/arch/etc
|
||||||
continue
|
continue
|
||||||
candidates.append((wheel.support_index_min(), wheel_name))
|
candidates.append((wheel.support_index_min(), wheel_name))
|
||||||
|
|
||||||
if not candidates:
|
if not candidates:
|
||||||
return link
|
return link
|
||||||
candidates.sort()
|
|
||||||
path = os.path.join(root, candidates[0][1])
|
return self._link_for_candidate(link, min(candidates)[1])
|
||||||
return pip.index.Link(path_to_url(path))
|
|
||||||
|
|
|
@ -298,7 +298,7 @@ class InstallRequirement(object):
|
||||||
self.link = finder.find_requirement(self, upgrade)
|
self.link = finder.find_requirement(self, upgrade)
|
||||||
if self._wheel_cache is not None and not require_hashes:
|
if self._wheel_cache is not None and not require_hashes:
|
||||||
old_link = self.link
|
old_link = self.link
|
||||||
self.link = self._wheel_cache.cached_wheel(self.link, self.name)
|
self.link = self._wheel_cache.get(self.link, self.name)
|
||||||
if old_link != self.link:
|
if old_link != self.link:
|
||||||
logger.debug('Using cached wheel link: %s', self.link)
|
logger.debug('Using cached wheel link: %s', self.link)
|
||||||
|
|
||||||
|
|
|
@ -727,7 +727,7 @@ class WheelBuilder(object):
|
||||||
:return: True if all the wheels built correctly.
|
:return: True if all the wheels built correctly.
|
||||||
"""
|
"""
|
||||||
building_is_possible = self._wheel_dir or (
|
building_is_possible = self._wheel_dir or (
|
||||||
autobuilding and self.wheel_cache._cache_dir
|
autobuilding and self.wheel_cache.cache_dir
|
||||||
)
|
)
|
||||||
assert building_is_possible
|
assert building_is_possible
|
||||||
|
|
||||||
|
@ -778,9 +778,7 @@ class WheelBuilder(object):
|
||||||
python_tag = None
|
python_tag = None
|
||||||
if autobuilding:
|
if autobuilding:
|
||||||
python_tag = pep425tags.implementation_tag
|
python_tag = pep425tags.implementation_tag
|
||||||
output_dir = self.wheel_cache.get_cache_path_for_link(
|
output_dir = self.wheel_cache.get_path_for_link(req.link)
|
||||||
req.link
|
|
||||||
)
|
|
||||||
try:
|
try:
|
||||||
ensure_dir(output_dir)
|
ensure_dir(output_dir)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
|
|
|
@ -6,8 +6,8 @@ class TestWheelCache:
|
||||||
|
|
||||||
def test_expands_path(self):
|
def test_expands_path(self):
|
||||||
wc = WheelCache("~/.foo/", None)
|
wc = WheelCache("~/.foo/", None)
|
||||||
assert wc._cache_dir == expanduser("~/.foo/")
|
assert wc.cache_dir == expanduser("~/.foo/")
|
||||||
|
|
||||||
def test_falsey_path_none(self):
|
def test_falsey_path_none(self):
|
||||||
wc = WheelCache(False, None)
|
wc = WheelCache(False, None)
|
||||||
assert wc._cache_dir is None
|
assert wc.cache_dir is None
|
||||||
|
|
Loading…
Reference in a new issue