mirror of
https://github.com/pypa/pip
synced 2023-12-13 21:30:23 +01:00
Rework Wheel Caching
This commit is contained in:
parent
b96618bedc
commit
165b10b9db
3 changed files with 62 additions and 7 deletions
|
@ -11,6 +11,7 @@ from pip._vendor.packaging.utils import canonicalize_name
|
||||||
from pip._internal import index
|
from pip._internal import index
|
||||||
from pip._internal.compat import expanduser
|
from pip._internal.compat import expanduser
|
||||||
from pip._internal.download import path_to_url
|
from pip._internal.download import path_to_url
|
||||||
|
from pip._internal.utils.temp_dir import TempDirectory
|
||||||
from pip._internal.wheel import InvalidWheelFilename, Wheel
|
from pip._internal.wheel import InvalidWheelFilename, Wheel
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -102,13 +103,18 @@ class Cache(object):
|
||||||
|
|
||||||
return index.Link(path_to_url(path))
|
return index.Link(path_to_url(path))
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
pass
|
||||||
|
|
||||||
class WheelCache(Cache):
|
|
||||||
|
class SimpleWheelCache(Cache):
|
||||||
"""A cache of wheels for future installs.
|
"""A cache of wheels for future installs.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, cache_dir, format_control):
|
def __init__(self, cache_dir, format_control):
|
||||||
super(WheelCache, self).__init__(cache_dir, format_control, {"binary"})
|
super(SimpleWheelCache, self).__init__(
|
||||||
|
cache_dir, format_control, {"binary"}
|
||||||
|
)
|
||||||
|
|
||||||
def get_path_for_link(self, link):
|
def get_path_for_link(self, link):
|
||||||
"""Return a directory to store cached wheels for link
|
"""Return a directory to store cached wheels for link
|
||||||
|
@ -127,8 +133,7 @@ class WheelCache(Cache):
|
||||||
"""
|
"""
|
||||||
parts = self._get_cache_path_parts(link)
|
parts = self._get_cache_path_parts(link)
|
||||||
|
|
||||||
# Inside of the base location for cached wheels, expand our parts and
|
# Store wheels within the root cache_dir
|
||||||
# join them all together.
|
|
||||||
return os.path.join(self.cache_dir, "wheels", *parts)
|
return os.path.join(self.cache_dir, "wheels", *parts)
|
||||||
|
|
||||||
def get(self, link, package_name):
|
def get(self, link, package_name):
|
||||||
|
@ -148,3 +153,50 @@ class WheelCache(Cache):
|
||||||
return link
|
return link
|
||||||
|
|
||||||
return self._link_for_candidate(link, min(candidates)[1])
|
return self._link_for_candidate(link, min(candidates)[1])
|
||||||
|
|
||||||
|
|
||||||
|
class EphemWheelCache(SimpleWheelCache):
|
||||||
|
"""A SimpleWheelCache that creates it's own temporary cache directory
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, format_control):
|
||||||
|
self._temp_dir = TempDirectory(kind="ephem-wheel-cache")
|
||||||
|
self._temp_dir.create()
|
||||||
|
|
||||||
|
super(EphemWheelCache, self).__init__(
|
||||||
|
self._temp_dir.path, format_control
|
||||||
|
)
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
self._temp_dir.cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
class WheelCache(Cache):
|
||||||
|
"""Wraps EphemWheelCache and SimpleWheelCache into a single Cache
|
||||||
|
|
||||||
|
This Cache allows for gracefully degradation, using the ephem wheel cache
|
||||||
|
when a certain link is not found in the simple wheel cache first.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, cache_dir, format_control):
|
||||||
|
super(WheelCache, self).__init__(
|
||||||
|
cache_dir, format_control, {'binary'}
|
||||||
|
)
|
||||||
|
self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
|
||||||
|
self._ephem_cache = EphemWheelCache(format_control)
|
||||||
|
|
||||||
|
def get_path_for_link(self, link):
|
||||||
|
return self._wheel_cache.get_path_for_link(link)
|
||||||
|
|
||||||
|
def get_ephem_path_for_link(self, link):
|
||||||
|
return self._ephem_cache.get_path_for_link(link)
|
||||||
|
|
||||||
|
def get(self, link, package_name):
|
||||||
|
retval = self._wheel_cache.get(link, package_name)
|
||||||
|
if retval is link:
|
||||||
|
retval = self._ephem_cache.get(link, package_name)
|
||||||
|
return retval
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
self._wheel_cache.cleanup()
|
||||||
|
self._ephem_cache.cleanup()
|
||||||
|
|
|
@ -224,10 +224,10 @@ class InstallCommand(RequirementCommand):
|
||||||
global_options = options.global_options or []
|
global_options = options.global_options or []
|
||||||
|
|
||||||
with self._build_session(options) as session:
|
with self._build_session(options) as session:
|
||||||
|
|
||||||
finder = self._build_package_finder(options, session)
|
finder = self._build_package_finder(options, session)
|
||||||
build_delete = (not (options.no_clean or options.build_dir))
|
build_delete = (not (options.no_clean or options.build_dir))
|
||||||
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
||||||
|
|
||||||
if options.cache_dir and not check_path_owner(options.cache_dir):
|
if options.cache_dir and not check_path_owner(options.cache_dir):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"The directory '%s' or its parent directory is not owned "
|
"The directory '%s' or its parent directory is not owned "
|
||||||
|
|
|
@ -847,14 +847,17 @@ class WheelBuilder(object):
|
||||||
'Building wheels for collected packages: %s',
|
'Building wheels for collected packages: %s',
|
||||||
', '.join([req.name for (req, _) in buildset]),
|
', '.join([req.name for (req, _) in buildset]),
|
||||||
)
|
)
|
||||||
|
_cache = self.wheel_cache # shorter name
|
||||||
with indent_log():
|
with indent_log():
|
||||||
build_success, build_failure = [], []
|
build_success, build_failure = [], []
|
||||||
for req, ephem in buildset:
|
for req, ephem in buildset:
|
||||||
python_tag = None
|
python_tag = None
|
||||||
if autobuilding:
|
if autobuilding:
|
||||||
python_tag = pep425tags.implementation_tag
|
python_tag = pep425tags.implementation_tag
|
||||||
# TODO: Incorporate ephem_cache
|
if ephem:
|
||||||
output_dir = self.wheel_cache.get_path_for_link(req.link)
|
output_dir = _cache.get_ephem_path_for_link(req.link)
|
||||||
|
else:
|
||||||
|
output_dir = _cache.get_path_for_link(req.link)
|
||||||
try:
|
try:
|
||||||
ensure_dir(output_dir)
|
ensure_dir(output_dir)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
|
|
Loading…
Reference in a new issue