mirror of https://github.com/pypa/pip
Merge pull request #4764 from pradyunsg/cache/ephem-wheel-cache
Do wheel installs from VCS/directories using ephemeral caching
This commit is contained in:
commit
81fb5159d0
|
@ -0,0 +1,3 @@
|
|||
Installing from a local directory or a VCS URL now builds a wheel to install,
|
||||
rather than running ``setup.py install``. Wheels from these sources are not
|
||||
cached.
|
|
@ -11,6 +11,7 @@ from pip._vendor.packaging.utils import canonicalize_name
|
|||
from pip._internal import index
|
||||
from pip._internal.compat import expanduser
|
||||
from pip._internal.download import path_to_url
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.wheel import InvalidWheelFilename, Wheel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -102,13 +103,18 @@ class Cache(object):
|
|||
|
||||
return index.Link(path_to_url(path))
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
|
||||
class WheelCache(Cache):
|
||||
|
||||
class SimpleWheelCache(Cache):
|
||||
"""A cache of wheels for future installs.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir, format_control):
|
||||
super(WheelCache, self).__init__(cache_dir, format_control, {"binary"})
|
||||
super(SimpleWheelCache, self).__init__(
|
||||
cache_dir, format_control, {"binary"}
|
||||
)
|
||||
|
||||
def get_path_for_link(self, link):
|
||||
"""Return a directory to store cached wheels for link
|
||||
|
@ -127,8 +133,7 @@ class WheelCache(Cache):
|
|||
"""
|
||||
parts = self._get_cache_path_parts(link)
|
||||
|
||||
# Inside of the base location for cached wheels, expand our parts and
|
||||
# join them all together.
|
||||
# Store wheels within the root cache_dir
|
||||
return os.path.join(self.cache_dir, "wheels", *parts)
|
||||
|
||||
def get(self, link, package_name):
|
||||
|
@ -148,3 +153,50 @@ class WheelCache(Cache):
|
|||
return link
|
||||
|
||||
return self._link_for_candidate(link, min(candidates)[1])
|
||||
|
||||
|
||||
class EphemWheelCache(SimpleWheelCache):
|
||||
"""A SimpleWheelCache that creates it's own temporary cache directory
|
||||
"""
|
||||
|
||||
def __init__(self, format_control):
|
||||
self._temp_dir = TempDirectory(kind="ephem-wheel-cache")
|
||||
self._temp_dir.create()
|
||||
|
||||
super(EphemWheelCache, self).__init__(
|
||||
self._temp_dir.path, format_control
|
||||
)
|
||||
|
||||
def cleanup(self):
|
||||
self._temp_dir.cleanup()
|
||||
|
||||
|
||||
class WheelCache(Cache):
|
||||
"""Wraps EphemWheelCache and SimpleWheelCache into a single Cache
|
||||
|
||||
This Cache allows for gracefully degradation, using the ephem wheel cache
|
||||
when a certain link is not found in the simple wheel cache first.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir, format_control):
|
||||
super(WheelCache, self).__init__(
|
||||
cache_dir, format_control, {'binary'}
|
||||
)
|
||||
self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
|
||||
self._ephem_cache = EphemWheelCache(format_control)
|
||||
|
||||
def get_path_for_link(self, link):
|
||||
return self._wheel_cache.get_path_for_link(link)
|
||||
|
||||
def get_ephem_path_for_link(self, link):
|
||||
return self._ephem_cache.get_path_for_link(link)
|
||||
|
||||
def get(self, link, package_name):
|
||||
retval = self._wheel_cache.get(link, package_name)
|
||||
if retval is link:
|
||||
retval = self._ephem_cache.get(link, package_name)
|
||||
return retval
|
||||
|
||||
def cleanup(self):
|
||||
self._wheel_cache.cleanup()
|
||||
self._ephem_cache.cleanup()
|
||||
|
|
|
@ -89,5 +89,8 @@ class FreezeCommand(Command):
|
|||
exclude_editable=options.exclude_editable,
|
||||
)
|
||||
|
||||
for line in freeze(**freeze_kwargs):
|
||||
sys.stdout.write(line + '\n')
|
||||
try:
|
||||
for line in freeze(**freeze_kwargs):
|
||||
sys.stdout.write(line + '\n')
|
||||
finally:
|
||||
wheel_cache.cleanup()
|
||||
|
|
|
@ -224,10 +224,10 @@ class InstallCommand(RequirementCommand):
|
|||
global_options = options.global_options or []
|
||||
|
||||
with self._build_session(options) as session:
|
||||
|
||||
finder = self._build_package_finder(options, session)
|
||||
build_delete = (not (options.no_clean or options.build_dir))
|
||||
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
||||
|
||||
if options.cache_dir and not check_path_owner(options.cache_dir):
|
||||
logger.warning(
|
||||
"The directory '%s' or its parent directory is not owned "
|
||||
|
@ -249,18 +249,19 @@ class InstallCommand(RequirementCommand):
|
|||
use_user_site=options.use_user_site,
|
||||
)
|
||||
|
||||
self.populate_requirement_set(
|
||||
requirement_set, args, options, finder, session, self.name,
|
||||
wheel_cache
|
||||
)
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
wheel_download_dir=None,
|
||||
progress_bar=options.progress_bar,
|
||||
)
|
||||
try:
|
||||
self.populate_requirement_set(
|
||||
requirement_set, args, options, finder, session,
|
||||
self.name, wheel_cache
|
||||
)
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
wheel_download_dir=None,
|
||||
progress_bar=options.progress_bar,
|
||||
)
|
||||
|
||||
resolver = Resolver(
|
||||
preparer=preparer,
|
||||
finder=finder,
|
||||
|
@ -349,6 +350,7 @@ class InstallCommand(RequirementCommand):
|
|||
# Clean up
|
||||
if not options.no_clean:
|
||||
requirement_set.cleanup_files()
|
||||
wheel_cache.cleanup()
|
||||
|
||||
if options.target_dir:
|
||||
self._handle_target_dir(
|
||||
|
|
|
@ -146,35 +146,35 @@ class WheelCommand(RequirementCommand):
|
|||
require_hashes=options.require_hashes,
|
||||
)
|
||||
|
||||
self.populate_requirement_set(
|
||||
requirement_set, args, options, finder, session, self.name,
|
||||
wheel_cache
|
||||
)
|
||||
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
wheel_download_dir=options.wheel_dir,
|
||||
progress_bar=options.progress_bar,
|
||||
)
|
||||
|
||||
resolver = Resolver(
|
||||
preparer=preparer,
|
||||
finder=finder,
|
||||
session=session,
|
||||
wheel_cache=wheel_cache,
|
||||
use_user_site=False,
|
||||
upgrade_strategy="to-satisfy-only",
|
||||
force_reinstall=False,
|
||||
ignore_dependencies=options.ignore_dependencies,
|
||||
ignore_requires_python=options.ignore_requires_python,
|
||||
ignore_installed=True,
|
||||
isolated=options.isolated_mode,
|
||||
)
|
||||
resolver.resolve(requirement_set)
|
||||
|
||||
try:
|
||||
self.populate_requirement_set(
|
||||
requirement_set, args, options, finder, session,
|
||||
self.name, wheel_cache
|
||||
)
|
||||
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
wheel_download_dir=options.wheel_dir,
|
||||
progress_bar=options.progress_bar,
|
||||
)
|
||||
|
||||
resolver = Resolver(
|
||||
preparer=preparer,
|
||||
finder=finder,
|
||||
session=session,
|
||||
wheel_cache=wheel_cache,
|
||||
use_user_site=False,
|
||||
upgrade_strategy="to-satisfy-only",
|
||||
force_reinstall=False,
|
||||
ignore_dependencies=options.ignore_dependencies,
|
||||
ignore_requires_python=options.ignore_requires_python,
|
||||
ignore_installed=True,
|
||||
isolated=options.isolated_mode,
|
||||
)
|
||||
resolver.resolve(requirement_set)
|
||||
|
||||
# build wheels
|
||||
wb = WheelBuilder(
|
||||
finder, preparer, wheel_cache,
|
||||
|
@ -195,3 +195,4 @@ class WheelCommand(RequirementCommand):
|
|||
finally:
|
||||
if not options.no_clean:
|
||||
requirement_set.cleanup_files()
|
||||
wheel_cache.cleanup()
|
||||
|
|
|
@ -220,7 +220,9 @@ class Resolver(object):
|
|||
if req.satisfied_by:
|
||||
should_modify = (
|
||||
self.upgrade_strategy != "to-satisfy-only" or
|
||||
self.force_reinstall or self.ignore_installed
|
||||
self.force_reinstall or
|
||||
self.ignore_installed or
|
||||
req.link.scheme == 'file'
|
||||
)
|
||||
if should_modify:
|
||||
self._set_req_to_reinstall(req)
|
||||
|
|
|
@ -806,6 +806,7 @@ class WheelBuilder(object):
|
|||
|
||||
buildset = []
|
||||
for req in requirements:
|
||||
ephem_cache = False
|
||||
if req.constraint:
|
||||
continue
|
||||
if req.is_wheel:
|
||||
|
@ -816,7 +817,8 @@ class WheelBuilder(object):
|
|||
elif autobuilding and req.editable:
|
||||
pass
|
||||
elif autobuilding and req.link and not req.link.is_artifact:
|
||||
pass
|
||||
# VCS checkout. Build wheel just for this run.
|
||||
ephem_cache = True
|
||||
elif autobuilding and not req.source_dir:
|
||||
pass
|
||||
else:
|
||||
|
@ -824,9 +826,8 @@ class WheelBuilder(object):
|
|||
link = req.link
|
||||
base, ext = link.splitext()
|
||||
if index.egg_info_matches(base, None, link) is None:
|
||||
# Doesn't look like a package - don't autobuild a wheel
|
||||
# because we'll have no way to lookup the result sanely
|
||||
continue
|
||||
# E.g. local directory. Build wheel just for this run.
|
||||
ephem_cache = True
|
||||
if "binary" not in index.fmt_ctl_formats(
|
||||
self.finder.format_control,
|
||||
canonicalize_name(req.name)):
|
||||
|
@ -835,7 +836,7 @@ class WheelBuilder(object):
|
|||
"being disabled for it.", req.name,
|
||||
)
|
||||
continue
|
||||
buildset.append(req)
|
||||
buildset.append((req, ephem_cache))
|
||||
|
||||
if not buildset:
|
||||
return True
|
||||
|
@ -843,15 +844,19 @@ class WheelBuilder(object):
|
|||
# Build the wheels.
|
||||
logger.info(
|
||||
'Building wheels for collected packages: %s',
|
||||
', '.join([req.name for req in buildset]),
|
||||
', '.join([req.name for (req, _) in buildset]),
|
||||
)
|
||||
_cache = self.wheel_cache # shorter name
|
||||
with indent_log():
|
||||
build_success, build_failure = [], []
|
||||
for req in buildset:
|
||||
for req, ephem in buildset:
|
||||
python_tag = None
|
||||
if autobuilding:
|
||||
python_tag = pep425tags.implementation_tag
|
||||
output_dir = self.wheel_cache.get_path_for_link(req.link)
|
||||
if ephem:
|
||||
output_dir = _cache.get_ephem_path_for_link(req.link)
|
||||
else:
|
||||
output_dir = _cache.get_path_for_link(req.link)
|
||||
try:
|
||||
ensure_dir(output_dir)
|
||||
except OSError as e:
|
||||
|
|
|
@ -994,16 +994,15 @@ def test_install_builds_wheels(script, data, common_wheels):
|
|||
# and built wheels for upper and wheelbroken
|
||||
assert "Running setup.py bdist_wheel for upper" in str(res), str(res)
|
||||
assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
|
||||
# But not requires_wheel... which is a local dir and thus uncachable.
|
||||
assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
|
||||
# Wheels are built for local directories, but not cached.
|
||||
assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
|
||||
# wheelbroken has to run install
|
||||
# into the cache
|
||||
assert wheels != [], str(res)
|
||||
# and installed from the wheel
|
||||
assert "Running setup.py install for upper" not in str(res), str(res)
|
||||
# the local tree can't build a wheel (because we can't assume that every
|
||||
# build will have a suitable unique key to cache on).
|
||||
assert "Running setup.py install for requires-wheel" in str(res), str(res)
|
||||
# Wheels are built for local directories, but not cached.
|
||||
assert "Running setup.py install for requir" not in str(res), str(res)
|
||||
# wheelbroken has to run install
|
||||
assert "Running setup.py install for wheelb" in str(res), str(res)
|
||||
# We want to make sure we used the correct implementation tag
|
||||
|
@ -1027,13 +1026,12 @@ def test_install_no_binary_disables_building_wheels(
|
|||
assert expected in str(res), str(res)
|
||||
# and built wheels for wheelbroken only
|
||||
assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
|
||||
# But not requires_wheel... which is a local dir and thus uncachable.
|
||||
assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
|
||||
# Nor upper, which was blacklisted
|
||||
# Wheels are built for local directories, but not cached across runs
|
||||
assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
|
||||
# Don't build wheel for upper which was blacklisted
|
||||
assert "Running setup.py bdist_wheel for upper" not in str(res), str(res)
|
||||
# the local tree can't build a wheel (because we can't assume that every
|
||||
# build will have a suitable unique key to cache on).
|
||||
assert "Running setup.py install for requires-wheel" in str(res), str(res)
|
||||
# Wheels are built for local directories, but not cached across runs
|
||||
assert "Running setup.py install for requir" not in str(res), str(res)
|
||||
# And these two fell back to sdist based installed.
|
||||
assert "Running setup.py install for wheelb" in str(res), str(res)
|
||||
assert "Running setup.py install for upper" in str(res), str(res)
|
||||
|
|
|
@ -31,7 +31,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
|
|||
build = script.base_path / 'pip-build'
|
||||
script.pip(
|
||||
'install', '--no-clean', '--no-index', '--build', build,
|
||||
'--find-links=%s' % data.find_links, 'simple',
|
||||
'--find-links=%s' % data.find_links, 'simple', expect_temp=True,
|
||||
)
|
||||
assert exists(build)
|
||||
|
||||
|
@ -134,7 +134,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data):
|
|||
result = script.pip(
|
||||
'install', '-f', data.find_links, '--no-index', 'simple',
|
||||
'--build', build,
|
||||
expect_error=True,
|
||||
expect_error=True, expect_temp=True,
|
||||
)
|
||||
|
||||
assert result.returncode == PREVIOUS_BUILD_DIR_ERROR
|
||||
|
|
|
@ -68,24 +68,29 @@ class Tests_UserSite:
|
|||
)
|
||||
result.assert_installed('INITools', use_user_site=True)
|
||||
|
||||
def test_install_curdir_usersite(self, script, virtualenv, data):
|
||||
@pytest.mark.network
|
||||
def test_install_from_current_directory_into_usersite(
|
||||
self, script, virtualenv, data, common_wheels):
|
||||
"""
|
||||
Test installing current directory ('.') into usersite
|
||||
"""
|
||||
virtualenv.system_site_packages = True
|
||||
script.pip("install", "wheel", '--no-index', '-f', common_wheels)
|
||||
|
||||
run_from = data.packages.join("FSPkg")
|
||||
result = script.pip(
|
||||
'install', '-vvv', '--user', curdir,
|
||||
cwd=run_from,
|
||||
expect_error=False,
|
||||
)
|
||||
|
||||
fspkg_folder = script.user_site / 'fspkg'
|
||||
egg_info_folder = (
|
||||
script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion
|
||||
)
|
||||
assert fspkg_folder in result.files_created, result.stdout
|
||||
|
||||
assert egg_info_folder in result.files_created
|
||||
dist_info_folder = (
|
||||
script.user_site / 'FSPkg-0.1.dev0.dist-info'
|
||||
)
|
||||
assert dist_info_folder in result.files_created
|
||||
|
||||
def test_install_user_venv_nositepkgs_fails(self, script, data):
|
||||
"""
|
||||
|
|
|
@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(
|
|||
result = script.pip(
|
||||
'wheel', '--no-index', '--find-links=%s' % data.find_links,
|
||||
'--build', script.venv_path / 'build',
|
||||
'simple==3.0', expect_error=True,
|
||||
'simple==3.0', expect_error=True, expect_temp=True,
|
||||
)
|
||||
|
||||
# Then I see that the error code is the right one
|
||||
|
|
Loading…
Reference in New Issue