mirror of https://github.com/pypa/pip
Merge pull request #5336 from benoit-pierre/src_build_reqs_in_pep518
PEP 518: enable source installs for build dependencies
This commit is contained in:
commit
0bd01d9462
|
@ -154,8 +154,9 @@ appropriately.
|
||||||
installation of build dependencies from source has been disabled until a safe
|
installation of build dependencies from source has been disabled until a safe
|
||||||
resolution of this issue is found.
|
resolution of this issue is found.
|
||||||
|
|
||||||
* ``pip<18.0`` does not support the use of environment markers and extras, only
|
* ``pip<18.0``: only support installing build requirements from wheels, and
|
||||||
version specifiers are respected.
|
does not support the use of environment markers and extras (only version
|
||||||
|
specifiers are respected).
|
||||||
|
|
||||||
|
|
||||||
Future Developments
|
Future Developments
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
Add support for installing PEP 518 build dependencies from source.
|
|
@ -79,10 +79,13 @@ class BuildEnvironment(object):
|
||||||
args = [
|
args = [
|
||||||
sys.executable, '-m', 'pip', 'install', '--ignore-installed',
|
sys.executable, '-m', 'pip', 'install', '--ignore-installed',
|
||||||
'--no-user', '--prefix', self.path, '--no-warn-script-location',
|
'--no-user', '--prefix', self.path, '--no-warn-script-location',
|
||||||
'--only-binary', ':all:',
|
|
||||||
]
|
]
|
||||||
if logger.getEffectiveLevel() <= logging.DEBUG:
|
if logger.getEffectiveLevel() <= logging.DEBUG:
|
||||||
args.append('-v')
|
args.append('-v')
|
||||||
|
for format_control in ('no_binary', 'only_binary'):
|
||||||
|
formats = getattr(finder.format_control, format_control)
|
||||||
|
args.extend(('--' + format_control.replace('_', '-'),
|
||||||
|
','.join(sorted(formats or {':none:'}))))
|
||||||
if finder.index_urls:
|
if finder.index_urls:
|
||||||
args.extend(['-i', finder.index_urls[0]])
|
args.extend(['-i', finder.index_urls[0]])
|
||||||
for extra_index in finder.index_urls[1:]:
|
for extra_index in finder.index_urls[1:]:
|
||||||
|
|
|
@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError
|
||||||
from pip._internal.index import FormatControl
|
from pip._internal.index import FormatControl
|
||||||
from pip._internal.operations.prepare import RequirementPreparer
|
from pip._internal.operations.prepare import RequirementPreparer
|
||||||
from pip._internal.req import RequirementSet
|
from pip._internal.req import RequirementSet
|
||||||
|
from pip._internal.req.req_tracker import RequirementTracker
|
||||||
from pip._internal.resolve import Resolver
|
from pip._internal.resolve import Resolver
|
||||||
from pip._internal.utils.filesystem import check_path_owner
|
from pip._internal.utils.filesystem import check_path_owner
|
||||||
from pip._internal.utils.misc import ensure_dir, normalize_path
|
from pip._internal.utils.misc import ensure_dir, normalize_path
|
||||||
|
@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand):
|
||||||
)
|
)
|
||||||
options.cache_dir = None
|
options.cache_dir = None
|
||||||
|
|
||||||
with TempDirectory(
|
with RequirementTracker() as req_tracker, TempDirectory(
|
||||||
options.build_dir, delete=build_delete, kind="download"
|
options.build_dir, delete=build_delete, kind="download"
|
||||||
) as directory:
|
) as directory:
|
||||||
|
|
||||||
|
@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand):
|
||||||
wheel_download_dir=None,
|
wheel_download_dir=None,
|
||||||
progress_bar=options.progress_bar,
|
progress_bar=options.progress_bar,
|
||||||
build_isolation=options.build_isolation,
|
build_isolation=options.build_isolation,
|
||||||
|
req_tracker=req_tracker,
|
||||||
)
|
)
|
||||||
|
|
||||||
resolver = Resolver(
|
resolver = Resolver(
|
||||||
|
|
|
@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global
|
||||||
from pip._internal.operations.check import check_install_conflicts
|
from pip._internal.operations.check import check_install_conflicts
|
||||||
from pip._internal.operations.prepare import RequirementPreparer
|
from pip._internal.operations.prepare import RequirementPreparer
|
||||||
from pip._internal.req import RequirementSet, install_given_reqs
|
from pip._internal.req import RequirementSet, install_given_reqs
|
||||||
|
from pip._internal.req.req_tracker import RequirementTracker
|
||||||
from pip._internal.resolve import Resolver
|
from pip._internal.resolve import Resolver
|
||||||
from pip._internal.status_codes import ERROR
|
from pip._internal.status_codes import ERROR
|
||||||
from pip._internal.utils.filesystem import check_path_owner
|
from pip._internal.utils.filesystem import check_path_owner
|
||||||
|
@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand):
|
||||||
)
|
)
|
||||||
options.cache_dir = None
|
options.cache_dir = None
|
||||||
|
|
||||||
with TempDirectory(
|
with RequirementTracker() as req_tracker, TempDirectory(
|
||||||
options.build_dir, delete=build_delete, kind="install"
|
options.build_dir, delete=build_delete, kind="install"
|
||||||
) as directory:
|
) as directory:
|
||||||
requirement_set = RequirementSet(
|
requirement_set = RequirementSet(
|
||||||
|
@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand):
|
||||||
wheel_download_dir=None,
|
wheel_download_dir=None,
|
||||||
progress_bar=options.progress_bar,
|
progress_bar=options.progress_bar,
|
||||||
build_isolation=options.build_isolation,
|
build_isolation=options.build_isolation,
|
||||||
|
req_tracker=req_tracker,
|
||||||
)
|
)
|
||||||
|
|
||||||
resolver = Resolver(
|
resolver = Resolver(
|
||||||
|
|
|
@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache
|
||||||
from pip._internal.exceptions import CommandError, PreviousBuildDirError
|
from pip._internal.exceptions import CommandError, PreviousBuildDirError
|
||||||
from pip._internal.operations.prepare import RequirementPreparer
|
from pip._internal.operations.prepare import RequirementPreparer
|
||||||
from pip._internal.req import RequirementSet
|
from pip._internal.req import RequirementSet
|
||||||
|
from pip._internal.req.req_tracker import RequirementTracker
|
||||||
from pip._internal.resolve import Resolver
|
from pip._internal.resolve import Resolver
|
||||||
from pip._internal.utils.temp_dir import TempDirectory
|
from pip._internal.utils.temp_dir import TempDirectory
|
||||||
from pip._internal.wheel import WheelBuilder
|
from pip._internal.wheel import WheelBuilder
|
||||||
|
@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand):
|
||||||
build_delete = (not (options.no_clean or options.build_dir))
|
build_delete = (not (options.no_clean or options.build_dir))
|
||||||
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
||||||
|
|
||||||
with TempDirectory(
|
with RequirementTracker() as req_tracker, TempDirectory(
|
||||||
options.build_dir, delete=build_delete, kind="wheel"
|
options.build_dir, delete=build_delete, kind="wheel"
|
||||||
) as directory:
|
) as directory:
|
||||||
|
|
||||||
requirement_set = RequirementSet(
|
requirement_set = RequirementSet(
|
||||||
require_hashes=options.require_hashes,
|
require_hashes=options.require_hashes,
|
||||||
)
|
)
|
||||||
|
@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand):
|
||||||
wheel_download_dir=options.wheel_dir,
|
wheel_download_dir=options.wheel_dir,
|
||||||
progress_bar=options.progress_bar,
|
progress_bar=options.progress_bar,
|
||||||
build_isolation=options.build_isolation,
|
build_isolation=options.build_isolation,
|
||||||
|
req_tracker=req_tracker,
|
||||||
)
|
)
|
||||||
|
|
||||||
resolver = Resolver(
|
resolver = Resolver(
|
||||||
|
|
|
@ -141,11 +141,12 @@ class RequirementPreparer(object):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
|
def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
|
||||||
progress_bar, build_isolation):
|
progress_bar, build_isolation, req_tracker):
|
||||||
super(RequirementPreparer, self).__init__()
|
super(RequirementPreparer, self).__init__()
|
||||||
|
|
||||||
self.src_dir = src_dir
|
self.src_dir = src_dir
|
||||||
self.build_dir = build_dir
|
self.build_dir = build_dir
|
||||||
|
self.req_tracker = req_tracker
|
||||||
|
|
||||||
# Where still packed archives should be written to. If None, they are
|
# Where still packed archives should be written to. If None, they are
|
||||||
# not saved, and are deleted immediately after unpacking.
|
# not saved, and are deleted immediately after unpacking.
|
||||||
|
@ -293,7 +294,8 @@ class RequirementPreparer(object):
|
||||||
(req, exc, req.link)
|
(req, exc, req.link)
|
||||||
)
|
)
|
||||||
abstract_dist = make_abstract_dist(req)
|
abstract_dist = make_abstract_dist(req)
|
||||||
abstract_dist.prep_for_dist(finder, self.build_isolation)
|
with self.req_tracker.track(req):
|
||||||
|
abstract_dist.prep_for_dist(finder, self.build_isolation)
|
||||||
if self._download_should_save:
|
if self._download_should_save:
|
||||||
# Make a .zip of the source_dir we already created.
|
# Make a .zip of the source_dir we already created.
|
||||||
if req.link.scheme in vcs.all_schemes:
|
if req.link.scheme in vcs.all_schemes:
|
||||||
|
@ -319,7 +321,8 @@ class RequirementPreparer(object):
|
||||||
req.update_editable(not self._download_should_save)
|
req.update_editable(not self._download_should_save)
|
||||||
|
|
||||||
abstract_dist = make_abstract_dist(req)
|
abstract_dist = make_abstract_dist(req)
|
||||||
abstract_dist.prep_for_dist(finder, self.build_isolation)
|
with self.req_tracker.track(req):
|
||||||
|
abstract_dist.prep_for_dist(finder, self.build_isolation)
|
||||||
|
|
||||||
if self._download_should_save:
|
if self._download_should_save:
|
||||||
req.archive(self.download_dir)
|
req.archive(self.download_dir)
|
||||||
|
|
|
@ -0,0 +1,77 @@
|
||||||
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import errno
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from pip._internal.utils.temp_dir import TempDirectory
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class RequirementTracker(object):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._root = os.environ.get('PIP_REQ_TRACKER')
|
||||||
|
if self._root is None:
|
||||||
|
self._temp_dir = TempDirectory(delete=False, kind='req-tracker')
|
||||||
|
self._temp_dir.create()
|
||||||
|
self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path
|
||||||
|
logger.debug('Created requirements tracker %r', self._root)
|
||||||
|
else:
|
||||||
|
self._temp_dir = None
|
||||||
|
logger.debug('Re-using requirements tracker %r', self._root)
|
||||||
|
self._entries = set()
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
self.cleanup()
|
||||||
|
|
||||||
|
def _entry_path(self, link):
|
||||||
|
hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
|
||||||
|
return os.path.join(self._root, hashed)
|
||||||
|
|
||||||
|
def add(self, req):
|
||||||
|
link = req.link
|
||||||
|
info = str(req)
|
||||||
|
entry_path = self._entry_path(link)
|
||||||
|
try:
|
||||||
|
with open(entry_path) as fp:
|
||||||
|
# Error, these's already a build in progress.
|
||||||
|
raise LookupError('%s is already being built: %s'
|
||||||
|
% (link, fp.read()))
|
||||||
|
except IOError as e:
|
||||||
|
if e.errno != errno.ENOENT:
|
||||||
|
raise
|
||||||
|
assert req not in self._entries
|
||||||
|
with open(entry_path, 'w') as fp:
|
||||||
|
fp.write(info)
|
||||||
|
self._entries.add(req)
|
||||||
|
logger.debug('Added %s to build tracker %r', req, self._root)
|
||||||
|
|
||||||
|
def remove(self, req):
|
||||||
|
link = req.link
|
||||||
|
self._entries.remove(req)
|
||||||
|
os.unlink(self._entry_path(link))
|
||||||
|
logger.debug('Removed %s from build tracker %r', req, self._root)
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
for req in set(self._entries):
|
||||||
|
self.remove(req)
|
||||||
|
remove = self._temp_dir is not None
|
||||||
|
if remove:
|
||||||
|
self._temp_dir.cleanup()
|
||||||
|
logger.debug('%s build tracker %r',
|
||||||
|
'Removed' if remove else 'Cleaned',
|
||||||
|
self._root)
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def track(self, req):
|
||||||
|
self.add(req)
|
||||||
|
yield
|
||||||
|
self.remove(req)
|
|
@ -129,6 +129,9 @@ def isolate(tmpdir):
|
||||||
# We want to disable the version check from running in the tests
|
# We want to disable the version check from running in the tests
|
||||||
os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true"
|
os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true"
|
||||||
|
|
||||||
|
# Make sure tests don't share a requirements tracker.
|
||||||
|
os.environ.pop('PIP_REQ_TRACKER', None)
|
||||||
|
|
||||||
# FIXME: Windows...
|
# FIXME: Windows...
|
||||||
os.makedirs(os.path.join(home_dir, ".config", "git"))
|
os.makedirs(os.path.join(home_dir, ".config", "git"))
|
||||||
with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp:
|
with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp:
|
||||||
|
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1 @@
|
||||||
|
include pyproject.toml
|
|
@ -0,0 +1,2 @@
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools", "wheel", "pep518_forkbomb"]
|
|
@ -0,0 +1,5 @@
|
||||||
|
from setuptools import setup
|
||||||
|
|
||||||
|
setup(name='pep518_forkbomb',
|
||||||
|
version='235',
|
||||||
|
py_modules=['pep518_forkbomb'])
|
|
@ -0,0 +1 @@
|
||||||
|
include pyproject.toml
|
|
@ -0,0 +1,2 @@
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools", "wheel", "pep518_twin_forkbombs_second"]
|
|
@ -0,0 +1,5 @@
|
||||||
|
from setuptools import setup
|
||||||
|
|
||||||
|
setup(name='pep518_twin_forkbombs_first',
|
||||||
|
version='234',
|
||||||
|
py_modules=['pep518_twin_forkbombs_first'])
|
|
@ -0,0 +1 @@
|
||||||
|
include pyproject.toml
|
|
@ -0,0 +1,2 @@
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools", "wheel", "pep518_twin_forkbombs_first"]
|
|
@ -0,0 +1,5 @@
|
||||||
|
from setuptools import setup
|
||||||
|
|
||||||
|
setup(name='pep518_twin_forkbombs_second',
|
||||||
|
version='238',
|
||||||
|
py_modules=['pep518_twin_forkbombs_second'])
|
|
@ -84,14 +84,30 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels):
|
||||||
'wheel', '--no-index',
|
'wheel', '--no-index',
|
||||||
'-f', common_wheels,
|
'-f', common_wheels,
|
||||||
'-f', data.find_links,
|
'-f', data.find_links,
|
||||||
# Add tests/data/packages4, which contains a wheel for
|
|
||||||
# simple==1.0 (needed by requires_simple_extra[extra]).
|
|
||||||
'-f', data.find_links4,
|
|
||||||
data.src.join("pep518_with_extra_and_markers-1.0"),
|
data.src.join("pep518_with_extra_and_markers-1.0"),
|
||||||
use_module=True,
|
use_module=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.timeout(60)
|
||||||
|
@pytest.mark.parametrize('command', ('install', 'wheel'))
|
||||||
|
@pytest.mark.parametrize('package', ('pep518_forkbomb',
|
||||||
|
'pep518_twin_forkbombs_first',
|
||||||
|
'pep518_twin_forkbombs_second'))
|
||||||
|
def test_pep518_forkbombs(script, data, common_wheels, command, package):
|
||||||
|
package_source = next(data.packages.glob(package + '-[0-9]*.tar.gz'))
|
||||||
|
result = script.pip(
|
||||||
|
'wheel', '--no-index', '-v',
|
||||||
|
'-f', common_wheels,
|
||||||
|
'-f', data.find_links,
|
||||||
|
package,
|
||||||
|
expect_error=True,
|
||||||
|
)
|
||||||
|
assert '{1} is already being built: {0} from {1}'.format(
|
||||||
|
package, path_to_url(package_source),
|
||||||
|
) in result.stdout, str(result)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.network
|
@pytest.mark.network
|
||||||
def test_pip_second_command_line_interface_works(script, data):
|
def test_pip_second_command_line_interface_works(script, data):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -100,10 +100,6 @@ class TestData(object):
|
||||||
def packages3(self):
|
def packages3(self):
|
||||||
return self.root.join("packages3")
|
return self.root.join("packages3")
|
||||||
|
|
||||||
@property
|
|
||||||
def packages4(self):
|
|
||||||
return self.root.join("packages4")
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def src(self):
|
def src(self):
|
||||||
return self.root.join("src")
|
return self.root.join("src")
|
||||||
|
@ -132,10 +128,6 @@ class TestData(object):
|
||||||
def find_links3(self):
|
def find_links3(self):
|
||||||
return path_to_url(self.packages3)
|
return path_to_url(self.packages3)
|
||||||
|
|
||||||
@property
|
|
||||||
def find_links4(self):
|
|
||||||
return path_to_url(self.packages4)
|
|
||||||
|
|
||||||
def index_url(self, index="simple"):
|
def index_url(self, index="simple"):
|
||||||
return path_to_url(self.root.join("indexes", index))
|
return path_to_url(self.root.join("indexes", index))
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@ from pip._internal.operations.prepare import RequirementPreparer
|
||||||
from pip._internal.req import InstallRequirement, RequirementSet
|
from pip._internal.req import InstallRequirement, RequirementSet
|
||||||
from pip._internal.req.req_file import process_line
|
from pip._internal.req.req_file import process_line
|
||||||
from pip._internal.req.req_install import parse_editable
|
from pip._internal.req.req_install import parse_editable
|
||||||
|
from pip._internal.req.req_tracker import RequirementTracker
|
||||||
from pip._internal.resolve import Resolver
|
from pip._internal.resolve import Resolver
|
||||||
from pip._internal.utils.misc import read_text_file
|
from pip._internal.utils.misc import read_text_file
|
||||||
from tests.lib import DATA_DIR, assert_raises_regexp, requirements_file
|
from tests.lib import DATA_DIR, assert_raises_regexp, requirements_file
|
||||||
|
@ -47,6 +48,7 @@ class TestRequirementSet(object):
|
||||||
wheel_download_dir=None,
|
wheel_download_dir=None,
|
||||||
progress_bar="on",
|
progress_bar="on",
|
||||||
build_isolation=True,
|
build_isolation=True,
|
||||||
|
req_tracker=RequirementTracker(),
|
||||||
)
|
)
|
||||||
return Resolver(
|
return Resolver(
|
||||||
preparer=preparer, wheel_cache=None,
|
preparer=preparer, wheel_cache=None,
|
||||||
|
|
Loading…
Reference in New Issue