diff --git a/docs/reference/pip.rst b/docs/reference/pip.rst index dd68f65ad..905be1ea6 100644 --- a/docs/reference/pip.rst +++ b/docs/reference/pip.rst @@ -154,8 +154,9 @@ appropriately. installation of build dependencies from source has been disabled until a safe resolution of this issue is found. -* ``pip<18.0`` does not support the use of environment markers and extras, only - version specifiers are respected. +* ``pip<18.0``: only support installing build requirements from wheels, and + does not support the use of environment markers and extras (only version + specifiers are respected). Future Developments diff --git a/news/5229.feature b/news/5229.feature new file mode 100644 index 000000000..8df75ad16 --- /dev/null +++ b/news/5229.feature @@ -0,0 +1 @@ +Add support for installing PEP 518 build dependencies from source. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 568570885..f225f76d6 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -79,10 +79,13 @@ class BuildEnvironment(object): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--no-user', '--prefix', self.path, '--no-warn-script-location', - '--only-binary', ':all:', ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) if finder.index_urls: args.extend(['-i', finder.index_urls[0]]) for extra_index in finder.index_urls[1:]: diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 66bcbd5c8..cf4827c58 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError from pip._internal.index import FormatControl from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.misc import ensure_dir, normalize_path @@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: @@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index da1146b9c..f42a1d137 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global from pip._internal.operations.check import check_install_conflicts from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet, install_given_reqs +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.status_codes import ERROR from pip._internal.utils.filesystem import check_path_owner @@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="install" ) as directory: requirement_set = RequirementSet( @@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 0fb72c1a9..41893876d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache from pip._internal.exceptions import CommandError, PreviousBuildDirError from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import WheelBuilder @@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand): build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="wheel" ) as directory: + requirement_set = RequirementSet( require_hashes=options.require_hashes, ) @@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand): wheel_download_dir=options.wheel_dir, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3ad721f01..7740c2843 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -141,11 +141,12 @@ class RequirementPreparer(object): """ def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation): + progress_bar, build_isolation, req_tracker): super(RequirementPreparer, self).__init__() self.src_dir = src_dir self.build_dir = build_dir + self.req_tracker = req_tracker # Where still packed archives should be written to. If None, they are # not saved, and are deleted immediately after unpacking. @@ -293,7 +294,8 @@ class RequirementPreparer(object): (req, exc, req.link) ) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: @@ -319,7 +321,8 @@ class RequirementPreparer(object): req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: req.archive(self.download_dir) diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py new file mode 100644 index 000000000..4869bb3a9 --- /dev/null +++ b/src/pip/_internal/req/req_tracker.py @@ -0,0 +1,77 @@ +from __future__ import absolute_import + +import contextlib +import errno +import hashlib +import logging +import os + +from pip._internal.utils.temp_dir import TempDirectory + + +logger = logging.getLogger(__name__) + + +class RequirementTracker(object): + + def __init__(self): + self._root = os.environ.get('PIP_REQ_TRACKER') + if self._root is None: + self._temp_dir = TempDirectory(delete=False, kind='req-tracker') + self._temp_dir.create() + self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path + logger.debug('Created requirements tracker %r', self._root) + else: + self._temp_dir = None + logger.debug('Re-using requirements tracker %r', self._root) + self._entries = set() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + def _entry_path(self, link): + hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() + return os.path.join(self._root, hashed) + + def add(self, req): + link = req.link + info = str(req) + entry_path = self._entry_path(link) + try: + with open(entry_path) as fp: + # Error, these's already a build in progress. + raise LookupError('%s is already being built: %s' + % (link, fp.read())) + except IOError as e: + if e.errno != errno.ENOENT: + raise + assert req not in self._entries + with open(entry_path, 'w') as fp: + fp.write(info) + self._entries.add(req) + logger.debug('Added %s to build tracker %r', req, self._root) + + def remove(self, req): + link = req.link + self._entries.remove(req) + os.unlink(self._entry_path(link)) + logger.debug('Removed %s from build tracker %r', req, self._root) + + def cleanup(self): + for req in set(self._entries): + self.remove(req) + remove = self._temp_dir is not None + if remove: + self._temp_dir.cleanup() + logger.debug('%s build tracker %r', + 'Removed' if remove else 'Cleaned', + self._root) + + @contextlib.contextmanager + def track(self, req): + self.add(req) + yield + self.remove(req) diff --git a/tests/conftest.py b/tests/conftest.py index 1e927fc13..f8e3a41b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,6 +129,9 @@ def isolate(tmpdir): # We want to disable the version check from running in the tests os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: diff --git a/tests/data/packages/pep518_forkbomb-235.tar.gz b/tests/data/packages/pep518_forkbomb-235.tar.gz new file mode 100644 index 000000000..1edce52d8 Binary files /dev/null and b/tests/data/packages/pep518_forkbomb-235.tar.gz differ diff --git a/tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz b/tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz new file mode 100644 index 000000000..2ca9452b1 Binary files /dev/null and b/tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz differ diff --git a/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz b/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz new file mode 100644 index 000000000..75998c3d4 Binary files /dev/null and b/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz differ diff --git a/tests/data/packages4/simple-1.0-py2.py3-none-any.whl b/tests/data/packages4/simple-1.0-py2.py3-none-any.whl deleted file mode 100644 index 3b91f5e01..000000000 Binary files a/tests/data/packages4/simple-1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/data/src/pep518_forkbomb-235/MANIFEST.in b/tests/data/src/pep518_forkbomb-235/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_forkbomb-235/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/tests/data/src/pep518_forkbomb-235/pep518_forkbomb.py b/tests/data/src/pep518_forkbomb-235/pep518_forkbomb.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/src/pep518_forkbomb-235/pyproject.toml b/tests/data/src/pep518_forkbomb-235/pyproject.toml new file mode 100644 index 000000000..1138bfb0d --- /dev/null +++ b/tests/data/src/pep518_forkbomb-235/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "pep518_forkbomb"] diff --git a/tests/data/src/pep518_forkbomb-235/setup.cfg b/tests/data/src/pep518_forkbomb-235/setup.cfg new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/src/pep518_forkbomb-235/setup.py b/tests/data/src/pep518_forkbomb-235/setup.py new file mode 100644 index 000000000..c8bc29287 --- /dev/null +++ b/tests/data/src/pep518_forkbomb-235/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup(name='pep518_forkbomb', + version='235', + py_modules=['pep518_forkbomb']) diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in b/tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/pep518_twin_forkbombs_first.py b/tests/data/src/pep518_twin_forkbombs_first-234/pep518_twin_forkbombs_first.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml b/tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml new file mode 100644 index 000000000..ca106ad69 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "pep518_twin_forkbombs_second"] diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/setup.cfg b/tests/data/src/pep518_twin_forkbombs_first-234/setup.cfg new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/setup.py b/tests/data/src/pep518_twin_forkbombs_first-234/setup.py new file mode 100644 index 000000000..55e9bbfb1 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_first-234/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup(name='pep518_twin_forkbombs_first', + version='234', + py_modules=['pep518_twin_forkbombs_first']) diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in b/tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/pep518_twin_forkbombs_second.py b/tests/data/src/pep518_twin_forkbombs_second-238/pep518_twin_forkbombs_second.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml b/tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml new file mode 100644 index 000000000..194a86848 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "pep518_twin_forkbombs_first"] diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/setup.cfg b/tests/data/src/pep518_twin_forkbombs_second-238/setup.cfg new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/setup.py b/tests/data/src/pep518_twin_forkbombs_second-238/setup.py new file mode 100644 index 000000000..985af51df --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_second-238/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup(name='pep518_twin_forkbombs_second', + version='238', + py_modules=['pep518_twin_forkbombs_second']) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index c8095c224..201d9a6a9 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -84,14 +84,30 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels): 'wheel', '--no-index', '-f', common_wheels, '-f', data.find_links, - # Add tests/data/packages4, which contains a wheel for - # simple==1.0 (needed by requires_simple_extra[extra]). - '-f', data.find_links4, data.src.join("pep518_with_extra_and_markers-1.0"), use_module=True, ) +@pytest.mark.timeout(60) +@pytest.mark.parametrize('command', ('install', 'wheel')) +@pytest.mark.parametrize('package', ('pep518_forkbomb', + 'pep518_twin_forkbombs_first', + 'pep518_twin_forkbombs_second')) +def test_pep518_forkbombs(script, data, common_wheels, command, package): + package_source = next(data.packages.glob(package + '-[0-9]*.tar.gz')) + result = script.pip( + 'wheel', '--no-index', '-v', + '-f', common_wheels, + '-f', data.find_links, + package, + expect_error=True, + ) + assert '{1} is already being built: {0} from {1}'.format( + package, path_to_url(package_source), + ) in result.stdout, str(result) + + @pytest.mark.network def test_pip_second_command_line_interface_works(script, data): """ diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b445e89e8..e0e08fd76 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -100,10 +100,6 @@ class TestData(object): def packages3(self): return self.root.join("packages3") - @property - def packages4(self): - return self.root.join("packages4") - @property def src(self): return self.root.join("src") @@ -132,10 +128,6 @@ class TestData(object): def find_links3(self): return path_to_url(self.packages3) - @property - def find_links4(self): - return path_to_url(self.packages4) - def index_url(self, index="simple"): return path_to_url(self.root.join("indexes", index)) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index f9e73afc3..218eb21be 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -19,6 +19,7 @@ from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import InstallRequirement, RequirementSet from pip._internal.req.req_file import process_line from pip._internal.req.req_install import parse_editable +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.misc import read_text_file from tests.lib import DATA_DIR, assert_raises_regexp, requirements_file @@ -47,6 +48,7 @@ class TestRequirementSet(object): wheel_download_dir=None, progress_bar="on", build_isolation=True, + req_tracker=RequirementTracker(), ) return Resolver( preparer=preparer, wheel_cache=None,