Merge pull request #5336 from benoit-pierre/src_build_reqs_in_pep518

PEP 518: enable source installs for build dependencies

docs/reference/pip.rst

@@ -154,8 +154,9 @@ appropriately.
   installation of build dependencies from source has been disabled until a safe
   resolution of this issue is found.
 
-* ``pip<18.0`` does not support the use of environment markers and extras, only
+* ``pip<18.0``: only support installing build requirements from wheels, and
-  version specifiers are respected.
+  does not support the use of environment markers and extras (only version
+  specifiers are respected).
 
 
 Future Developments

news/5229.feature

@@ -0,0 +1 @@
+Add support for installing PEP 518 build dependencies from source.

src/pip/_internal/build_env.py

@@ -79,10 +79,13 @@ class BuildEnvironment(object):
         args = [
             sys.executable, '-m', 'pip', 'install', '--ignore-installed',
             '--no-user', '--prefix', self.path, '--no-warn-script-location',
-            '--only-binary', ':all:',
         ]
         if logger.getEffectiveLevel() <= logging.DEBUG:
             args.append('-v')
+        for format_control in ('no_binary', 'only_binary'):
+            formats = getattr(finder.format_control, format_control)
+            args.extend(('--' + format_control.replace('_', '-'),
+                         ','.join(sorted(formats or {':none:'}))))
         if finder.index_urls:
             args.extend(['-i', finder.index_urls[0]])
             for extra_index in finder.index_urls[1:]:

src/pip/_internal/commands/download.py

@@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError
 from pip._internal.index import FormatControl
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import RequirementSet
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.utils.filesystem import check_path_owner
 from pip._internal.utils.misc import ensure_dir, normalize_path
@@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand):
                 )
                 options.cache_dir = None
 
-            with TempDirectory(
+            with RequirementTracker() as req_tracker, TempDirectory(
                 options.build_dir, delete=build_delete, kind="download"
             ) as directory:
 
@@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand):
                     wheel_download_dir=None,
                     progress_bar=options.progress_bar,
                     build_isolation=options.build_isolation,
+                    req_tracker=req_tracker,
                 )
 
                 resolver = Resolver(

src/pip/_internal/commands/install.py

@@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global
 from pip._internal.operations.check import check_install_conflicts
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import RequirementSet, install_given_reqs
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.status_codes import ERROR
 from pip._internal.utils.filesystem import check_path_owner
@@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand):
                 )
                 options.cache_dir = None
 
-            with TempDirectory(
+            with RequirementTracker() as req_tracker, TempDirectory(
                 options.build_dir, delete=build_delete, kind="install"
             ) as directory:
                 requirement_set = RequirementSet(
@@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand):
                         wheel_download_dir=None,
                         progress_bar=options.progress_bar,
                         build_isolation=options.build_isolation,
+                        req_tracker=req_tracker,
                     )
 
                     resolver = Resolver(

src/pip/_internal/commands/wheel.py

@@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache
 from pip._internal.exceptions import CommandError, PreviousBuildDirError
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import RequirementSet
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.wheel import WheelBuilder
@@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand):
             build_delete = (not (options.no_clean or options.build_dir))
             wheel_cache = WheelCache(options.cache_dir, options.format_control)
 
-            with TempDirectory(
+            with RequirementTracker() as req_tracker, TempDirectory(
                 options.build_dir, delete=build_delete, kind="wheel"
             ) as directory:
+
                 requirement_set = RequirementSet(
                     require_hashes=options.require_hashes,
                 )
@@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand):
                         wheel_download_dir=options.wheel_dir,
                         progress_bar=options.progress_bar,
                         build_isolation=options.build_isolation,
+                        req_tracker=req_tracker,
                     )
 
                     resolver = Resolver(

src/pip/_internal/operations/prepare.py

@@ -141,11 +141,12 @@ class RequirementPreparer(object):
     """
 
     def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
-                 progress_bar, build_isolation):
+                 progress_bar, build_isolation, req_tracker):
         super(RequirementPreparer, self).__init__()
 
         self.src_dir = src_dir
         self.build_dir = build_dir
+        self.req_tracker = req_tracker
 
         # Where still packed archives should be written to. If None, they are
         # not saved, and are deleted immediately after unpacking.
@@ -293,7 +294,8 @@ class RequirementPreparer(object):
                     (req, exc, req.link)
                 )
             abstract_dist = make_abstract_dist(req)
-            abstract_dist.prep_for_dist(finder, self.build_isolation)
+            with self.req_tracker.track(req):
+                abstract_dist.prep_for_dist(finder, self.build_isolation)
             if self._download_should_save:
                 # Make a .zip of the source_dir we already created.
                 if req.link.scheme in vcs.all_schemes:
@@ -319,7 +321,8 @@ class RequirementPreparer(object):
             req.update_editable(not self._download_should_save)
 
             abstract_dist = make_abstract_dist(req)
-            abstract_dist.prep_for_dist(finder, self.build_isolation)
+            with self.req_tracker.track(req):
+                abstract_dist.prep_for_dist(finder, self.build_isolation)
 
             if self._download_should_save:
                 req.archive(self.download_dir)

src/pip/_internal/req/req_tracker.py

@@ -0,0 +1,77 @@
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import hashlib
+import logging
+import os
+
+from pip._internal.utils.temp_dir import TempDirectory
+
+
+logger = logging.getLogger(__name__)
+
+
+class RequirementTracker(object):
+
+    def __init__(self):
+        self._root = os.environ.get('PIP_REQ_TRACKER')
+        if self._root is None:
+            self._temp_dir = TempDirectory(delete=False, kind='req-tracker')
+            self._temp_dir.create()
+            self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path
+            logger.debug('Created requirements tracker %r', self._root)
+        else:
+            self._temp_dir = None
+            logger.debug('Re-using requirements tracker %r', self._root)
+        self._entries = set()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.cleanup()
+
+    def _entry_path(self, link):
+        hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
+        return os.path.join(self._root, hashed)
+
+    def add(self, req):
+        link = req.link
+        info = str(req)
+        entry_path = self._entry_path(link)
+        try:
+            with open(entry_path) as fp:
+                # Error, these's already a build in progress.
+                raise LookupError('%s is already being built: %s'
+                                  % (link, fp.read()))
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            assert req not in self._entries
+            with open(entry_path, 'w') as fp:
+                fp.write(info)
+            self._entries.add(req)
+            logger.debug('Added %s to build tracker %r', req, self._root)
+
+    def remove(self, req):
+        link = req.link
+        self._entries.remove(req)
+        os.unlink(self._entry_path(link))
+        logger.debug('Removed %s from build tracker %r', req, self._root)
+
+    def cleanup(self):
+        for req in set(self._entries):
+            self.remove(req)
+        remove = self._temp_dir is not None
+        if remove:
+            self._temp_dir.cleanup()
+        logger.debug('%s build tracker %r',
+                     'Removed' if remove else 'Cleaned',
+                     self._root)
+
+    @contextlib.contextmanager
+    def track(self, req):
+        self.add(req)
+        yield
+        self.remove(req)

tests/conftest.py

@@ -129,6 +129,9 @@ def isolate(tmpdir):
     # We want to disable the version check from running in the tests
     os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true"
 
+    # Make sure tests don't share a requirements tracker.
+    os.environ.pop('PIP_REQ_TRACKER', None)
+
     # FIXME: Windows...
     os.makedirs(os.path.join(home_dir, ".config", "git"))
     with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp:

tests/data/src/pep518_forkbomb-235/MANIFEST.in

@@ -0,0 +1 @@
+include pyproject.toml

tests/data/src/pep518_forkbomb-235/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "pep518_forkbomb"]

tests/data/src/pep518_forkbomb-235/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+setup(name='pep518_forkbomb',
+      version='235',
+      py_modules=['pep518_forkbomb'])

tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in

@@ -0,0 +1 @@
+include pyproject.toml

tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "pep518_twin_forkbombs_second"]

tests/data/src/pep518_twin_forkbombs_first-234/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+setup(name='pep518_twin_forkbombs_first',
+      version='234',
+      py_modules=['pep518_twin_forkbombs_first'])

tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in

@@ -0,0 +1 @@
+include pyproject.toml

tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "pep518_twin_forkbombs_first"]

tests/data/src/pep518_twin_forkbombs_second-238/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+setup(name='pep518_twin_forkbombs_second',
+      version='238',
+      py_modules=['pep518_twin_forkbombs_second'])

tests/functional/test_install.py

@@ -84,14 +84,30 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels):
         'wheel', '--no-index',
         '-f', common_wheels,
         '-f', data.find_links,
-        # Add tests/data/packages4, which contains a wheel for
-        # simple==1.0 (needed by requires_simple_extra[extra]).
-        '-f', data.find_links4,
         data.src.join("pep518_with_extra_and_markers-1.0"),
         use_module=True,
     )
 
 
+@pytest.mark.timeout(60)
+@pytest.mark.parametrize('command', ('install', 'wheel'))
+@pytest.mark.parametrize('package', ('pep518_forkbomb',
+                                     'pep518_twin_forkbombs_first',
+                                     'pep518_twin_forkbombs_second'))
+def test_pep518_forkbombs(script, data, common_wheels, command, package):
+    package_source = next(data.packages.glob(package + '-[0-9]*.tar.gz'))
+    result = script.pip(
+        'wheel', '--no-index', '-v',
+        '-f', common_wheels,
+        '-f', data.find_links,
+        package,
+        expect_error=True,
+    )
+    assert '{1} is already being built: {0} from {1}'.format(
+        package, path_to_url(package_source),
+    ) in result.stdout, str(result)
+
+
 @pytest.mark.network
 def test_pip_second_command_line_interface_works(script, data):
     """

tests/lib/__init__.py

@@ -100,10 +100,6 @@ class TestData(object):
     def packages3(self):
         return self.root.join("packages3")
 
-    @property
-    def packages4(self):
-        return self.root.join("packages4")
-
     @property
     def src(self):
         return self.root.join("src")
@@ -132,10 +128,6 @@ class TestData(object):
     def find_links3(self):
         return path_to_url(self.packages3)
 
-    @property
-    def find_links4(self):
-        return path_to_url(self.packages4)
-
     def index_url(self, index="simple"):
         return path_to_url(self.root.join("indexes", index))
 

tests/unit/test_req.py

@@ -19,6 +19,7 @@ from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import InstallRequirement, RequirementSet
 from pip._internal.req.req_file import process_line
 from pip._internal.req.req_install import parse_editable
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.utils.misc import read_text_file
 from tests.lib import DATA_DIR, assert_raises_regexp, requirements_file
@@ -47,6 +48,7 @@ class TestRequirementSet(object):
             wheel_download_dir=None,
             progress_bar="on",
             build_isolation=True,
+            req_tracker=RequirementTracker(),
         )
         return Resolver(
             preparer=preparer, wheel_cache=None,