detect fork-bombs during build dependencies installs

src/pip/_internal/commands/download.py

@@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError
 from pip._internal.index import FormatControl
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import RequirementSet
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.utils.filesystem import check_path_owner
 from pip._internal.utils.misc import ensure_dir, normalize_path
@@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand):
                 )
                 options.cache_dir = None
 
-            with TempDirectory(
+            with RequirementTracker() as req_tracker, TempDirectory(
                 options.build_dir, delete=build_delete, kind="download"
             ) as directory:
 
@@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand):
                     wheel_download_dir=None,
                     progress_bar=options.progress_bar,
                     build_isolation=options.build_isolation,
+                    req_tracker=req_tracker,
                 )
 
                 resolver = Resolver(

src/pip/_internal/commands/install.py

@@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global
 from pip._internal.operations.check import check_install_conflicts
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import RequirementSet, install_given_reqs
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.status_codes import ERROR
 from pip._internal.utils.filesystem import check_path_owner
@@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand):
                 )
                 options.cache_dir = None
 
-            with TempDirectory(
+            with RequirementTracker() as req_tracker, TempDirectory(
                 options.build_dir, delete=build_delete, kind="install"
             ) as directory:
                 requirement_set = RequirementSet(
@@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand):
                         wheel_download_dir=None,
                         progress_bar=options.progress_bar,
                         build_isolation=options.build_isolation,
+                        req_tracker=req_tracker,
                     )
 
                     resolver = Resolver(

src/pip/_internal/commands/wheel.py

@@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache
 from pip._internal.exceptions import CommandError, PreviousBuildDirError
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import RequirementSet
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.wheel import WheelBuilder
@@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand):
             build_delete = (not (options.no_clean or options.build_dir))
             wheel_cache = WheelCache(options.cache_dir, options.format_control)
 
-            with TempDirectory(
+            with RequirementTracker() as req_tracker, TempDirectory(
                 options.build_dir, delete=build_delete, kind="wheel"
             ) as directory:
+
                 requirement_set = RequirementSet(
                     require_hashes=options.require_hashes,
                 )
@@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand):
                         wheel_download_dir=options.wheel_dir,
                         progress_bar=options.progress_bar,
                         build_isolation=options.build_isolation,
+                        req_tracker=req_tracker,
                     )
 
                     resolver = Resolver(

src/pip/_internal/operations/prepare.py

@@ -141,11 +141,12 @@ class RequirementPreparer(object):
     """
 
     def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
-                 progress_bar, build_isolation):
+                 progress_bar, build_isolation, req_tracker):
         super(RequirementPreparer, self).__init__()
 
         self.src_dir = src_dir
         self.build_dir = build_dir
+        self.req_tracker = req_tracker
 
         # Where still packed archives should be written to. If None, they are
         # not saved, and are deleted immediately after unpacking.
@@ -293,7 +294,8 @@ class RequirementPreparer(object):
                     (req, exc, req.link)
                 )
             abstract_dist = make_abstract_dist(req)
-            abstract_dist.prep_for_dist(finder, self.build_isolation)
+            with self.req_tracker.track(req):
+                abstract_dist.prep_for_dist(finder, self.build_isolation)
             if self._download_should_save:
                 # Make a .zip of the source_dir we already created.
                 if req.link.scheme in vcs.all_schemes:
@@ -319,7 +321,8 @@ class RequirementPreparer(object):
             req.update_editable(not self._download_should_save)
 
             abstract_dist = make_abstract_dist(req)
-            abstract_dist.prep_for_dist(finder, self.build_isolation)
+            with self.req_tracker.track(req):
+                abstract_dist.prep_for_dist(finder, self.build_isolation)
 
             if self._download_should_save:
                 req.archive(self.download_dir)

src/pip/_internal/req/req_tracker.py

@@ -0,0 +1,77 @@
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import hashlib
+import logging
+import os
+
+from pip._internal.utils.temp_dir import TempDirectory
+
+
+logger = logging.getLogger(__name__)
+
+
+class RequirementTracker(object):
+
+    def __init__(self):
+        self._root = os.environ.get('PIP_REQ_TRACKER')
+        if self._root is None:
+            self._temp_dir = TempDirectory(delete=False, kind='req-tracker')
+            self._temp_dir.create()
+            self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path
+            logger.debug('Created requirements tracker %r', self._root)
+        else:
+            self._temp_dir = None
+            logger.debug('Re-using requirements tracker %r', self._root)
+        self._entries = set()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.cleanup()
+
+    def _entry_path(self, link):
+        hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
+        return os.path.join(self._root, hashed)
+
+    def add(self, req):
+        link = req.link
+        info = str(req)
+        entry_path = self._entry_path(link)
+        try:
+            with open(entry_path) as fp:
+                # Error, these's already a build in progress.
+                raise LookupError('%s is already being built: %s'
+                                  % (link, fp.read()))
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            assert req not in self._entries
+            with open(entry_path, 'w') as fp:
+                fp.write(info)
+            self._entries.add(req)
+            logger.debug('Added %s to build tracker %r', req, self._root)
+
+    def remove(self, req):
+        link = req.link
+        self._entries.remove(req)
+        os.unlink(self._entry_path(link))
+        logger.debug('Removed %s from build tracker %r', req, self._root)
+
+    def cleanup(self):
+        for req in set(self._entries):
+            self.remove(req)
+        remove = self._temp_dir is not None
+        if remove:
+            self._temp_dir.cleanup()
+        logger.debug('%s build tracker %r',
+                     'Removed' if remove else 'Cleaned',
+                     self._root)
+
+    @contextlib.contextmanager
+    def track(self, req):
+        self.add(req)
+        yield
+        self.remove(req)

tests/conftest.py

@@ -129,6 +129,9 @@ def isolate(tmpdir):
     # We want to disable the version check from running in the tests
     os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true"
 
+    # Make sure tests don't share a requirements tracker.
+    os.environ.pop('PIP_REQ_TRACKER', None)
+
     # FIXME: Windows...
     os.makedirs(os.path.join(home_dir, ".config", "git"))
     with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp:

tests/data/src/pep518_forkbomb-235/MANIFEST.in

@@ -0,0 +1 @@
+include pyproject.toml

tests/data/src/pep518_forkbomb-235/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "pep518_forkbomb"]

tests/data/src/pep518_forkbomb-235/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+setup(name='pep518_forkbomb',
+      version='235',
+      py_modules=['pep518_forkbomb'])

tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in

@@ -0,0 +1 @@
+include pyproject.toml

tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "pep518_twin_forkbombs_second"]

tests/data/src/pep518_twin_forkbombs_first-234/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+setup(name='pep518_twin_forkbombs_first',
+      version='234',
+      py_modules=['pep518_twin_forkbombs_first'])

tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in

@@ -0,0 +1 @@
+include pyproject.toml

tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "pep518_twin_forkbombs_first"]

tests/data/src/pep518_twin_forkbombs_second-238/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+setup(name='pep518_twin_forkbombs_second',
+      version='238',
+      py_modules=['pep518_twin_forkbombs_second'])

tests/functional/test_install.py

@@ -88,6 +88,25 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels):
     )
 
 
+@pytest.mark.timeout(60)
+@pytest.mark.parametrize('command', ('install', 'wheel'))
+@pytest.mark.parametrize('package', ('pep518_forkbomb',
+                                     'pep518_twin_forkbombs_first',
+                                     'pep518_twin_forkbombs_second'))
+def test_pep518_forkbombs(script, data, common_wheels, command, package):
+    package_source = next(data.packages.glob(package + '-[0-9]*.tar.gz'))
+    result = script.pip(
+        'wheel', '--no-index', '-v',
+        '-f', common_wheels,
+        '-f', data.find_links,
+        package,
+        expect_error=True,
+    )
+    assert '{1} is already being built: {0} from {1}'.format(
+        package, path_to_url(package_source),
+    ) in result.stdout, str(result)
+
+
 @pytest.mark.network
 def test_pip_second_command_line_interface_works(script, data):
     """

tests/unit/test_req.py

@@ -19,6 +19,7 @@ from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req import InstallRequirement, RequirementSet
 from pip._internal.req.req_file import process_line
 from pip._internal.req.req_install import parse_editable
+from pip._internal.req.req_tracker import RequirementTracker
 from pip._internal.resolve import Resolver
 from pip._internal.utils.misc import read_text_file
 from tests.lib import DATA_DIR, assert_raises_regexp, requirements_file
@@ -47,6 +48,7 @@ class TestRequirementSet(object):
             wheel_download_dir=None,
             progress_bar="on",
             build_isolation=True,
+            req_tracker=RequirementTracker(),
         )
         return Resolver(
             preparer=preparer, wheel_cache=None,