From 3ae521974f2d5913585ecb6ffc90990f9fb9aa43 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Thu, 19 Apr 2018 20:01:26 +0200 Subject: [PATCH 1/3] enable source installs for build dependencies --- docs/reference/pip.rst | 5 +++-- news/5229.feature | 1 + src/pip/_internal/build_env.py | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 news/5229.feature diff --git a/docs/reference/pip.rst b/docs/reference/pip.rst index dd68f65ad..905be1ea6 100644 --- a/docs/reference/pip.rst +++ b/docs/reference/pip.rst @@ -154,8 +154,9 @@ appropriately. installation of build dependencies from source has been disabled until a safe resolution of this issue is found. -* ``pip<18.0`` does not support the use of environment markers and extras, only - version specifiers are respected. +* ``pip<18.0``: only support installing build requirements from wheels, and + does not support the use of environment markers and extras (only version + specifiers are respected). Future Developments diff --git a/news/5229.feature b/news/5229.feature new file mode 100644 index 000000000..8df75ad16 --- /dev/null +++ b/news/5229.feature @@ -0,0 +1 @@ +Add support for installing PEP 518 build dependencies from source. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 612b46352..875ad3f84 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -80,10 +80,13 @@ class BuildEnvironment(object): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--no-user', '--prefix', self.path, '--no-warn-script-location', - '--only-binary', ':all:', ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) if finder.index_urls: args.extend(['-i', finder.index_urls[0]]) for extra_index in finder.index_urls[1:]: From 43b8ed4945f259deed630f7eddfc4d22c12872e7 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 23 Apr 2018 11:41:34 +0200 Subject: [PATCH 2/3] detect fork-bombs during build dependencies installs --- src/pip/_internal/commands/download.py | 4 +- src/pip/_internal/commands/install.py | 4 +- src/pip/_internal/commands/wheel.py | 5 +- src/pip/_internal/operations/prepare.py | 9 +- src/pip/_internal/req/req_tracker.py | 77 ++++++++++++++++++ tests/conftest.py | 3 + .../data/packages/pep518_forkbomb-235.tar.gz | Bin 0 -> 752 bytes .../pep518_twin_forkbombs_first-234.tar.gz | Bin 0 -> 806 bytes .../pep518_twin_forkbombs_second-238.tar.gz | Bin 0 -> 808 bytes .../data/src/pep518_forkbomb-235/MANIFEST.in | 1 + .../pep518_forkbomb-235/pep518_forkbomb.py | 0 .../src/pep518_forkbomb-235/pyproject.toml | 2 + tests/data/src/pep518_forkbomb-235/setup.cfg | 0 tests/data/src/pep518_forkbomb-235/setup.py | 5 ++ .../MANIFEST.in | 1 + .../pep518_twin_forkbombs_first.py | 0 .../pyproject.toml | 2 + .../pep518_twin_forkbombs_first-234/setup.cfg | 0 .../pep518_twin_forkbombs_first-234/setup.py | 5 ++ .../MANIFEST.in | 1 + .../pep518_twin_forkbombs_second.py | 0 .../pyproject.toml | 2 + .../setup.cfg | 0 .../pep518_twin_forkbombs_second-238/setup.py | 5 ++ tests/functional/test_install.py | 19 +++++ tests/unit/test_req.py | 2 + 26 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 src/pip/_internal/req/req_tracker.py create mode 100644 tests/data/packages/pep518_forkbomb-235.tar.gz create mode 100644 tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz create mode 100644 tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz create mode 100644 tests/data/src/pep518_forkbomb-235/MANIFEST.in create mode 100644 tests/data/src/pep518_forkbomb-235/pep518_forkbomb.py create mode 100644 tests/data/src/pep518_forkbomb-235/pyproject.toml create mode 100644 tests/data/src/pep518_forkbomb-235/setup.cfg create mode 100644 tests/data/src/pep518_forkbomb-235/setup.py create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/pep518_twin_forkbombs_first.py create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/setup.cfg create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/setup.py create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/pep518_twin_forkbombs_second.py create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/setup.cfg create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/setup.py diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 66bcbd5c8..cf4827c58 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError from pip._internal.index import FormatControl from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.misc import ensure_dir, normalize_path @@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: @@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index da1146b9c..f42a1d137 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global from pip._internal.operations.check import check_install_conflicts from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet, install_given_reqs +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.status_codes import ERROR from pip._internal.utils.filesystem import check_path_owner @@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="install" ) as directory: requirement_set = RequirementSet( @@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 0fb72c1a9..41893876d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache from pip._internal.exceptions import CommandError, PreviousBuildDirError from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import WheelBuilder @@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand): build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="wheel" ) as directory: + requirement_set = RequirementSet( require_hashes=options.require_hashes, ) @@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand): wheel_download_dir=options.wheel_dir, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3ad721f01..7740c2843 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -141,11 +141,12 @@ class RequirementPreparer(object): """ def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation): + progress_bar, build_isolation, req_tracker): super(RequirementPreparer, self).__init__() self.src_dir = src_dir self.build_dir = build_dir + self.req_tracker = req_tracker # Where still packed archives should be written to. If None, they are # not saved, and are deleted immediately after unpacking. @@ -293,7 +294,8 @@ class RequirementPreparer(object): (req, exc, req.link) ) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: @@ -319,7 +321,8 @@ class RequirementPreparer(object): req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: req.archive(self.download_dir) diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py new file mode 100644 index 000000000..4869bb3a9 --- /dev/null +++ b/src/pip/_internal/req/req_tracker.py @@ -0,0 +1,77 @@ +from __future__ import absolute_import + +import contextlib +import errno +import hashlib +import logging +import os + +from pip._internal.utils.temp_dir import TempDirectory + + +logger = logging.getLogger(__name__) + + +class RequirementTracker(object): + + def __init__(self): + self._root = os.environ.get('PIP_REQ_TRACKER') + if self._root is None: + self._temp_dir = TempDirectory(delete=False, kind='req-tracker') + self._temp_dir.create() + self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path + logger.debug('Created requirements tracker %r', self._root) + else: + self._temp_dir = None + logger.debug('Re-using requirements tracker %r', self._root) + self._entries = set() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + def _entry_path(self, link): + hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() + return os.path.join(self._root, hashed) + + def add(self, req): + link = req.link + info = str(req) + entry_path = self._entry_path(link) + try: + with open(entry_path) as fp: + # Error, these's already a build in progress. + raise LookupError('%s is already being built: %s' + % (link, fp.read())) + except IOError as e: + if e.errno != errno.ENOENT: + raise + assert req not in self._entries + with open(entry_path, 'w') as fp: + fp.write(info) + self._entries.add(req) + logger.debug('Added %s to build tracker %r', req, self._root) + + def remove(self, req): + link = req.link + self._entries.remove(req) + os.unlink(self._entry_path(link)) + logger.debug('Removed %s from build tracker %r', req, self._root) + + def cleanup(self): + for req in set(self._entries): + self.remove(req) + remove = self._temp_dir is not None + if remove: + self._temp_dir.cleanup() + logger.debug('%s build tracker %r', + 'Removed' if remove else 'Cleaned', + self._root) + + @contextlib.contextmanager + def track(self, req): + self.add(req) + yield + self.remove(req) diff --git a/tests/conftest.py b/tests/conftest.py index 1e927fc13..f8e3a41b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,6 +129,9 @@ def isolate(tmpdir): # We want to disable the version check from running in the tests os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: diff --git a/tests/data/packages/pep518_forkbomb-235.tar.gz b/tests/data/packages/pep518_forkbomb-235.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1edce52d8a4dd3d54cc58893710e99abb6fb9654 GIT binary patch literal 752 zcmVV(ErKlfB62bvESW0 zxQG7UHTqk=UDMy@uB$N){U4o=q{30CLgNc%c_h+pX12NB4HMdC^L=0}_JO`!Qf(5w zS0qW8mD}v7`@VbdwX44q2^lh+&?>Z3s1uRZuMJAVC|*TBL_$=NQm!-_aHyWz85=pG*qzc&{jwip*rYml0 z7WLa~r>hr@NK!hYbXW#)l%D3-ufAu6g8y&j|I4&kNB!q!&HrB20>J-GfPxSI*ZqIb zF@JsYzv($1`u`^~{|A`=pPB#9^@NT1ze)ge|J$Zz`F`bp->lLA{NDgkI*f}Ev09@K zV?1O3FQ>eQ>OXh9TK)Gej`_b4j;rA=m=1S+U4@fiP(<;Fb=d5p>I1KIo^iwFqtg(h19|72-%bT4paa5XVFUvzhAZeM0^a%*C5ZDMm@W@&PBbS*M7 zG%j>uascg`U2mH(6o$FxSD@T?V^ehAt#g-Fay($e?O8e zNy}Q2Zb_D|?{kAOcD{}u+s8^miKV@9#BC5eL!MsudDPFGVUT7*)eUn`S^=!YW^8oT zZ0p{2iliBarddq4TV=goHeixsNgE)~gqtdg)K3CRQ+l-BqUX~uGQB<%k_GXvdEO-7 zSA%%S_f{T%o0*9J^ND}=^y2g3QMW06h{t}KR!#g(qY{6E*}5Sq4Dnw(pAI@F$4A}s zW)RDDu}%s(DUy_bBTqC1k3u;kkta=W=>IB>S}ZeI&OW?TPddk^@>7S_avcVgP&}a6 zE1WQhue0TQ3Z6J2-B9@0IQ}15iT~4()~HeEzhP-r|23lp|2N`433uQMSN%e17Vvmq z(V9%|xDo9ulh~`H*sHVHtK-k-K>mxoNSlTy#EkPQP~K_dFssaebOs9pvJQ zr`3CvA~y)%Wfme3(mt#38I9jVp1bdPWE5L52~|in_yAA<0Kr9>RR910 literal 0 HcmV?d00001 diff --git a/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz b/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..75998c3d4df1b86c8f4b770692f7ff3d94a5ca75 GIT binary patch literal 808 zcmV+@1K0c?iwFqZg(h19|72-%bT4paa5XVFUvzhAZeM0^a%*C5ZDMm@b7f<1Ze%So zGdM1EVR8WNnNL%jKpciS^C_5IY-c3y0s+UF92~2yV;h}D+k=x~2-zSN*exttzI{p3 zBpM^enxxUb&jqqs^6%Ye+5JT@(V)AoybTk#&(rHJkGq+hF`p+r-7t4|l$S$F96|<1 z>NM^h$0*b^DQytLF)f7{gj!V58p;~TGwG$8rgTNf(v&^g@2Tgk9{)73lE_ocO8I!rF`~Be0*`UWlgXQexqkh~vI#r)Nwidji#{^4y zEb$9B3X|*X8J9{f+=$(v_`Czp*q6 zd9tU`I#F9*%=WZV^!gxreGHnP#;!Em(mHr#f z$bZW*2{jdBP}_n3YavYhDDN?CW^WHM;Qs>me^D&O|8f0y;=fI81O9JB{*{%T<;?#N z>OTfGEQ0(uqyF>A-VpVl|6~3i*Y;QFzhO-GfA*lgT>n{E|E-07%42PqvE)3;v@jMt zm7^AQ^vBM-q?`?!TU8@vYg^TZ+>B<0=2kg$CXJ$SW8TXnmNhS{U#9aOnr_ zCH_<6ZvDq7{WnqnT@#nxJdApJR%DXJ9W`Y?^Dt$Z*3>R*(`li$t<`R?7>foc^M_17 zOfy;Qz~47{|M&aB>hAx Date: Fri, 27 Apr 2018 11:43:25 +0200 Subject: [PATCH 3/3] tests: drop `package4` test data No need for a wheel of `simple` now that build dependencies support source installs. --- .../packages4/simple-1.0-py2.py3-none-any.whl | Bin 1745 -> 0 bytes tests/functional/test_install.py | 3 --- tests/lib/__init__.py | 8 -------- 3 files changed, 11 deletions(-) delete mode 100644 tests/data/packages4/simple-1.0-py2.py3-none-any.whl diff --git a/tests/data/packages4/simple-1.0-py2.py3-none-any.whl b/tests/data/packages4/simple-1.0-py2.py3-none-any.whl deleted file mode 100644 index 3b91f5e0102903daa87cd40f85828d90ca61433b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1745 zcmWIWW@Zs#U|`^2(4S@C*wDlp#tG!H1F;Yg7iZ=c%u{r100!{f1#7ekK%S|mwOi3(B)XOT)&x`Fp&2`8?z_os9 z%2I`euguOzk_F^WX^)h2|ibYKD}3q2#7gXJLfcyi_?m|MJv`k+-l0fX8Dcp z#3}{xZRuC1PKS)Pi3{YI8*0!s$w=IRg*i^omgE=2=cJaU=IE7FlxUwjqkF>hDSD`V zyL)M0JTSEKfLIDwsD*pDy83XR_wYS?k=I*S>)e_1n}ZCl7(XcTKkKdQrE^k$Q;>(( zNgcgTefE`s7Yr^Kn_V!z@PzBsdHsuDb*}Mh=z5(!>9Z+FLsQH5DVL|u_8^TFCYR2h zKJR_i_sr^+P6=g~9cW2oaaH&K2B33J0kJGDXZgB@IJ!87IL7wwYHYwTd3<~3RgQ(t_Zwx{9B z-Mp38cDP*FE|Pb|=g=QCr-xl>+nvhDz@W&)z#xsw=|Qf}{y{FK7rpZr8;Gzzs9pL` z*oXJ&)uTFRlv~@Jm^d%w2(WON`6o_xs;@OwPAyxr{)_C~V$FBQw$*qB-$|af?9Yy( zE$3V!e@@;L8lNY0fLXcv<($iU!2wT>o|IZRJNe%94Jwa!=X9@`Xz}%D=fCEL1J3&n zOkuneez-fc)$M_dU*G$W!m0eX%>6#U(R^n*@AfUec<#zo9*+GwVn^e)>gJ1I;WC`^ z$*h3s&IbFn!bTT4Z7<}1Y2IoZV;57?sde0cj*-p%>nm8N&yN)rPH>-9<$CJ~n_tVL zHmQg<`57|X3;S|U7s+?}I@&7ymVfam;WqP)=}o(?9{DSE=uVGz=C%FypO2kh<@vbf ztX%BLO-jj<{x2@rXIOlVU%;B~IRD_pl4G|^moLlcQ9hdaHe7RMKYMM;z6HWZIyY@_ z+??oh`%~ODo9KW2^Iu=>dLwF5FsnG}N%i%!>}PlBxb9jti8Vo@=WlG(uJ5<+@BhaX z;LXS+!i>AP0!9%SG&F)J%)$#%mbGg=;okjYJ@oUZWMYZLm0)+ ZjKe5MmJ9G^WdkW@2SPret!r68JOF_(W}N^4 diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index a2bcd3d38..0ef340af0 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -80,9 +80,6 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels): 'wheel', '--no-index', '-f', common_wheels, '-f', data.find_links, - # Add tests/data/packages4, which contains a wheel for - # simple==1.0 (needed by requires_simple_extra[extra]). - '-f', data.find_links4, data.src.join("pep518_with_extra_and_markers-1.0"), use_module=True, ) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b445e89e8..e0e08fd76 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -100,10 +100,6 @@ class TestData(object): def packages3(self): return self.root.join("packages3") - @property - def packages4(self): - return self.root.join("packages4") - @property def src(self): return self.root.join("src") @@ -132,10 +128,6 @@ class TestData(object): def find_links3(self): return path_to_url(self.packages3) - @property - def find_links4(self): - return path_to_url(self.packages4) - def index_url(self, index="simple"): return path_to_url(self.root.join("indexes", index))