From 72f219c410b0ce25f7da44be6172c1e3766bbe6b Mon Sep 17 00:00:00 2001 From: BrownTruck Date: Mon, 5 Feb 2018 06:20:50 -0500 Subject: [PATCH 1/2] Add expansion of environment variables in requirement files (#3728) --- docs/reference/pip_install.rst | 37 ++++++++++++++++++++++ news/3728.feature | 2 ++ src/pip/_internal/req/req_file.py | 34 ++++++++++++++++++++ tests/unit/test_req_file.py | 52 +++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 news/3728.feature diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index e54337f82..e9d0ec523 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -171,6 +171,28 @@ You can also refer to :ref:`constraints files `, like this:: -c some_constraints.txt +.. _`Using Environment Variables`: + +Using Environment Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since version 10, pip supports the use of environment variables inside the +requirements file. You can now store sensitive data (tokens, keys, etc.) in +environment variables and only specify the variable name for your requirements, +letting pip lookup the value at runtime. This approach aligns with the commonly +used `12-factor configuration pattern `_. + +You have to use the POSIX format for variable names including brackets around +the uppercase name as shown in this example: ``${API_TOKEN}``. pip will attempt +to find the corresponding environment variable defined on the host system at +runtime. + +.. note:: + + There is no support for other variable expansion syntaxes such as + ``$VARIABLE`` and ``%VARIABLE%``. + + .. _`Example Requirements File`: Example Requirements File @@ -432,6 +454,21 @@ Tags or revisions can be installed like so:: [-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject [-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject +Using Environment Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since version 10, pip also makes it possible to use environment variables which +makes it possible to reference private repositories without having to store +access tokens in the requirements file. For example, a private git repository +allowing Basic Auth for authentication can be refenced like this:: + + [-e] git+http://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject + [-e] git+https://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject + +.. note:: + + Only ``${VARIABLE}`` is supported, other formats like ``$VARIABLE`` or + ``%VARIABLE%`` won't work. Finding Packages ++++++++++++++++ diff --git a/news/3728.feature b/news/3728.feature new file mode 100644 index 000000000..3080f88f7 --- /dev/null +++ b/news/3728.feature @@ -0,0 +1,2 @@ +pip now supports environment variable expansion in requirement files using +only ``${VARIABLE}`` syntax on all platforms. diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index 08e1b119f..f86849796 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -23,6 +23,12 @@ __all__ = ['parse_requirements'] SCHEME_RE = re.compile(r'^(http|https|file):', re.I) COMMENT_RE = re.compile(r'(^|\s)+#.*$') +# Matches environment variable-style values in '${MY_VARIABLE_1}' with the +# variable name consisting of only uppercase letters, digits or the '_' +# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1, +# 2013 Edition. +ENV_VAR_RE = re.compile(r'(?P\$\{(?P[A-Z0-9_]+)\})') + SUPPORTED_OPTIONS = [ cmdoptions.constraints, cmdoptions.editable, @@ -94,6 +100,7 @@ def preprocess(content, options): lines_enum = join_lines(lines_enum) lines_enum = ignore_comments(lines_enum) lines_enum = skip_regex(lines_enum, options) + lines_enum = expand_env_variables(lines_enum) return lines_enum @@ -302,3 +309,30 @@ def skip_regex(lines_enum, options): pattern = re.compile(skip_regex) lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum) return lines_enum + + +def expand_env_variables(lines_enum): + """Replace all environment variables that can be retrieved via `os.getenv`. + + The only allowed format for environment variables defined in the + requirement file is `${MY_VARIABLE_1}` to ensure two things: + + 1. Strings that contain a `$` aren't accidentally (partially) expanded. + 2. Ensure consistency across platforms for requirement files. + + These points are the result of a discusssion on the `github pull + request #3514 `_. + + Valid characters in variable names follow the `POSIX standard + `_ and are limited + to uppercase letter, digits and the `_` (underscore). + """ + for line_number, line in lines_enum: + for env_var, var_name in ENV_VAR_RE.findall(line): + value = os.getenv(var_name) + if not value: + continue + + line = line.replace(env_var, value) + + yield line_number, line diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 3a4af1b92..93fec8e73 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -495,6 +495,58 @@ class TestParseRequirements(object): assert finder.index_urls == ['Good'] + def test_expand_existing_env_variables(self, tmpdir, finder): + template = ( + 'https://%s:x-oauth-basic@github.com/user/%s/archive/master.zip' + ) + + env_vars = ( + ('GITHUB_TOKEN', 'notarealtoken'), + ('DO_12_FACTOR', 'awwyeah'), + ) + + with open(tmpdir.join('req1.txt'), 'w') as fp: + fp.write(template % tuple(['${%s}' % k for k, _ in env_vars])) + + with patch('pip._internal.req.req_file.os.getenv') as getenv: + getenv.side_effect = lambda n: dict(env_vars)[n] + + reqs = list(parse_requirements( + tmpdir.join('req1.txt'), + finder=finder, + session=PipSession() + )) + + assert len(reqs) == 1, \ + 'parsing requirement file with env variable failed' + + expected_url = template % tuple([v for _, v in env_vars]) + assert reqs[0].link.url == expected_url, \ + 'variable expansion in req file failed' + + def test_expand_missing_env_variables(self, tmpdir, finder): + req_url = ( + 'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@' + '%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip' + ) + + with open(tmpdir.join('req1.txt'), 'w') as fp: + fp.write(req_url) + + with patch('pip._internal.req.req_file.os.getenv') as getenv: + getenv.return_value = '' + + reqs = list(parse_requirements( + tmpdir.join('req1.txt'), + finder=finder, + session=PipSession() + )) + + assert len(reqs) == 1, \ + 'parsing requirement file with env variable failed' + assert reqs[0].link.url == req_url, \ + 'ignoring invalid env variable in req file failed' + def test_join_lines(self, tmpdir, finder): with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \\\n--extra-index-url url2") From 163149fff390d17f3a5cea86869def66f7188706 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 1 Mar 2018 02:50:06 -0600 Subject: [PATCH 2/2] Install build requirements before calling setup.py (#4799) Setup build environment before first invocation of setup.py and improve test to actually make check that build requirements are installed upon invocation. --- news/4799.trivial | 0 src/pip/_internal/build_env.py | 22 +++++++- src/pip/_internal/operations/prepare.py | 67 +++++++++++++++++++---- src/pip/_internal/req/req_install.py | 3 + src/pip/_internal/resolve.py | 7 ++- src/pip/_internal/wheel.py | 26 +-------- tests/data/packages/pep518-3.0.tar.gz | Bin 832 -> 853 bytes tests/data/src/pep518-3.0/pyproject.toml | 2 +- tests/data/src/pep518-3.0/setup.py | 2 + 9 files changed, 90 insertions(+), 39 deletions(-) create mode 100644 news/4799.trivial diff --git a/news/4799.trivial b/news/4799.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 52a53392c..1989c8aad 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -8,7 +8,7 @@ from pip._internal.utils.temp_dir import TempDirectory class BuildEnvironment(object): - """Manages a temporary environment to install build deps + """Creates and manages an isolated environment to install build deps """ def __init__(self, no_clean): @@ -62,3 +62,23 @@ class BuildEnvironment(object): os.environ.pop('PYTHONPATH', None) else: os.environ['PYTHONPATH'] = self.save_pythonpath + + def cleanup(self): + self._temp_dir.cleanup() + + +class NoOpBuildEnvironment(BuildEnvironment): + """A no-op drop-in replacement for BuildEnvironment + """ + + def __init__(self, no_clean): + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def cleanup(self): + pass diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index e1d8a2baa..86e2e7b99 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -1,11 +1,15 @@ """Prepares a distribution for installation """ +import itertools import logging import os +import sys +from copy import copy from pip._vendor import pkg_resources, requests +from pip._internal.build_env import NoOpBuildEnvironment from pip._internal.compat import expanduser from pip._internal.download import ( is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, @@ -14,9 +18,14 @@ from pip._internal.exceptions import ( DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, PreviousBuildDirError, VcsHashUnsupported, ) +from pip._internal.index import FormatControl +from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, normalize_path +from pip._internal.utils.misc import ( + call_subprocess, display_path, normalize_path, +) +from pip._internal.utils.ui import open_spinner from pip._internal.vcs import vcs logger = logging.getLogger(__name__) @@ -38,6 +47,26 @@ def make_abstract_dist(req): return IsSDist(req) +def _install_build_reqs(finder, prefix, build_requirements): + # NOTE: What follows is not a very good thing. + # Eventually, this should move into the BuildEnvironment class and + # that should handle all the isolation and sub-process invocation. + finder = copy(finder) + finder.format_control = FormatControl(set(), set([":all:"])) + urls = [ + finder.find_requirement( + InstallRequirement.from_line(r), upgrade=False).url + for r in build_requirements + ] + args = [ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', prefix, + ] + list(urls) + + with open_spinner("Installing build dependencies") as spinner: + call_subprocess(args, show_stdout=False, spinner=spinner) + + class DistAbstraction(object): """Abstracts out the wheel vs non-wheel Resolver.resolve() logic. @@ -64,7 +93,7 @@ class DistAbstraction(object): """Return a setuptools Dist object.""" raise NotImplementedError(self.dist) - def prep_for_dist(self): + def prep_for_dist(self, finder): """Ensure that we can get a Dist for this requirement.""" raise NotImplementedError(self.dist) @@ -75,7 +104,7 @@ class IsWheel(DistAbstraction): return list(pkg_resources.find_distributions( self.req.source_dir))[0] - def prep_for_dist(self): + def prep_for_dist(self, finder): # FIXME:https://github.com/pypa/pip/issues/1112 pass @@ -91,9 +120,27 @@ class IsSDist(DistAbstraction): ) return dist - def prep_for_dist(self): - self.req.run_egg_info() - self.req.assert_source_matches_version() + def prep_for_dist(self, finder): + # Before calling "setup.py egg_info", we need to set-up the build + # environment. + + build_requirements, isolate = self.req.get_pep_518_info() + + if 'setuptools' not in build_requirements: + logger.warning( + "This version of pip does not implement PEP 516, so " + "it cannot build a wheel without setuptools. You may need to " + "upgrade to a newer version of pip.") + + if not isolate: + self.req.build_env = NoOpBuildEnvironment(no_clean=False) + + with self.req.build_env as prefix: + if isolate: + _install_build_reqs(finder, prefix, build_requirements) + + self.req.run_egg_info() + self.req.assert_source_matches_version() class Installed(DistAbstraction): @@ -101,7 +148,7 @@ class Installed(DistAbstraction): def dist(self, finder): return self.req.satisfied_by - def prep_for_dist(self): + def prep_for_dist(self, finder): pass @@ -259,14 +306,14 @@ class RequirementPreparer(object): (req, exc, req.link) ) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist() + abstract_dist.prep_for_dist(finder) if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: req.archive(self.download_dir) return abstract_dist - def prepare_editable_requirement(self, req, require_hashes): + def prepare_editable_requirement(self, req, require_hashes, finder): """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" @@ -284,7 +331,7 @@ class RequirementPreparer(object): req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist() + abstract_dist.prep_for_dist(finder) if self._download_should_save: req.archive(self.download_dir) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index d25f17c95..c81a6ff38 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -22,6 +22,7 @@ from pip._vendor.packaging.version import Version from pip._vendor.pkg_resources import RequirementParseError, parse_requirements from pip._internal import wheel +from pip._internal.build_env import BuildEnvironment from pip._internal.compat import native_str from pip._internal.download import ( is_archive_file, is_url, path_to_url, url_to_path, @@ -128,6 +129,7 @@ class InstallRequirement(object): self.prepared = False self.isolated = isolated + self.build_env = BuildEnvironment(no_clean=True) @classmethod def from_editable(cls, editable_req, comes_from=None, isolated=False, @@ -880,6 +882,7 @@ class InstallRequirement(object): rmtree(self.source_dir) self.source_dir = None self._temp_build_dir.cleanup() + self.build_env.cleanup() def install_editable(self, install_options, global_options=(), prefix=None): diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index 247c561c5..d7b5ab0ee 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -187,7 +187,9 @@ class Resolver(object): if req.editable: return self.preparer.prepare_editable_requirement( - req, self.require_hashes + req, + self.require_hashes, + self.finder, ) # satisfied_by is only evaluated by calling _check_skip_installed, @@ -246,11 +248,12 @@ class Resolver(object): return [] req_to_install.prepared = True - abstract_dist = self._get_abstract_dist_for(req_to_install) # register tmp src for cleanup in case something goes wrong requirement_set.reqs_to_cleanup.append(req_to_install) + abstract_dist = self._get_abstract_dist_for(req_to_install) + # Parse and return dependencies dist = abstract_dist.dist(self.finder) try: diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 03c1b22b9..17d529116 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -617,37 +617,13 @@ class WheelBuilder(object): self.global_options = global_options or [] self.no_clean = no_clean - def _install_build_reqs(self, reqs, prefix): - # Local import to avoid circular import (wheel <-> req_install) - from pip._internal.req.req_install import InstallRequirement - from pip._internal.index import FormatControl - # Ignore the --no-binary option when installing the build system, so - # we don't recurse trying to build a self-hosting build system. - finder = copy.copy(self.finder) - finder.format_control = FormatControl(set(), set([":all:"])) - urls = [finder.find_requirement(InstallRequirement.from_line(r), - upgrade=False).url - for r in reqs] - - args = [sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--prefix', prefix] + list(urls) - with open_spinner("Installing build dependencies") as spinner: - call_subprocess(args, show_stdout=False, spinner=spinner) - def _build_one(self, req, output_dir, python_tag=None): """Build one wheel. :return: The filename of the built wheel, or None if the build failed. """ - build_reqs, isolate = req.get_pep_518_info() - if 'setuptools' not in build_reqs: - logger.warning( - "This version of pip does not implement PEP 516, so " - "it cannot build a wheel without setuptools. You may need to " - "upgrade to a newer version of pip.") # Install build deps into temporary directory (PEP 518) - with BuildEnvironment(self.no_clean) as prefix: - self._install_build_reqs(build_reqs, prefix) + with req.build_env: return self._build_one_inside_env(req, output_dir, python_tag=python_tag, isolate=True) diff --git a/tests/data/packages/pep518-3.0.tar.gz b/tests/data/packages/pep518-3.0.tar.gz index cfd03320375ad458026502c95660c407ff594098..f912e6d85a2e689bb46f266d6571fbbf8714a36a 100644 GIT binary patch delta 850 zcmV-Y1FihP2Gs_CABzYGvbk(p0{>)bb97vAWpFhyI4v_SFfMdqascg`TTk0C6vz7< zso%jX4}e6{IM)fO4~W5r7zM4cJs=c?)-4{1lf_QAeET^GkWkbOSQ7+0zn8|BD2 z?v)T%_CMMg9_;UpPBqF_Z-2*emfPQO3;P?E?%Fn%_BRa2?I0cO-#C zos5MhqA*Z8!2dx_|K+7dW;2ztY1Ch-{-5Umr7Lj3|G%*PCnP3pLfAO-0?MwrCVq)# z{J)NC_Rsx)ziVO({C^vOrw9MPME@s;AJv25{&C~^pJ}=a`nTi-=)Vn)i0~)A@YSy* z;WT2~2y1#*8TujFM&*j=DtBXMJ~m3j&`+{$bT&L39)BBll}}Mf)YzYqTHj78uA<~Y zhe|@92DR?bbW9kp-}!;?aYADupQ!FeOb{m2(=1+pGNsmg68K^oC1I@#1pa*ekB-m2 z?Cy=89rhosf5X)m^lxJmgZ^95e^vA=3pF4oB8|22barzUXUeh=q5NU(D38f2A50Qa z>;+^_f?|}@Fb+uH^C+Xj^WaQz=a(rBCMwUkAmN3QkRK^c z2p^pH%FN*B6(PZ3U}$=83-yXZKtxf%^Hwp|y8w~B4gKHMdu_*m{NVeKAsqndzZLz< za?i`ly^F3Q{Fzs{12jNoOJ)W{HuSQ^=gdog$eiZ6K-mic_RZ)bb966oWpFhyI4v_SFfMdqascg`T~nJt6o$Fx zS8VNtIwQ%#^3ic77ml^9V~tLu?S z!XoF{bjfw;kQW=DS42;h-%~-;>k=eugFo@|ksy2jvyP`IUKFZ@bNQ-TBIy@L2eHrQ> zl9(DIn9_P`CzqE=x;Qpy9I&XK{m90YXLoLUOS5r*%7j$kW$Y$gQZ9{?Lj3-Udh$35 zkPCxDLjs809KUn{kw~+r?*d#9KzZ3n}hkkwG446@wi{5xLz3z!Z zUlt!H4*jv!Qsn$XN zT~H~13qmI7e|7o(zvut9O9<#6@_)$xyNv&h^shtxW3P_?mg&vwzqSMV?}W39oJFCL z6`7>*xt`K*IZJ6aI9sbSgYTD=Mr&JWtvmzBB#E;5sG2?psl6cm-=6onj{jK2{AT=j zh>Jo0o#^+}qr+&O4`&c^2T@+LTzC(?gaWYt>H|Kr{v z|8OwD@czfs*njm!_J`Gf@A`kwwBi4sZv20k$8iBJ7n3mq7!3fPBR>JK``J|hPyhh# CMYN0n diff --git a/tests/data/src/pep518-3.0/pyproject.toml b/tests/data/src/pep518-3.0/pyproject.toml index 5e3bb2332..b51a1cec7 100644 --- a/tests/data/src/pep518-3.0/pyproject.toml +++ b/tests/data/src/pep518-3.0/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires=["simple==3.0", "setuptools", "wheel"] +requires=["simplewheel==2.0", "setuptools", "wheel"] diff --git a/tests/data/src/pep518-3.0/setup.py b/tests/data/src/pep518-3.0/setup.py index f9d4c4891..84e7feb70 100644 --- a/tests/data/src/pep518-3.0/setup.py +++ b/tests/data/src/pep518-3.0/setup.py @@ -1,6 +1,8 @@ #!/usr/bin/env python from setuptools import find_packages, setup +import simple # ensure dependency is installed + setup(name='pep518', version='3.0', packages=find_packages()