Merge branch 'master' into resolver/warn-after-resolution

docs/reference/pip_install.rst

@@ -171,6 +171,28 @@ You can also refer to :ref:`constraints files <Constraints Files>`, like this::
 
     -c some_constraints.txt
 
+.. _`Using Environment Variables`:
+
+Using Environment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since version 10, pip supports the use of environment variables inside the
+requirements file. You can now store sensitive data (tokens, keys, etc.) in
+environment variables and only specify the variable name for your requirements,
+letting pip lookup the value at runtime. This approach aligns with the commonly
+used `12-factor configuration pattern <https://12factor.net/config>`_.
+
+You have to use the POSIX format for variable names including brackets around
+the uppercase name as shown in this example: ``${API_TOKEN}``. pip will attempt
+to find the corresponding environment variable defined on the host system at
+runtime.
+
+.. note::
+
+   There is no support for other variable expansion syntaxes such as
+   ``$VARIABLE`` and ``%VARIABLE%``.
+
+
 .. _`Example Requirements File`:
 
 Example Requirements File
@@ -432,6 +454,21 @@ Tags or revisions can be installed like so::
     [-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject
     [-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject
 
+Using Environment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since version 10, pip also makes it possible to use environment variables which
+makes it possible to reference private repositories without having to store
+access tokens in the requirements file. For example, a private git repository
+allowing Basic Auth for authentication can be refenced like this::
+
+    [-e] git+http://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
+    [-e] git+https://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
+
+.. note::
+
+   Only ``${VARIABLE}`` is supported, other formats like ``$VARIABLE`` or
+   ``%VARIABLE%`` won't work.
 
 Finding Packages
 ++++++++++++++++

news/3728.feature

@@ -0,0 +1,2 @@
+pip now supports environment variable expansion in requirement files using
+only ``${VARIABLE}`` syntax on all platforms.

src/pip/_internal/build_env.py

@@ -8,7 +8,7 @@ from pip._internal.utils.temp_dir import TempDirectory
 
 
 class BuildEnvironment(object):
-    """Manages a temporary environment to install build deps
+    """Creates and manages an isolated environment to install build deps
     """
 
     def __init__(self, no_clean):
@@ -62,3 +62,23 @@ class BuildEnvironment(object):
             os.environ.pop('PYTHONPATH', None)
         else:
             os.environ['PYTHONPATH'] = self.save_pythonpath
+
+    def cleanup(self):
+        self._temp_dir.cleanup()
+
+
+class NoOpBuildEnvironment(BuildEnvironment):
+    """A no-op drop-in replacement for BuildEnvironment
+    """
+
+    def __init__(self, no_clean):
+        pass
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def cleanup(self):
+        pass

src/pip/_internal/operations/prepare.py

@@ -1,11 +1,15 @@
 """Prepares a distribution for installation
 """
 
+import itertools
 import logging
 import os
+import sys
+from copy import copy
 
 from pip._vendor import pkg_resources, requests
 
+from pip._internal.build_env import NoOpBuildEnvironment
 from pip._internal.compat import expanduser
 from pip._internal.download import (
     is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path,
@@ -14,9 +18,14 @@ from pip._internal.exceptions import (
     DirectoryUrlHashUnsupported, HashUnpinned, InstallationError,
     PreviousBuildDirError, VcsHashUnsupported,
 )
+from pip._internal.index import FormatControl
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.hashes import MissingHashes
 from pip._internal.utils.logging import indent_log
-from pip._internal.utils.misc import display_path, normalize_path
+from pip._internal.utils.misc import (
+    call_subprocess, display_path, normalize_path,
+)
+from pip._internal.utils.ui import open_spinner
 from pip._internal.vcs import vcs
 
 logger = logging.getLogger(__name__)
@@ -38,6 +47,26 @@ def make_abstract_dist(req):
         return IsSDist(req)
 
 
+def _install_build_reqs(finder, prefix, build_requirements):
+    # NOTE: What follows is not a very good thing.
+    #       Eventually, this should move into the BuildEnvironment class and
+    #       that should handle all the isolation and sub-process invocation.
+    finder = copy(finder)
+    finder.format_control = FormatControl(set(), set([":all:"]))
+    urls = [
+        finder.find_requirement(
+            InstallRequirement.from_line(r), upgrade=False).url
+        for r in build_requirements
+    ]
+    args = [
+        sys.executable, '-m', 'pip', 'install', '--ignore-installed',
+        '--prefix', prefix,
+    ] + list(urls)
+
+    with open_spinner("Installing build dependencies") as spinner:
+        call_subprocess(args, show_stdout=False, spinner=spinner)
+
+
 class DistAbstraction(object):
     """Abstracts out the wheel vs non-wheel Resolver.resolve() logic.
 
@@ -64,7 +93,7 @@ class DistAbstraction(object):
         """Return a setuptools Dist object."""
         raise NotImplementedError(self.dist)
 
-    def prep_for_dist(self):
+    def prep_for_dist(self, finder):
         """Ensure that we can get a Dist for this requirement."""
         raise NotImplementedError(self.dist)
 
@@ -75,7 +104,7 @@ class IsWheel(DistAbstraction):
         return list(pkg_resources.find_distributions(
             self.req.source_dir))[0]
 
-    def prep_for_dist(self):
+    def prep_for_dist(self, finder):
         # FIXME:https://github.com/pypa/pip/issues/1112
         pass
 
@@ -91,9 +120,27 @@ class IsSDist(DistAbstraction):
             )
         return dist
 
-    def prep_for_dist(self):
-        self.req.run_egg_info()
-        self.req.assert_source_matches_version()
+    def prep_for_dist(self, finder):
+        # Before calling "setup.py egg_info", we need to set-up the build
+        # environment.
+
+        build_requirements, isolate = self.req.get_pep_518_info()
+
+        if 'setuptools' not in build_requirements:
+            logger.warning(
+                "This version of pip does not implement PEP 516, so "
+                "it cannot build a wheel without setuptools. You may need to "
+                "upgrade to a newer version of pip.")
+
+        if not isolate:
+            self.req.build_env = NoOpBuildEnvironment(no_clean=False)
+
+        with self.req.build_env as prefix:
+            if isolate:
+                _install_build_reqs(finder, prefix, build_requirements)
+
+            self.req.run_egg_info()
+            self.req.assert_source_matches_version()
 
 
 class Installed(DistAbstraction):
@@ -101,7 +148,7 @@ class Installed(DistAbstraction):
     def dist(self, finder):
         return self.req.satisfied_by
 
-    def prep_for_dist(self):
+    def prep_for_dist(self, finder):
         pass
 
 
@@ -259,14 +306,15 @@ class RequirementPreparer(object):
                     (req, exc, req.link)
                 )
             abstract_dist = make_abstract_dist(req)
-            abstract_dist.prep_for_dist()
+            abstract_dist.prep_for_dist(finder)
             if self._download_should_save:
                 # Make a .zip of the source_dir we already created.
                 if req.link.scheme in vcs.all_schemes:
                     req.archive(self.download_dir)
         return abstract_dist
 
-    def prepare_editable_requirement(self, req, require_hashes, use_user_site):
+    def prepare_editable_requirement(self, req, require_hashes, use_user_site,
+                                     finder):
         """Prepare an editable requirement
         """
         assert req.editable, "cannot prepare a non-editable req as editable"
@@ -284,7 +332,7 @@ class RequirementPreparer(object):
             req.update_editable(not self._download_should_save)
 
             abstract_dist = make_abstract_dist(req)
-            abstract_dist.prep_for_dist()
+            abstract_dist.prep_for_dist(finder)
 
             if self._download_should_save:
                 req.archive(self.download_dir)

src/pip/_internal/req/req_file.py

@@ -23,6 +23,12 @@ __all__ = ['parse_requirements']
 SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
 COMMENT_RE = re.compile(r'(^|\s)+#.*$')
 
+# Matches environment variable-style values in '${MY_VARIABLE_1}' with the
+# variable name consisting of only uppercase letters, digits or the '_'
+# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
+# 2013 Edition.
+ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
+
 SUPPORTED_OPTIONS = [
     cmdoptions.constraints,
     cmdoptions.editable,
@@ -94,6 +100,7 @@ def preprocess(content, options):
     lines_enum = join_lines(lines_enum)
     lines_enum = ignore_comments(lines_enum)
     lines_enum = skip_regex(lines_enum, options)
+    lines_enum = expand_env_variables(lines_enum)
     return lines_enum
 
 
@@ -302,3 +309,30 @@ def skip_regex(lines_enum, options):
         pattern = re.compile(skip_regex)
         lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum)
     return lines_enum
+
+
+def expand_env_variables(lines_enum):
+    """Replace all environment variables that can be retrieved via `os.getenv`.
+
+    The only allowed format for environment variables defined in the
+    requirement file is `${MY_VARIABLE_1}` to ensure two things:
+
+    1. Strings that contain a `$` aren't accidentally (partially) expanded.
+    2. Ensure consistency across platforms for requirement files.
+
+    These points are the result of a discusssion on the `github pull
+    request #3514 <https://github.com/pypa/pip/pull/3514>`_.
+
+    Valid characters in variable names follow the `POSIX standard
+    <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
+    to uppercase letter, digits and the `_` (underscore).
+    """
+    for line_number, line in lines_enum:
+        for env_var, var_name in ENV_VAR_RE.findall(line):
+            value = os.getenv(var_name)
+            if not value:
+                continue
+
+            line = line.replace(env_var, value)
+
+        yield line_number, line

src/pip/_internal/req/req_install.py

@@ -22,6 +22,7 @@ from pip._vendor.packaging.version import Version
 from pip._vendor.pkg_resources import RequirementParseError, parse_requirements
 
 from pip._internal import wheel
+from pip._internal.build_env import BuildEnvironment
 from pip._internal.compat import native_str
 from pip._internal.download import (
     is_archive_file, is_url, path_to_url, url_to_path,
@@ -126,6 +127,7 @@ class InstallRequirement(object):
         self.is_direct = False
 
         self.isolated = isolated
+        self.build_env = BuildEnvironment(no_clean=True)
 
     @classmethod
     def from_editable(cls, editable_req, comes_from=None, isolated=False,
@@ -882,6 +884,7 @@ class InstallRequirement(object):
             rmtree(self.source_dir)
         self.source_dir = None
         self._temp_build_dir.cleanup()
+        self.build_env.cleanup()
 
     def install_editable(self, install_options,
                          global_options=(), prefix=None):

src/pip/_internal/resolve.py

@@ -191,7 +191,7 @@ class Resolver(object):
 
         if req.editable:
             return self.preparer.prepare_editable_requirement(
-                req, self.require_hashes, self.use_user_site,
+                req, self.require_hashes, self.use_user_site, self.finder,
             )
 
         # satisfied_by is only evaluated by calling _check_skip_installed,
@@ -250,11 +250,12 @@ class Resolver(object):
             return []
 
         req_to_install.prepared = True
-        abstract_dist = self._get_abstract_dist_for(req_to_install)
 
         # register tmp src for cleanup in case something goes wrong
         requirement_set.reqs_to_cleanup.append(req_to_install)
 
+        abstract_dist = self._get_abstract_dist_for(req_to_install)
+
         # Parse and return dependencies
         dist = abstract_dist.dist(self.finder)
         try:

src/pip/_internal/wheel.py

@@ -617,37 +617,13 @@ class WheelBuilder(object):
         self.global_options = global_options or []
         self.no_clean = no_clean
 
-    def _install_build_reqs(self, reqs, prefix):
-        # Local import to avoid circular import (wheel <-> req_install)
-        from pip._internal.req.req_install import InstallRequirement
-        from pip._internal.index import FormatControl
-        # Ignore the --no-binary option when installing the build system, so
-        # we don't recurse trying to build a self-hosting build system.
-        finder = copy.copy(self.finder)
-        finder.format_control = FormatControl(set(), set([":all:"]))
-        urls = [finder.find_requirement(InstallRequirement.from_line(r),
-                                        upgrade=False).url
-                for r in reqs]
-
-        args = [sys.executable, '-m', 'pip', 'install', '--ignore-installed',
-                '--prefix', prefix] + list(urls)
-        with open_spinner("Installing build dependencies") as spinner:
-            call_subprocess(args, show_stdout=False, spinner=spinner)
-
     def _build_one(self, req, output_dir, python_tag=None):
         """Build one wheel.
 
         :return: The filename of the built wheel, or None if the build failed.
         """
-        build_reqs, isolate = req.get_pep_518_info()
-        if 'setuptools' not in build_reqs:
-            logger.warning(
-                "This version of pip does not implement PEP 516, so "
-                "it cannot build a wheel without setuptools. You may need to "
-                "upgrade to a newer version of pip.")
         # Install build deps into temporary directory (PEP 518)
-        with BuildEnvironment(self.no_clean) as prefix:
-            self._install_build_reqs(build_reqs, prefix)
+        with req.build_env:
             return self._build_one_inside_env(req, output_dir,
                                               python_tag=python_tag,
                                               isolate=True)

tests/data/src/pep518-3.0/pyproject.toml

@@ -1,2 +1,2 @@
 [build-system]
-requires=["simple==3.0", "setuptools", "wheel"]
+requires=["simplewheel==2.0", "setuptools", "wheel"]

tests/data/src/pep518-3.0/setup.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 from setuptools import find_packages, setup
 
+import simple  # ensure dependency is installed
+
 setup(name='pep518',
       version='3.0',
       packages=find_packages()

tests/unit/test_req_file.py

@@ -495,6 +495,58 @@ class TestParseRequirements(object):
 
         assert finder.index_urls == ['Good']
 
+    def test_expand_existing_env_variables(self, tmpdir, finder):
+        template = (
+            'https://%s:x-oauth-basic@github.com/user/%s/archive/master.zip'
+        )
+
+        env_vars = (
+            ('GITHUB_TOKEN', 'notarealtoken'),
+            ('DO_12_FACTOR', 'awwyeah'),
+        )
+
+        with open(tmpdir.join('req1.txt'), 'w') as fp:
+            fp.write(template % tuple(['${%s}' % k for k, _ in env_vars]))
+
+        with patch('pip._internal.req.req_file.os.getenv') as getenv:
+            getenv.side_effect = lambda n: dict(env_vars)[n]
+
+            reqs = list(parse_requirements(
+                tmpdir.join('req1.txt'),
+                finder=finder,
+                session=PipSession()
+            ))
+
+            assert len(reqs) == 1, \
+                'parsing requirement file with env variable failed'
+
+            expected_url = template % tuple([v for _, v in env_vars])
+            assert reqs[0].link.url == expected_url, \
+                'variable expansion in req file failed'
+
+    def test_expand_missing_env_variables(self, tmpdir, finder):
+        req_url = (
+            'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@'
+            '%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip'
+        )
+
+        with open(tmpdir.join('req1.txt'), 'w') as fp:
+            fp.write(req_url)
+
+        with patch('pip._internal.req.req_file.os.getenv') as getenv:
+            getenv.return_value = ''
+
+            reqs = list(parse_requirements(
+                tmpdir.join('req1.txt'),
+                finder=finder,
+                session=PipSession()
+            ))
+
+            assert len(reqs) == 1, \
+                'parsing requirement file with env variable failed'
+            assert reqs[0].link.url == req_url, \
+                'ignoring invalid env variable in req file failed'
+
     def test_join_lines(self, tmpdir, finder):
         with open(tmpdir.join("req1.txt"), "w") as fp:
             fp.write("--extra-index-url url1 \\\n--extra-index-url url2")