Add expansion of environment variables in requirement files (#3728)

docs/reference/pip_install.rst

@@ -171,6 +171,28 @@ You can also refer to :ref:`constraints files <Constraints Files>`, like this::
 
     -c some_constraints.txt
 
+.. _`Using Environment Variables`:
+
+Using Environment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since version 10, pip supports the use of environment variables inside the
+requirements file. You can now store sensitive data (tokens, keys, etc.) in
+environment variables and only specify the variable name for your requirements,
+letting pip lookup the value at runtime. This approach aligns with the commonly
+used `12-factor configuration pattern <https://12factor.net/config>`_.
+
+You have to use the POSIX format for variable names including brackets around
+the uppercase name as shown in this example: ``${API_TOKEN}``. pip will attempt
+to find the corresponding environment variable defined on the host system at
+runtime.
+
+.. note::
+
+   There is no support for other variable expansion syntaxes such as
+   ``$VARIABLE`` and ``%VARIABLE%``.
+
+
 .. _`Example Requirements File`:
 
 Example Requirements File
@@ -432,6 +454,21 @@ Tags or revisions can be installed like so::
     [-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject
     [-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject
 
+Using Environment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since version 10, pip also makes it possible to use environment variables which
+makes it possible to reference private repositories without having to store
+access tokens in the requirements file. For example, a private git repository
+allowing Basic Auth for authentication can be refenced like this::
+
+    [-e] git+http://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
+    [-e] git+https://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
+
+.. note::
+
+   Only ``${VARIABLE}`` is supported, other formats like ``$VARIABLE`` or
+   ``%VARIABLE%`` won't work.
 
 Finding Packages
 ++++++++++++++++

news/3728.feature

@@ -0,0 +1,2 @@
+pip now supports environment variable expansion in requirement files using
+only ``${VARIABLE}`` syntax on all platforms.

src/pip/_internal/req/req_file.py

@@ -23,6 +23,12 @@ __all__ = ['parse_requirements']
 SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
 COMMENT_RE = re.compile(r'(^|\s)+#.*$')
 
+# Matches environment variable-style values in '${MY_VARIABLE_1}' with the
+# variable name consisting of only uppercase letters, digits or the '_'
+# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
+# 2013 Edition.
+ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
+
 SUPPORTED_OPTIONS = [
     cmdoptions.constraints,
     cmdoptions.editable,
@@ -94,6 +100,7 @@ def preprocess(content, options):
     lines_enum = join_lines(lines_enum)
     lines_enum = ignore_comments(lines_enum)
     lines_enum = skip_regex(lines_enum, options)
+    lines_enum = expand_env_variables(lines_enum)
     return lines_enum
 
 
@@ -302,3 +309,30 @@ def skip_regex(lines_enum, options):
         pattern = re.compile(skip_regex)
         lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum)
     return lines_enum
+
+
+def expand_env_variables(lines_enum):
+    """Replace all environment variables that can be retrieved via `os.getenv`.
+
+    The only allowed format for environment variables defined in the
+    requirement file is `${MY_VARIABLE_1}` to ensure two things:
+
+    1. Strings that contain a `$` aren't accidentally (partially) expanded.
+    2. Ensure consistency across platforms for requirement files.
+
+    These points are the result of a discusssion on the `github pull
+    request #3514 <https://github.com/pypa/pip/pull/3514>`_.
+
+    Valid characters in variable names follow the `POSIX standard
+    <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
+    to uppercase letter, digits and the `_` (underscore).
+    """
+    for line_number, line in lines_enum:
+        for env_var, var_name in ENV_VAR_RE.findall(line):
+            value = os.getenv(var_name)
+            if not value:
+                continue
+
+            line = line.replace(env_var, value)
+
+        yield line_number, line

tests/unit/test_req_file.py

@@ -495,6 +495,58 @@ class TestParseRequirements(object):
 
         assert finder.index_urls == ['Good']
 
+    def test_expand_existing_env_variables(self, tmpdir, finder):
+        template = (
+            'https://%s:x-oauth-basic@github.com/user/%s/archive/master.zip'
+        )
+
+        env_vars = (
+            ('GITHUB_TOKEN', 'notarealtoken'),
+            ('DO_12_FACTOR', 'awwyeah'),
+        )
+
+        with open(tmpdir.join('req1.txt'), 'w') as fp:
+            fp.write(template % tuple(['${%s}' % k for k, _ in env_vars]))
+
+        with patch('pip._internal.req.req_file.os.getenv') as getenv:
+            getenv.side_effect = lambda n: dict(env_vars)[n]
+
+            reqs = list(parse_requirements(
+                tmpdir.join('req1.txt'),
+                finder=finder,
+                session=PipSession()
+            ))
+
+            assert len(reqs) == 1, \
+                'parsing requirement file with env variable failed'
+
+            expected_url = template % tuple([v for _, v in env_vars])
+            assert reqs[0].link.url == expected_url, \
+                'variable expansion in req file failed'
+
+    def test_expand_missing_env_variables(self, tmpdir, finder):
+        req_url = (
+            'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@'
+            '%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip'
+        )
+
+        with open(tmpdir.join('req1.txt'), 'w') as fp:
+            fp.write(req_url)
+
+        with patch('pip._internal.req.req_file.os.getenv') as getenv:
+            getenv.return_value = ''
+
+            reqs = list(parse_requirements(
+                tmpdir.join('req1.txt'),
+                finder=finder,
+                session=PipSession()
+            ))
+
+            assert len(reqs) == 1, \
+                'parsing requirement file with env variable failed'
+            assert reqs[0].link.url == req_url, \
+                'ignoring invalid env variable in req file failed'
+
     def test_join_lines(self, tmpdir, finder):
         with open(tmpdir.join("req1.txt"), "w") as fp:
             fp.write("--extra-index-url url1 \\\n--extra-index-url url2")