diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index e54337f82..e9d0ec523 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -171,6 +171,28 @@ You can also refer to :ref:`constraints files `, like this:: -c some_constraints.txt +.. _`Using Environment Variables`: + +Using Environment Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since version 10, pip supports the use of environment variables inside the +requirements file. You can now store sensitive data (tokens, keys, etc.) in +environment variables and only specify the variable name for your requirements, +letting pip lookup the value at runtime. This approach aligns with the commonly +used `12-factor configuration pattern `_. + +You have to use the POSIX format for variable names including brackets around +the uppercase name as shown in this example: ``${API_TOKEN}``. pip will attempt +to find the corresponding environment variable defined on the host system at +runtime. + +.. note:: + + There is no support for other variable expansion syntaxes such as + ``$VARIABLE`` and ``%VARIABLE%``. + + .. _`Example Requirements File`: Example Requirements File @@ -432,6 +454,21 @@ Tags or revisions can be installed like so:: [-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject [-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject +Using Environment Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since version 10, pip also makes it possible to use environment variables which +makes it possible to reference private repositories without having to store +access tokens in the requirements file. For example, a private git repository +allowing Basic Auth for authentication can be refenced like this:: + + [-e] git+http://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject + [-e] git+https://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject + +.. note:: + + Only ``${VARIABLE}`` is supported, other formats like ``$VARIABLE`` or + ``%VARIABLE%`` won't work. Finding Packages ++++++++++++++++ diff --git a/news/3728.feature b/news/3728.feature new file mode 100644 index 000000000..3080f88f7 --- /dev/null +++ b/news/3728.feature @@ -0,0 +1,2 @@ +pip now supports environment variable expansion in requirement files using +only ``${VARIABLE}`` syntax on all platforms. diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index 08e1b119f..f86849796 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -23,6 +23,12 @@ __all__ = ['parse_requirements'] SCHEME_RE = re.compile(r'^(http|https|file):', re.I) COMMENT_RE = re.compile(r'(^|\s)+#.*$') +# Matches environment variable-style values in '${MY_VARIABLE_1}' with the +# variable name consisting of only uppercase letters, digits or the '_' +# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1, +# 2013 Edition. +ENV_VAR_RE = re.compile(r'(?P\$\{(?P[A-Z0-9_]+)\})') + SUPPORTED_OPTIONS = [ cmdoptions.constraints, cmdoptions.editable, @@ -94,6 +100,7 @@ def preprocess(content, options): lines_enum = join_lines(lines_enum) lines_enum = ignore_comments(lines_enum) lines_enum = skip_regex(lines_enum, options) + lines_enum = expand_env_variables(lines_enum) return lines_enum @@ -302,3 +309,30 @@ def skip_regex(lines_enum, options): pattern = re.compile(skip_regex) lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum) return lines_enum + + +def expand_env_variables(lines_enum): + """Replace all environment variables that can be retrieved via `os.getenv`. + + The only allowed format for environment variables defined in the + requirement file is `${MY_VARIABLE_1}` to ensure two things: + + 1. Strings that contain a `$` aren't accidentally (partially) expanded. + 2. Ensure consistency across platforms for requirement files. + + These points are the result of a discusssion on the `github pull + request #3514 `_. + + Valid characters in variable names follow the `POSIX standard + `_ and are limited + to uppercase letter, digits and the `_` (underscore). + """ + for line_number, line in lines_enum: + for env_var, var_name in ENV_VAR_RE.findall(line): + value = os.getenv(var_name) + if not value: + continue + + line = line.replace(env_var, value) + + yield line_number, line diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 3a4af1b92..93fec8e73 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -495,6 +495,58 @@ class TestParseRequirements(object): assert finder.index_urls == ['Good'] + def test_expand_existing_env_variables(self, tmpdir, finder): + template = ( + 'https://%s:x-oauth-basic@github.com/user/%s/archive/master.zip' + ) + + env_vars = ( + ('GITHUB_TOKEN', 'notarealtoken'), + ('DO_12_FACTOR', 'awwyeah'), + ) + + with open(tmpdir.join('req1.txt'), 'w') as fp: + fp.write(template % tuple(['${%s}' % k for k, _ in env_vars])) + + with patch('pip._internal.req.req_file.os.getenv') as getenv: + getenv.side_effect = lambda n: dict(env_vars)[n] + + reqs = list(parse_requirements( + tmpdir.join('req1.txt'), + finder=finder, + session=PipSession() + )) + + assert len(reqs) == 1, \ + 'parsing requirement file with env variable failed' + + expected_url = template % tuple([v for _, v in env_vars]) + assert reqs[0].link.url == expected_url, \ + 'variable expansion in req file failed' + + def test_expand_missing_env_variables(self, tmpdir, finder): + req_url = ( + 'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@' + '%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip' + ) + + with open(tmpdir.join('req1.txt'), 'w') as fp: + fp.write(req_url) + + with patch('pip._internal.req.req_file.os.getenv') as getenv: + getenv.return_value = '' + + reqs = list(parse_requirements( + tmpdir.join('req1.txt'), + finder=finder, + session=PipSession() + )) + + assert len(reqs) == 1, \ + 'parsing requirement file with env variable failed' + assert reqs[0].link.url == req_url, \ + 'ignoring invalid env variable in req file failed' + def test_join_lines(self, tmpdir, finder): with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \\\n--extra-index-url url2")