1
1
Fork 0
mirror of https://github.com/pypa/pip synced 2023-12-13 21:30:23 +01:00

Add expansion of environment variables in requirement files (#3728)

This commit is contained in:
BrownTruck 2018-02-05 06:20:50 -05:00 committed by Pradyun Gedam
parent e81b602f90
commit 72f219c410
4 changed files with 125 additions and 0 deletions

View file

@ -171,6 +171,28 @@ You can also refer to :ref:`constraints files <Constraints Files>`, like this::
-c some_constraints.txt -c some_constraints.txt
.. _`Using Environment Variables`:
Using Environment Variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Since version 10, pip supports the use of environment variables inside the
requirements file. You can now store sensitive data (tokens, keys, etc.) in
environment variables and only specify the variable name for your requirements,
letting pip lookup the value at runtime. This approach aligns with the commonly
used `12-factor configuration pattern <https://12factor.net/config>`_.
You have to use the POSIX format for variable names including brackets around
the uppercase name as shown in this example: ``${API_TOKEN}``. pip will attempt
to find the corresponding environment variable defined on the host system at
runtime.
.. note::
There is no support for other variable expansion syntaxes such as
``$VARIABLE`` and ``%VARIABLE%``.
.. _`Example Requirements File`: .. _`Example Requirements File`:
Example Requirements File Example Requirements File
@ -432,6 +454,21 @@ Tags or revisions can be installed like so::
[-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject [-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject
[-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject [-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject
Using Environment Variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Since version 10, pip also makes it possible to use environment variables which
makes it possible to reference private repositories without having to store
access tokens in the requirements file. For example, a private git repository
allowing Basic Auth for authentication can be refenced like this::
[-e] git+http://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
[-e] git+https://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
.. note::
Only ``${VARIABLE}`` is supported, other formats like ``$VARIABLE`` or
``%VARIABLE%`` won't work.
Finding Packages Finding Packages
++++++++++++++++ ++++++++++++++++

2
news/3728.feature Normal file
View file

@ -0,0 +1,2 @@
pip now supports environment variable expansion in requirement files using
only ``${VARIABLE}`` syntax on all platforms.

View file

@ -23,6 +23,12 @@ __all__ = ['parse_requirements']
SCHEME_RE = re.compile(r'^(http|https|file):', re.I) SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
COMMENT_RE = re.compile(r'(^|\s)+#.*$') COMMENT_RE = re.compile(r'(^|\s)+#.*$')
# Matches environment variable-style values in '${MY_VARIABLE_1}' with the
# variable name consisting of only uppercase letters, digits or the '_'
# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
# 2013 Edition.
ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
SUPPORTED_OPTIONS = [ SUPPORTED_OPTIONS = [
cmdoptions.constraints, cmdoptions.constraints,
cmdoptions.editable, cmdoptions.editable,
@ -94,6 +100,7 @@ def preprocess(content, options):
lines_enum = join_lines(lines_enum) lines_enum = join_lines(lines_enum)
lines_enum = ignore_comments(lines_enum) lines_enum = ignore_comments(lines_enum)
lines_enum = skip_regex(lines_enum, options) lines_enum = skip_regex(lines_enum, options)
lines_enum = expand_env_variables(lines_enum)
return lines_enum return lines_enum
@ -302,3 +309,30 @@ def skip_regex(lines_enum, options):
pattern = re.compile(skip_regex) pattern = re.compile(skip_regex)
lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum) lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum)
return lines_enum return lines_enum
def expand_env_variables(lines_enum):
"""Replace all environment variables that can be retrieved via `os.getenv`.
The only allowed format for environment variables defined in the
requirement file is `${MY_VARIABLE_1}` to ensure two things:
1. Strings that contain a `$` aren't accidentally (partially) expanded.
2. Ensure consistency across platforms for requirement files.
These points are the result of a discusssion on the `github pull
request #3514 <https://github.com/pypa/pip/pull/3514>`_.
Valid characters in variable names follow the `POSIX standard
<http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
to uppercase letter, digits and the `_` (underscore).
"""
for line_number, line in lines_enum:
for env_var, var_name in ENV_VAR_RE.findall(line):
value = os.getenv(var_name)
if not value:
continue
line = line.replace(env_var, value)
yield line_number, line

View file

@ -495,6 +495,58 @@ class TestParseRequirements(object):
assert finder.index_urls == ['Good'] assert finder.index_urls == ['Good']
def test_expand_existing_env_variables(self, tmpdir, finder):
template = (
'https://%s:x-oauth-basic@github.com/user/%s/archive/master.zip'
)
env_vars = (
('GITHUB_TOKEN', 'notarealtoken'),
('DO_12_FACTOR', 'awwyeah'),
)
with open(tmpdir.join('req1.txt'), 'w') as fp:
fp.write(template % tuple(['${%s}' % k for k, _ in env_vars]))
with patch('pip._internal.req.req_file.os.getenv') as getenv:
getenv.side_effect = lambda n: dict(env_vars)[n]
reqs = list(parse_requirements(
tmpdir.join('req1.txt'),
finder=finder,
session=PipSession()
))
assert len(reqs) == 1, \
'parsing requirement file with env variable failed'
expected_url = template % tuple([v for _, v in env_vars])
assert reqs[0].link.url == expected_url, \
'variable expansion in req file failed'
def test_expand_missing_env_variables(self, tmpdir, finder):
req_url = (
'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@'
'%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip'
)
with open(tmpdir.join('req1.txt'), 'w') as fp:
fp.write(req_url)
with patch('pip._internal.req.req_file.os.getenv') as getenv:
getenv.return_value = ''
reqs = list(parse_requirements(
tmpdir.join('req1.txt'),
finder=finder,
session=PipSession()
))
assert len(reqs) == 1, \
'parsing requirement file with env variable failed'
assert reqs[0].link.url == req_url, \
'ignoring invalid env variable in req file failed'
def test_join_lines(self, tmpdir, finder): def test_join_lines(self, tmpdir, finder):
with open(tmpdir.join("req1.txt"), "w") as fp: with open(tmpdir.join("req1.txt"), "w") as fp:
fp.write("--extra-index-url url1 \\\n--extra-index-url url2") fp.write("--extra-index-url url1 \\\n--extra-index-url url2")