From 0fb0e3b54743a0810ba72929e49db822b8eb56cc Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Fri, 23 Jul 2021 12:03:53 +0300 Subject: [PATCH] Upgrade vendored requests to 2.26.0 (#10174) Co-authored-by: Tzu-ping Chung --- news/requests.vendor.rst | 1 + src/pip/_vendor/requests/__init__.py | 34 ++++++++---- src/pip/_vendor/requests/__version__.py | 4 +- src/pip/_vendor/requests/api.py | 2 - src/pip/_vendor/requests/exceptions.py | 4 ++ src/pip/_vendor/requests/help.py | 17 +++++- src/pip/_vendor/requests/models.py | 22 +++++--- src/pip/_vendor/requests/sessions.py | 2 +- src/pip/_vendor/requests/utils.py | 29 ++++++++-- src/pip/_vendor/vendor.txt | 2 +- tools/vendoring/patches/requests.patch | 71 ++++++++++++++++++++++--- 11 files changed, 152 insertions(+), 36 deletions(-) create mode 100644 news/requests.vendor.rst diff --git a/news/requests.vendor.rst b/news/requests.vendor.rst new file mode 100644 index 000000000..3356747a6 --- /dev/null +++ b/news/requests.vendor.rst @@ -0,0 +1 @@ +Upgrade requests to 2.26.0. diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py index 18046c451..4f80e28fc 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py @@ -41,12 +41,17 @@ is at . """ from pip._vendor import urllib3 -from pip._vendor import chardet import warnings from .exceptions import RequestsDependencyWarning +charset_normalizer_version = None -def check_compatibility(urllib3_version, chardet_version): +try: + from pip._vendor.chardet import __version__ as chardet_version +except ImportError: + chardet_version = None + +def check_compatibility(urllib3_version, chardet_version, charset_normalizer_version): urllib3_version = urllib3_version.split('.') assert urllib3_version != ['dev'] # Verify urllib3 isn't installed from git. @@ -62,12 +67,19 @@ def check_compatibility(urllib3_version, chardet_version): assert minor >= 21 assert minor <= 26 - # Check chardet for compatibility. - major, minor, patch = chardet_version.split('.')[:3] - major, minor, patch = int(major), int(minor), int(patch) - # chardet >= 3.0.2, < 5.0.0 - assert (3, 0, 2) <= (major, minor, patch) < (5, 0, 0) - + # Check charset_normalizer for compatibility. + if chardet_version: + major, minor, patch = chardet_version.split('.')[:3] + major, minor, patch = int(major), int(minor), int(patch) + # chardet_version >= 3.0.2, < 5.0.0 + assert (3, 0, 2) <= (major, minor, patch) < (5, 0, 0) + elif charset_normalizer_version: + major, minor, patch = charset_normalizer_version.split('.')[:3] + major, minor, patch = int(major), int(minor), int(patch) + # charset_normalizer >= 2.0.0 < 3.0.0 + assert (2, 0, 0) <= (major, minor, patch) < (3, 0, 0) + else: + raise Exception("You need either charset_normalizer or chardet installed") def _check_cryptography(cryptography_version): # cryptography < 1.3.4 @@ -82,10 +94,10 @@ def _check_cryptography(cryptography_version): # Check imported dependencies for compatibility. try: - check_compatibility(urllib3.__version__, chardet.__version__) + check_compatibility(urllib3.__version__, chardet_version, charset_normalizer_version) except (AssertionError, ValueError): - warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " - "version!".format(urllib3.__version__, chardet.__version__), + warnings.warn("urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " + "version!".format(urllib3.__version__, chardet_version, charset_normalizer_version), RequestsDependencyWarning) # Attempt to enable urllib3's fallback for SNI support diff --git a/src/pip/_vendor/requests/__version__.py b/src/pip/_vendor/requests/__version__.py index 1267488d2..0d7cde1df 100644 --- a/src/pip/_vendor/requests/__version__.py +++ b/src/pip/_vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'https://requests.readthedocs.io' -__version__ = '2.25.1' -__build__ = 0x022501 +__version__ = '2.26.0' +__build__ = 0x022600 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/src/pip/_vendor/requests/api.py b/src/pip/_vendor/requests/api.py index e978e2031..4cba90eef 100644 --- a/src/pip/_vendor/requests/api.py +++ b/src/pip/_vendor/requests/api.py @@ -72,7 +72,6 @@ def get(url, params=None, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return request('get', url, params=params, **kwargs) @@ -85,7 +84,6 @@ def options(url, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return request('options', url, **kwargs) diff --git a/src/pip/_vendor/requests/exceptions.py b/src/pip/_vendor/requests/exceptions.py index 9ef9e6e97..9f0ad778b 100644 --- a/src/pip/_vendor/requests/exceptions.py +++ b/src/pip/_vendor/requests/exceptions.py @@ -25,6 +25,10 @@ class RequestException(IOError): super(RequestException, self).__init__(*args, **kwargs) +class InvalidJSONError(RequestException): + """A JSON error occurred.""" + + class HTTPError(RequestException): """An HTTP error occurred.""" diff --git a/src/pip/_vendor/requests/help.py b/src/pip/_vendor/requests/help.py index 3c3072ba1..745f0d7b3 100644 --- a/src/pip/_vendor/requests/help.py +++ b/src/pip/_vendor/requests/help.py @@ -8,10 +8,16 @@ import ssl from pip._vendor import idna from pip._vendor import urllib3 -from pip._vendor import chardet from . import __version__ as requests_version +charset_normalizer = None + +try: + from pip._vendor import chardet +except ImportError: + chardet = None + try: from pip._vendor.urllib3.contrib import pyopenssl except ImportError: @@ -71,7 +77,12 @@ def info(): implementation_info = _implementation() urllib3_info = {'version': urllib3.__version__} - chardet_info = {'version': chardet.__version__} + charset_normalizer_info = {'version': None} + chardet_info = {'version': None} + if charset_normalizer: + charset_normalizer_info = {'version': charset_normalizer.__version__} + if chardet: + chardet_info = {'version': chardet.__version__} pyopenssl_info = { 'version': None, @@ -99,9 +110,11 @@ def info(): 'implementation': implementation_info, 'system_ssl': system_ssl_info, 'using_pyopenssl': pyopenssl is not None, + 'using_charset_normalizer': chardet is None, 'pyOpenSSL': pyopenssl_info, 'urllib3': urllib3_info, 'chardet': chardet_info, + 'charset_normalizer': charset_normalizer_info, 'cryptography': cryptography_info, 'idna': idna_info, 'requests': { diff --git a/src/pip/_vendor/requests/models.py b/src/pip/_vendor/requests/models.py index b0ce2950f..c10c6011b 100644 --- a/src/pip/_vendor/requests/models.py +++ b/src/pip/_vendor/requests/models.py @@ -29,7 +29,7 @@ from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .exceptions import ( HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError) + ContentDecodingError, ConnectionError, StreamConsumedError, InvalidJSONError) from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( guess_filename, get_auth_from_url, requote_uri, @@ -466,7 +466,12 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # urllib3 requires a bytes-like body. Python 2's json.dumps # provides this natively, but Python 3 gives a Unicode string. content_type = 'application/json' - body = complexjson.dumps(json) + + try: + body = complexjson.dumps(json, allow_nan=False) + except ValueError as ve: + raise InvalidJSONError(ve, request=self) + if not isinstance(body, bytes): body = body.encode('utf-8') @@ -726,7 +731,7 @@ class Response(object): @property def apparent_encoding(self): - """The apparent encoding, provided by the chardet library.""" + """The apparent encoding, provided by the charset_normalizer or chardet libraries.""" return chardet.detect(self.content)['encoding'] def iter_content(self, chunk_size=1, decode_unicode=False): @@ -840,7 +845,7 @@ class Response(object): """Content of the response, in unicode. If Response.encoding is None, encoding will be guessed using - ``chardet``. + ``charset_normalizer`` or ``chardet``. The encoding of the response content is determined based solely on HTTP headers, following RFC 2616 to the letter. If you can take advantage of @@ -877,13 +882,18 @@ class Response(object): r"""Returns the json-encoded content of a response, if any. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. - :raises ValueError: If the response body does not contain valid json. + :raises simplejson.JSONDecodeError: If the response body does not + contain valid json and simplejson is installed. + :raises json.JSONDecodeError: If the response body does not contain + valid json and simplejson is not installed on Python 3. + :raises ValueError: If the response body does not contain valid + json and simplejson is not installed on Python 2. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or - # decoding fails, fall back to `self.text` (using chardet to make + # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: diff --git a/src/pip/_vendor/requests/sessions.py b/src/pip/_vendor/requests/sessions.py index 45ab8a5d3..ae4bcc8e7 100644 --- a/src/pip/_vendor/requests/sessions.py +++ b/src/pip/_vendor/requests/sessions.py @@ -633,7 +633,7 @@ class Session(SessionRedirectMixin): kwargs.setdefault('stream', self.stream) kwargs.setdefault('verify', self.verify) kwargs.setdefault('cert', self.cert) - kwargs.setdefault('proxies', self.proxies) + kwargs.setdefault('proxies', self.rebuild_proxies(request, self.proxies)) # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. diff --git a/src/pip/_vendor/requests/utils.py b/src/pip/_vendor/requests/utils.py index db67938e6..fcb996690 100644 --- a/src/pip/_vendor/requests/utils.py +++ b/src/pip/_vendor/requests/utils.py @@ -20,6 +20,7 @@ import tempfile import warnings import zipfile from collections import OrderedDict +from pip._vendor.urllib3.util import make_headers from .__version__ import __version__ from . import certs @@ -41,6 +42,11 @@ DEFAULT_CA_BUNDLE_PATH = certs.where() DEFAULT_PORTS = {'http': 80, 'https': 443} +# Ensure that ', ' is used to preserve previous delimiter behavior. +DEFAULT_ACCEPT_ENCODING = ", ".join( + re.split(r",\s*", make_headers(accept_encoding=True)["accept-encoding"]) +) + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -256,13 +262,28 @@ def extract_zipped_paths(path): # we have a valid zip archive and a valid member of that archive tmp = tempfile.gettempdir() - extracted_path = os.path.join(tmp, *member.split('/')) + extracted_path = os.path.join(tmp, member.split('/')[-1]) if not os.path.exists(extracted_path): - extracted_path = zip_file.extract(member, path=tmp) - + # use read + write to avoid the creating nested folders, we only want the file, avoids mkdir racing condition + with atomic_open(extracted_path) as file_handler: + file_handler.write(zip_file.read(member)) return extracted_path +@contextlib.contextmanager +def atomic_open(filename): + """Write a file to the disk in an atomic fashion""" + replacer = os.rename if sys.version_info[0] == 2 else os.replace + tmp_descriptor, tmp_name = tempfile.mkstemp(dir=os.path.dirname(filename)) + try: + with os.fdopen(tmp_descriptor, 'wb') as tmp_handler: + yield tmp_handler + replacer(tmp_name, filename) + except BaseException: + os.remove(tmp_name) + raise + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an @@ -820,7 +841,7 @@ def default_headers(): """ return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate')), + 'Accept-Encoding': DEFAULT_ACCEPT_ENCODING, 'Accept': '*/*', 'Connection': 'keep-alive', }) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 5bcaef348..73e493434 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -9,7 +9,7 @@ packaging==21.0 pep517==0.11.0 progress==1.5 pyparsing==2.4.7 -requests==2.25.1 +requests==2.26.0 certifi==2021.05.30 chardet==4.0.0 idna==3.2 diff --git a/tools/vendoring/patches/requests.patch b/tools/vendoring/patches/requests.patch index 08795ad3a..d1bb4309b 100644 --- a/tools/vendoring/patches/requests.patch +++ b/tools/vendoring/patches/requests.patch @@ -1,12 +1,24 @@ diff --git a/src/pip/_vendor/requests/packages.py b/src/pip/_vendor/requests/packages.py -index 6336a07d..9582fa73 100644 +index 0f8ae0d38..9582fa730 100644 --- a/src/pip/_vendor/requests/packages.py +++ b/src/pip/_vendor/requests/packages.py -@@ -4,11 +4,13 @@ import sys +@@ -1,26 +1,16 @@ + import sys + +-try: +- from pip._vendor import chardet +-except ImportError: +- import charset_normalizer as chardet +- import warnings +- +- warnings.filterwarnings('ignore', 'Trying to detect', module='charset_normalizer') +- + # This code exists for backwards compatibility reasons. # I don't like it either. Just look the other way. :) - for package in ('urllib3', 'idna', 'chardet'): +-for package in ('urllib3', 'idna'): - locals()[package] = __import__(package) ++for package in ('urllib3', 'idna', 'chardet'): + vendored_package = "pip._vendor." + package + locals()[package] = __import__(vendored_package) # This traversal is apparently necessary such that the identities are @@ -18,13 +30,29 @@ index 6336a07d..9582fa73 100644 + unprefixed_mod = mod[len("pip._vendor."):] + sys.modules['pip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] +-target = chardet.__name__ +-for mod in list(sys.modules): +- if mod == target or mod.startswith(target + '.'): +- sys.modules['requests.packages.' + target.replace(target, 'chardet')] = sys.modules[mod] # Kinda cool, though, right? diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py -index dc83261a8..517458b5a 100644 +index 973497f5e..4f80e28fc 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py -@@ -94,6 +94,11 @@ except (AssertionError, ValueError): +@@ -44,10 +44,7 @@ from pip._vendor import urllib3 + import warnings + from .exceptions import RequestsDependencyWarning + +-try: +- from charset_normalizer import __version__ as charset_normalizer_version +-except ImportError: +- charset_normalizer_version = None ++charset_normalizer_version = None + + try: + from pip._vendor.chardet import __version__ as chardet_version +@@ -107,6 +104,11 @@ except (AssertionError, ValueError): # if the standard library doesn't support SNI or the # 'ssl' library isn't available. try: @@ -38,10 +66,22 @@ index dc83261a8..517458b5a 100644 except ImportError: diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py -index eb6530d..353ec29 100644 +index 409b7b028..9e2937167 100644 --- a/src/pip/_vendor/requests/compat.py +++ b/src/pip/_vendor/requests/compat.py -@@ -25,10 +25,14 @@ +@@ -8,10 +8,7 @@ This module handles import compatibility issues between Python 2 and + Python 3. + """ + +-try: +- from pip._vendor import chardet +-except ImportError: +- import charset_normalizer as chardet ++from pip._vendor import chardet + + import sys + +@@ -28,10 +25,14 @@ is_py2 = (_ver[0] == 2) #: Python 3.x? is_py3 = (_ver[0] == 3) @@ -60,3 +100,20 @@ index eb6530d..353ec29 100644 # --------- # Specifics + +diff --git a/src/pip/_vendor/requests/help.py b/src/pip/_vendor/requests/help.py +index 3a843404c..745f0d7b3 100644 +--- a/src/pip/_vendor/requests/help.py ++++ b/src/pip/_vendor/requests/help.py +@@ -11,10 +11,7 @@ from pip._vendor import urllib3 + + from . import __version__ as requests_version + +-try: +- import charset_normalizer +-except ImportError: +- charset_normalizer = None ++charset_normalizer = None + + try: + from pip._vendor import chardet