mirror of
https://github.com/pypa/pip
synced 2023-12-13 21:30:23 +01:00
Upgrade the bundled requests to 2.5.0
This commit is contained in:
parent
0c2bf1ae97
commit
0bde4fb9c3
|
@ -13,7 +13,7 @@ Requests is an HTTP library, written in Python, for human beings. Basic GET
|
|||
usage:
|
||||
|
||||
>>> import requests
|
||||
>>> r = requests.get('http://python.org')
|
||||
>>> r = requests.get('https://www.python.org')
|
||||
>>> r.status_code
|
||||
200
|
||||
>>> 'Python is a programming language' in r.content
|
||||
|
@ -22,7 +22,7 @@ usage:
|
|||
... or POST:
|
||||
|
||||
>>> payload = dict(key1='value1', key2='value2')
|
||||
>>> r = requests.post("http://httpbin.org/post", data=payload)
|
||||
>>> r = requests.post('http://httpbin.org/post', data=payload)
|
||||
>>> print(r.text)
|
||||
{
|
||||
...
|
||||
|
@ -42,8 +42,8 @@ is at <http://python-requests.org>.
|
|||
"""
|
||||
|
||||
__title__ = 'requests'
|
||||
__version__ = '2.4.1'
|
||||
__build__ = 0x020401
|
||||
__version__ = '2.5.0'
|
||||
__build__ = 0x020500
|
||||
__author__ = 'Kenneth Reitz'
|
||||
__license__ = 'Apache 2.0'
|
||||
__copyright__ = 'Copyright 2014 Kenneth Reitz'
|
||||
|
|
|
@ -15,9 +15,9 @@ from .packages.urllib3 import Retry
|
|||
from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
|
||||
from .packages.urllib3.response import HTTPResponse
|
||||
from .packages.urllib3.util import Timeout as TimeoutSauce
|
||||
from .compat import urlparse, basestring, urldefrag
|
||||
from .compat import urlparse, basestring
|
||||
from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
|
||||
prepend_scheme_if_needed, get_auth_from_url)
|
||||
prepend_scheme_if_needed, get_auth_from_url, urldefragauth)
|
||||
from .structures import CaseInsensitiveDict
|
||||
from .packages.urllib3.exceptions import ConnectTimeoutError
|
||||
from .packages.urllib3.exceptions import HTTPError as _HTTPError
|
||||
|
@ -26,9 +26,10 @@ from .packages.urllib3.exceptions import ProxyError as _ProxyError
|
|||
from .packages.urllib3.exceptions import ProtocolError
|
||||
from .packages.urllib3.exceptions import ReadTimeoutError
|
||||
from .packages.urllib3.exceptions import SSLError as _SSLError
|
||||
from .packages.urllib3.exceptions import ResponseError
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
|
||||
ProxyError)
|
||||
ProxyError, RetryError)
|
||||
from .auth import _basic_auth_str
|
||||
|
||||
DEFAULT_POOLBLOCK = False
|
||||
|
@ -60,8 +61,12 @@ class HTTPAdapter(BaseAdapter):
|
|||
:param pool_connections: The number of urllib3 connection pools to cache.
|
||||
:param pool_maxsize: The maximum number of connections to save in the pool.
|
||||
:param int max_retries: The maximum number of retries each connection
|
||||
should attempt. Note, this applies only to failed connections and
|
||||
timeouts, never to requests where the server returns a response.
|
||||
should attempt. Note, this applies only to failed DNS lookups, socket
|
||||
connections and connection timeouts, never to requests where data has
|
||||
made it to the server. By default, Requests does not retry failed
|
||||
connections. If you need granular control over the conditions under
|
||||
which we retry a request, import urllib3's ``Retry`` class and pass
|
||||
that instead.
|
||||
:param pool_block: Whether the connection pool should block for connections.
|
||||
|
||||
Usage::
|
||||
|
@ -77,7 +82,10 @@ class HTTPAdapter(BaseAdapter):
|
|||
def __init__(self, pool_connections=DEFAULT_POOLSIZE,
|
||||
pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
|
||||
pool_block=DEFAULT_POOLBLOCK):
|
||||
self.max_retries = max_retries
|
||||
if max_retries == DEFAULT_RETRIES:
|
||||
self.max_retries = Retry(0, read=False)
|
||||
else:
|
||||
self.max_retries = Retry.from_int(max_retries)
|
||||
self.config = {}
|
||||
self.proxy_manager = {}
|
||||
|
||||
|
@ -123,7 +131,7 @@ class HTTPAdapter(BaseAdapter):
|
|||
self._pool_block = block
|
||||
|
||||
self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
|
||||
block=block, **pool_kwargs)
|
||||
block=block, strict=True, **pool_kwargs)
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
"""Return urllib3 ProxyManager for the given proxy.
|
||||
|
@ -270,7 +278,7 @@ class HTTPAdapter(BaseAdapter):
|
|||
proxy = proxies.get(scheme)
|
||||
|
||||
if proxy and scheme != 'https':
|
||||
url, _ = urldefrag(request.url)
|
||||
url = urldefragauth(request.url)
|
||||
else:
|
||||
url = request.path_url
|
||||
|
||||
|
@ -358,7 +366,7 @@ class HTTPAdapter(BaseAdapter):
|
|||
assert_same_host=False,
|
||||
preload_content=False,
|
||||
decode_content=False,
|
||||
retries=Retry(self.max_retries, read=False),
|
||||
retries=self.max_retries,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
|
@ -410,6 +418,9 @@ class HTTPAdapter(BaseAdapter):
|
|||
if isinstance(e.reason, ConnectTimeoutError):
|
||||
raise ConnectTimeout(e, request=request)
|
||||
|
||||
if isinstance(e.reason, ResponseError):
|
||||
raise RetryError(e, request=request)
|
||||
|
||||
raise ConnectionError(e, request=request)
|
||||
|
||||
except _ProxyError as e:
|
||||
|
|
|
@ -22,6 +22,7 @@ def request(method, url, **kwargs):
|
|||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json data to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload.
|
||||
|
@ -45,7 +46,12 @@ def request(method, url, **kwargs):
|
|||
"""
|
||||
|
||||
session = sessions.Session()
|
||||
return session.request(method=method, url=url, **kwargs)
|
||||
response = session.request(method=method, url=url, **kwargs)
|
||||
# By explicitly closing the session, we avoid leaving sockets open which
|
||||
# can trigger a ResourceWarning in some cases, and look like a memory leak
|
||||
# in others.
|
||||
session.close()
|
||||
return response
|
||||
|
||||
|
||||
def get(url, **kwargs):
|
||||
|
@ -81,15 +87,16 @@ def head(url, **kwargs):
|
|||
return request('head', url, **kwargs)
|
||||
|
||||
|
||||
def post(url, data=None, **kwargs):
|
||||
def post(url, data=None, json=None, **kwargs):
|
||||
"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json data to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('post', url, data=data, **kwargs)
|
||||
return request('post', url, data=data, json=json, **kwargs)
|
||||
|
||||
|
||||
def put(url, data=None, **kwargs):
|
||||
|
|
|
@ -17,6 +17,7 @@ from base64 import b64encode
|
|||
from .compat import urlparse, str
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .utils import parse_dict_header, to_native_string
|
||||
from .status_codes import codes
|
||||
|
||||
CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
|
||||
CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
|
||||
|
@ -150,6 +151,11 @@ class HTTPDigestAuth(AuthBase):
|
|||
|
||||
return 'Digest %s' % (base)
|
||||
|
||||
def handle_redirect(self, r, **kwargs):
|
||||
"""Reset num_401_calls counter on redirects."""
|
||||
if r.is_redirect:
|
||||
setattr(self, 'num_401_calls', 1)
|
||||
|
||||
def handle_401(self, r, **kwargs):
|
||||
"""Takes the given response and tries digest-auth, if needed."""
|
||||
|
||||
|
@ -182,7 +188,7 @@ class HTTPDigestAuth(AuthBase):
|
|||
|
||||
return _r
|
||||
|
||||
setattr(self, 'num_401_calls', 1)
|
||||
setattr(self, 'num_401_calls', num_401_calls + 1)
|
||||
return r
|
||||
|
||||
def __call__(self, r):
|
||||
|
@ -192,6 +198,11 @@ class HTTPDigestAuth(AuthBase):
|
|||
try:
|
||||
self.pos = r.body.tell()
|
||||
except AttributeError:
|
||||
pass
|
||||
# In the case of HTTPDigestAuth being reused and the body of
|
||||
# the previous request was a file-like object, pos has the
|
||||
# file position of the previous body. Ensure it's set to
|
||||
# None.
|
||||
self.pos = None
|
||||
r.register_hook('response', self.handle_401)
|
||||
r.register_hook('response', self.handle_redirect)
|
||||
return r
|
||||
|
|
|
@ -89,3 +89,11 @@ class ChunkedEncodingError(RequestException):
|
|||
|
||||
class ContentDecodingError(RequestException, BaseHTTPError):
|
||||
"""Failed to decode response content"""
|
||||
|
||||
|
||||
class StreamConsumedError(RequestException, TypeError):
|
||||
"""The content for this response was already consumed"""
|
||||
|
||||
|
||||
class RetryError(RequestException):
|
||||
"""Custom retries logic failed"""
|
||||
|
|
|
@ -22,8 +22,9 @@ from .packages.urllib3.util import parse_url
|
|||
from .packages.urllib3.exceptions import (
|
||||
DecodeError, ReadTimeoutError, ProtocolError)
|
||||
from .exceptions import (
|
||||
HTTPError, RequestException, MissingSchema, InvalidURL,
|
||||
ChunkedEncodingError, ContentDecodingError, ConnectionError)
|
||||
HTTPError, RequestException, MissingSchema, InvalidURL,
|
||||
ChunkedEncodingError, ContentDecodingError, ConnectionError,
|
||||
StreamConsumedError)
|
||||
from .utils import (
|
||||
guess_filename, get_auth_from_url, requote_uri,
|
||||
stream_decode_response_unicode, to_key_val_list, parse_header_links,
|
||||
|
@ -46,6 +47,8 @@ DEFAULT_REDIRECT_LIMIT = 30
|
|||
CONTENT_CHUNK_SIZE = 10 * 1024
|
||||
ITER_CHUNK_SIZE = 512
|
||||
|
||||
json_dumps = json.dumps
|
||||
|
||||
|
||||
class RequestEncodingMixin(object):
|
||||
@property
|
||||
|
@ -189,7 +192,8 @@ class Request(RequestHooksMixin):
|
|||
:param url: URL to send.
|
||||
:param headers: dictionary of headers to send.
|
||||
:param files: dictionary of {filename: fileobject} files to multipart upload.
|
||||
:param data: the body to attach the request. If a dictionary is provided, form-encoding will take place.
|
||||
:param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
|
||||
:param json: json for the body to attach to the request (if data is not specified).
|
||||
:param params: dictionary of URL parameters to append to the URL.
|
||||
:param auth: Auth handler or (user, pass) tuple.
|
||||
:param cookies: dictionary or CookieJar of cookies to attach to this request.
|
||||
|
@ -212,7 +216,8 @@ class Request(RequestHooksMixin):
|
|||
params=None,
|
||||
auth=None,
|
||||
cookies=None,
|
||||
hooks=None):
|
||||
hooks=None,
|
||||
json=None):
|
||||
|
||||
# Default empty dicts for dict params.
|
||||
data = [] if data is None else data
|
||||
|
@ -230,6 +235,7 @@ class Request(RequestHooksMixin):
|
|||
self.headers = headers
|
||||
self.files = files
|
||||
self.data = data
|
||||
self.json = json
|
||||
self.params = params
|
||||
self.auth = auth
|
||||
self.cookies = cookies
|
||||
|
@ -246,6 +252,7 @@ class Request(RequestHooksMixin):
|
|||
headers=self.headers,
|
||||
files=self.files,
|
||||
data=self.data,
|
||||
json=self.json,
|
||||
params=self.params,
|
||||
auth=self.auth,
|
||||
cookies=self.cookies,
|
||||
|
@ -289,14 +296,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
self.hooks = default_hooks()
|
||||
|
||||
def prepare(self, method=None, url=None, headers=None, files=None,
|
||||
data=None, params=None, auth=None, cookies=None, hooks=None):
|
||||
data=None, params=None, auth=None, cookies=None, hooks=None,
|
||||
json=None):
|
||||
"""Prepares the entire request with the given parameters."""
|
||||
|
||||
self.prepare_method(method)
|
||||
self.prepare_url(url, params)
|
||||
self.prepare_headers(headers)
|
||||
self.prepare_cookies(cookies)
|
||||
self.prepare_body(data, files)
|
||||
self.prepare_body(data, files, json)
|
||||
self.prepare_auth(auth, url)
|
||||
# Note that prepare_auth must be last to enable authentication schemes
|
||||
# such as OAuth to work on a fully prepared request.
|
||||
|
@ -326,15 +334,18 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
def prepare_url(self, url, params):
|
||||
"""Prepares the given HTTP URL."""
|
||||
#: Accept objects that have string representations.
|
||||
try:
|
||||
url = unicode(url)
|
||||
except NameError:
|
||||
# We're on Python 3.
|
||||
url = str(url)
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
#: We're unable to blindy call unicode/str functions
|
||||
#: as this will include the bytestring indicator (b'')
|
||||
#: on python 3.x.
|
||||
#: https://github.com/kennethreitz/requests/pull/2238
|
||||
if isinstance(url, bytes):
|
||||
url = url.decode('utf8')
|
||||
else:
|
||||
url = unicode(url) if is_py2 else str(url)
|
||||
|
||||
# Don't do any URL preparation for oddball schemes
|
||||
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
|
||||
# `data` etc to work around exceptions from `url_parse`, which
|
||||
# handles RFC 3986 only.
|
||||
if ':' in url and not url.lower().startswith('http'):
|
||||
self.url = url
|
||||
return
|
||||
|
@ -397,7 +408,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
else:
|
||||
self.headers = CaseInsensitiveDict()
|
||||
|
||||
def prepare_body(self, data, files):
|
||||
def prepare_body(self, data, files, json=None):
|
||||
"""Prepares the given HTTP body data."""
|
||||
|
||||
# Check if file, fo, generator, iterator.
|
||||
|
@ -408,6 +419,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
content_type = None
|
||||
length = None
|
||||
|
||||
if json is not None:
|
||||
content_type = 'application/json'
|
||||
body = json_dumps(json)
|
||||
|
||||
is_stream = all([
|
||||
hasattr(data, '__iter__'),
|
||||
not isinstance(data, (basestring, list, tuple, dict))
|
||||
|
@ -433,7 +448,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
if files:
|
||||
(body, content_type) = self._encode_files(files, data)
|
||||
else:
|
||||
if data:
|
||||
if data and json is None:
|
||||
body = self._encode_params(data)
|
||||
if isinstance(data, basestring) or hasattr(data, 'read'):
|
||||
content_type = None
|
||||
|
@ -443,7 +458,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
self.prepare_content_length(body)
|
||||
|
||||
# Add content-type if it wasn't explicitly provided.
|
||||
if (content_type) and (not 'content-type' in self.headers):
|
||||
if content_type and ('content-type' not in self.headers):
|
||||
self.headers['Content-Type'] = content_type
|
||||
|
||||
self.body = body
|
||||
|
@ -457,7 +472,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
l = super_len(body)
|
||||
if l:
|
||||
self.headers['Content-Length'] = builtin_str(l)
|
||||
elif self.method not in ('GET', 'HEAD'):
|
||||
elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None):
|
||||
self.headers['Content-Length'] = '0'
|
||||
|
||||
def prepare_auth(self, auth, url=''):
|
||||
|
@ -653,6 +668,8 @@ class Response(object):
|
|||
|
||||
self._content_consumed = True
|
||||
|
||||
if self._content_consumed and isinstance(self._content, bool):
|
||||
raise StreamConsumedError()
|
||||
# simulate reading small chunks of the content
|
||||
reused_chunks = iter_slices(self._content, chunk_size)
|
||||
|
||||
|
@ -665,7 +682,7 @@ class Response(object):
|
|||
|
||||
return chunks
|
||||
|
||||
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
|
||||
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
|
||||
"""Iterates over the response data, one line at a time. When
|
||||
stream=True is set on the request, this avoids reading the
|
||||
content at once into memory for large responses.
|
||||
|
@ -677,7 +694,11 @@ class Response(object):
|
|||
|
||||
if pending is not None:
|
||||
chunk = pending + chunk
|
||||
lines = chunk.splitlines()
|
||||
|
||||
if delimiter:
|
||||
lines = chunk.split(delimiter)
|
||||
else:
|
||||
lines = chunk.splitlines()
|
||||
|
||||
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
|
||||
pending = lines.pop()
|
||||
|
|
|
@ -15,7 +15,7 @@
|
|||
# 02110-1301 USA
|
||||
######################### END LICENSE BLOCK #########################
|
||||
|
||||
__version__ = "2.2.1"
|
||||
__version__ = "2.3.0"
|
||||
from sys import version_info
|
||||
|
||||
|
||||
|
|
|
@ -12,34 +12,68 @@ Example::
|
|||
If no paths are provided, it takes its input from stdin.
|
||||
|
||||
"""
|
||||
from io import open
|
||||
from sys import argv, stdin
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from io import open
|
||||
|
||||
from chardet import __version__
|
||||
from chardet.universaldetector import UniversalDetector
|
||||
|
||||
|
||||
def description_of(file, name='stdin'):
|
||||
"""Return a string describing the probable encoding of a file."""
|
||||
def description_of(lines, name='stdin'):
|
||||
"""
|
||||
Return a string describing the probable encoding of a file or
|
||||
list of strings.
|
||||
|
||||
:param lines: The lines to get the encoding of.
|
||||
:type lines: Iterable of bytes
|
||||
:param name: Name of file or collection of lines
|
||||
:type name: str
|
||||
"""
|
||||
u = UniversalDetector()
|
||||
for line in file:
|
||||
for line in lines:
|
||||
u.feed(line)
|
||||
u.close()
|
||||
result = u.result
|
||||
if result['encoding']:
|
||||
return '%s: %s with confidence %s' % (name,
|
||||
result['encoding'],
|
||||
result['confidence'])
|
||||
return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
|
||||
result['confidence'])
|
||||
else:
|
||||
return '%s: no result' % name
|
||||
return '{0}: no result'.format(name)
|
||||
|
||||
|
||||
def main():
|
||||
if len(argv) <= 1:
|
||||
print(description_of(stdin))
|
||||
else:
|
||||
for path in argv[1:]:
|
||||
with open(path, 'rb') as f:
|
||||
print(description_of(f, path))
|
||||
def main(argv=None):
|
||||
'''
|
||||
Handles command line arguments and gets things started.
|
||||
|
||||
:param argv: List of arguments, as if specified on the command-line.
|
||||
If None, ``sys.argv[1:]`` is used instead.
|
||||
:type argv: list of str
|
||||
'''
|
||||
# Get command line arguments
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Takes one or more file paths and reports their detected \
|
||||
encodings",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
conflict_handler='resolve')
|
||||
parser.add_argument('input',
|
||||
help='File whose encoding we would like to determine.',
|
||||
type=argparse.FileType('rb'), nargs='*',
|
||||
default=[sys.stdin])
|
||||
parser.add_argument('--version', action='version',
|
||||
version='%(prog)s {0}'.format(__version__))
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
for f in args.input:
|
||||
if f.isatty():
|
||||
print("You are running chardetect interactively. Press " +
|
||||
"CTRL-D twice at the start of a blank line to signal the " +
|
||||
"end of your input. If you want help, run chardetect " +
|
||||
"--help\n", file=sys.stderr)
|
||||
print(description_of(f, f.name))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
@ -177,6 +177,12 @@ class JapaneseContextAnalysis:
|
|||
return -1, 1
|
||||
|
||||
class SJISContextAnalysis(JapaneseContextAnalysis):
|
||||
def __init__(self):
|
||||
self.charset_name = "SHIFT_JIS"
|
||||
|
||||
def get_charset_name(self):
|
||||
return self.charset_name
|
||||
|
||||
def get_order(self, aBuf):
|
||||
if not aBuf:
|
||||
return -1, 1
|
||||
|
@ -184,6 +190,8 @@ class SJISContextAnalysis(JapaneseContextAnalysis):
|
|||
first_char = wrap_ord(aBuf[0])
|
||||
if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)):
|
||||
charLen = 2
|
||||
if (first_char == 0x87) or (0xFA <= first_char <= 0xFC):
|
||||
self.charset_name = "CP932"
|
||||
else:
|
||||
charLen = 1
|
||||
|
||||
|
|
|
@ -129,11 +129,11 @@ class Latin1Prober(CharSetProber):
|
|||
if total < 0.01:
|
||||
confidence = 0.0
|
||||
else:
|
||||
confidence = ((self._mFreqCounter[3] / total)
|
||||
- (self._mFreqCounter[1] * 20.0 / total))
|
||||
confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0)
|
||||
/ total)
|
||||
if confidence < 0.0:
|
||||
confidence = 0.0
|
||||
# lower the confidence of latin1 so that other more accurate
|
||||
# detector can take priority.
|
||||
confidence = confidence * 0.5
|
||||
confidence = confidence * 0.73
|
||||
return confidence
|
||||
|
|
|
@ -353,7 +353,7 @@ SJIS_cls = (
|
|||
2,2,2,2,2,2,2,2, # 68 - 6f
|
||||
2,2,2,2,2,2,2,2, # 70 - 77
|
||||
2,2,2,2,2,2,2,1, # 78 - 7f
|
||||
3,3,3,3,3,3,3,3, # 80 - 87
|
||||
3,3,3,3,3,2,2,3, # 80 - 87
|
||||
3,3,3,3,3,3,3,3, # 88 - 8f
|
||||
3,3,3,3,3,3,3,3, # 90 - 97
|
||||
3,3,3,3,3,3,3,3, # 98 - 9f
|
||||
|
@ -369,9 +369,8 @@ SJIS_cls = (
|
|||
2,2,2,2,2,2,2,2, # d8 - df
|
||||
3,3,3,3,3,3,3,3, # e0 - e7
|
||||
3,3,3,3,3,4,4,4, # e8 - ef
|
||||
4,4,4,4,4,4,4,4, # f0 - f7
|
||||
4,4,4,4,4,0,0,0 # f8 - ff
|
||||
)
|
||||
3,3,3,3,3,3,3,3, # f0 - f7
|
||||
3,3,3,3,3,0,0,0) # f8 - ff
|
||||
|
||||
|
||||
SJIS_st = (
|
||||
|
@ -571,5 +570,3 @@ UTF8SMModel = {'classTable': UTF8_cls,
|
|||
'stateTable': UTF8_st,
|
||||
'charLenTable': UTF8CharLenTable,
|
||||
'name': 'UTF-8'}
|
||||
|
||||
# flake8: noqa
|
||||
|
|
|
@ -47,7 +47,7 @@ class SJISProber(MultiByteCharSetProber):
|
|||
self._mContextAnalyzer.reset()
|
||||
|
||||
def get_charset_name(self):
|
||||
return "SHIFT_JIS"
|
||||
return self._mContextAnalyzer.get_charset_name()
|
||||
|
||||
def feed(self, aBuf):
|
||||
aLen = len(aBuf)
|
||||
|
|
|
@ -71,9 +71,9 @@ class UniversalDetector:
|
|||
|
||||
if not self._mGotData:
|
||||
# If the data starts with BOM, we know it is UTF
|
||||
if aBuf[:3] == codecs.BOM:
|
||||
if aBuf[:3] == codecs.BOM_UTF8:
|
||||
# EF BB BF UTF-8 with BOM
|
||||
self.result = {'encoding': "UTF-8", 'confidence': 1.0}
|
||||
self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0}
|
||||
elif aBuf[:4] == codecs.BOM_UTF32_LE:
|
||||
# FF FE 00 00 UTF-32, little-endian BOM
|
||||
self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}
|
||||
|
|
|
@ -57,7 +57,7 @@ del NullHandler
|
|||
|
||||
# Set security warning to only go off once by default.
|
||||
import warnings
|
||||
warnings.simplefilter('module', exceptions.SecurityWarning)
|
||||
warnings.simplefilter('always', exceptions.SecurityWarning)
|
||||
|
||||
def disable_warnings(category=exceptions.HTTPWarning):
|
||||
"""
|
||||
|
|
|
@ -14,7 +14,7 @@ try: # Python 2.7+
|
|||
from collections import OrderedDict
|
||||
except ImportError:
|
||||
from .packages.ordered_dict import OrderedDict
|
||||
from .packages.six import itervalues
|
||||
from .packages.six import iterkeys, itervalues
|
||||
|
||||
|
||||
__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
|
||||
|
@ -85,8 +85,7 @@ class RecentlyUsedContainer(MutableMapping):
|
|||
def clear(self):
|
||||
with self.lock:
|
||||
# Copy pointers to all values, then wipe the mapping
|
||||
# under Python 2, this copies the list of values twice :-|
|
||||
values = list(self._container.values())
|
||||
values = list(itervalues(self._container))
|
||||
self._container.clear()
|
||||
|
||||
if self.dispose_func:
|
||||
|
@ -95,7 +94,7 @@ class RecentlyUsedContainer(MutableMapping):
|
|||
|
||||
def keys(self):
|
||||
with self.lock:
|
||||
return self._container.keys()
|
||||
return list(iterkeys(self._container))
|
||||
|
||||
|
||||
class HTTPHeaderDict(MutableMapping):
|
||||
|
|
|
@ -3,6 +3,7 @@ import sys
|
|||
import socket
|
||||
from socket import timeout as SocketTimeout
|
||||
import warnings
|
||||
from .packages import six
|
||||
|
||||
try: # Python 3
|
||||
from http.client import HTTPConnection as _HTTPConnection, HTTPException
|
||||
|
@ -26,12 +27,20 @@ except (ImportError, AttributeError): # Platform-specific: No SSL.
|
|||
pass
|
||||
|
||||
|
||||
try: # Python 3:
|
||||
# Not a no-op, we're adding this to the namespace so it can be imported.
|
||||
ConnectionError = ConnectionError
|
||||
except NameError: # Python 2:
|
||||
class ConnectionError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
from .exceptions import (
|
||||
ConnectTimeoutError,
|
||||
SystemTimeWarning,
|
||||
SecurityWarning,
|
||||
)
|
||||
from .packages.ssl_match_hostname import match_hostname
|
||||
from .packages import six
|
||||
|
||||
from .util.ssl_ import (
|
||||
resolve_cert_reqs,
|
||||
|
@ -40,8 +49,8 @@ from .util.ssl_ import (
|
|||
assert_fingerprint,
|
||||
)
|
||||
|
||||
from .util import connection
|
||||
|
||||
from .util import connection
|
||||
|
||||
port_by_scheme = {
|
||||
'http': 80,
|
||||
|
@ -233,8 +242,15 @@ class VerifiedHTTPSConnection(HTTPSConnection):
|
|||
self.assert_fingerprint)
|
||||
elif resolved_cert_reqs != ssl.CERT_NONE \
|
||||
and self.assert_hostname is not False:
|
||||
match_hostname(self.sock.getpeercert(),
|
||||
self.assert_hostname or hostname)
|
||||
cert = self.sock.getpeercert()
|
||||
if not cert.get('subjectAltName', ()):
|
||||
warnings.warn((
|
||||
'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. '
|
||||
'This feature is being removed by major browsers and deprecated by RFC 2818. '
|
||||
'(See https://github.com/shazow/urllib3/issues/497 for details.)'),
|
||||
SecurityWarning
|
||||
)
|
||||
match_hostname(cert, self.assert_hostname or hostname)
|
||||
|
||||
self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
|
||||
or self.assert_fingerprint is not None)
|
||||
|
|
|
@ -32,7 +32,7 @@ from .connection import (
|
|||
port_by_scheme,
|
||||
DummyConnection,
|
||||
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
|
||||
HTTPException, BaseSSLError,
|
||||
HTTPException, BaseSSLError, ConnectionError
|
||||
)
|
||||
from .request import RequestMethods
|
||||
from .response import HTTPResponse
|
||||
|
@ -278,6 +278,23 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
# can be removed later
|
||||
return Timeout.from_float(timeout)
|
||||
|
||||
def _raise_timeout(self, err, url, timeout_value):
|
||||
"""Is the error actually a timeout? Will raise a ReadTimeout or pass"""
|
||||
|
||||
if isinstance(err, SocketTimeout):
|
||||
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
|
||||
|
||||
# See the above comment about EAGAIN in Python 3. In Python 2 we have
|
||||
# to specifically catch it and throw the timeout error
|
||||
if hasattr(err, 'errno') and err.errno in _blocking_errnos:
|
||||
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
|
||||
|
||||
# Catch possible read timeouts thrown as SSL errors. If not the
|
||||
# case, rethrow the original. We need to do this because of:
|
||||
# http://bugs.python.org/issue10272
|
||||
if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6
|
||||
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
|
||||
|
||||
def _make_request(self, conn, method, url, timeout=_Default,
|
||||
**httplib_request_kw):
|
||||
"""
|
||||
|
@ -301,7 +318,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
conn.timeout = timeout_obj.connect_timeout
|
||||
|
||||
# Trigger any extra validation we need to do.
|
||||
self._validate_conn(conn)
|
||||
try:
|
||||
self._validate_conn(conn)
|
||||
except (SocketTimeout, BaseSSLError) as e:
|
||||
# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
|
||||
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
|
||||
raise
|
||||
|
||||
# conn.request() calls httplib.*.request, not the method in
|
||||
# urllib3.request. It also calls makefile (recv) on the socket.
|
||||
|
@ -331,28 +353,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
httplib_response = conn.getresponse(buffering=True)
|
||||
except TypeError: # Python 2.6 and older
|
||||
httplib_response = conn.getresponse()
|
||||
except SocketTimeout:
|
||||
raise ReadTimeoutError(
|
||||
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
|
||||
|
||||
except BaseSSLError as e:
|
||||
# Catch possible read timeouts thrown as SSL errors. If not the
|
||||
# case, rethrow the original. We need to do this because of:
|
||||
# http://bugs.python.org/issue10272
|
||||
if 'timed out' in str(e) or \
|
||||
'did not complete (read)' in str(e): # Python 2.6
|
||||
raise ReadTimeoutError(
|
||||
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
|
||||
|
||||
raise
|
||||
|
||||
except SocketError as e: # Platform-specific: Python 2
|
||||
# See the above comment about EAGAIN in Python 3. In Python 2 we
|
||||
# have to specifically catch it and throw the timeout error
|
||||
if e.errno in _blocking_errnos:
|
||||
raise ReadTimeoutError(
|
||||
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
|
||||
|
||||
except (SocketTimeout, BaseSSLError, SocketError) as e:
|
||||
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
|
||||
raise
|
||||
|
||||
# AppEngine doesn't have a version attr.
|
||||
|
@ -537,12 +539,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
raise EmptyPoolError(self, "No pool connections are available.")
|
||||
|
||||
except (BaseSSLError, CertificateError) as e:
|
||||
# Release connection unconditionally because there is no way to
|
||||
# close it externally in case of exception.
|
||||
release_conn = True
|
||||
# Close the connection. If a connection is reused on which there
|
||||
# was a Certificate error, the next request will certainly raise
|
||||
# another Certificate error.
|
||||
if conn:
|
||||
conn.close()
|
||||
conn = None
|
||||
raise SSLError(e)
|
||||
|
||||
except (TimeoutError, HTTPException, SocketError) as e:
|
||||
except (TimeoutError, HTTPException, SocketError, ConnectionError) as e:
|
||||
if conn:
|
||||
# Discard the connection for these exceptions. It will be
|
||||
# be replaced during the next _get_conn() call.
|
||||
|
@ -725,8 +730,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
|
|||
warnings.warn((
|
||||
'Unverified HTTPS request is being made. '
|
||||
'Adding certificate verification is strongly advised. See: '
|
||||
'https://urllib3.readthedocs.org/en/latest/security.html '
|
||||
'(This warning will only appear once by default.)'),
|
||||
'https://urllib3.readthedocs.org/en/latest/security.html'),
|
||||
InsecureRequestWarning)
|
||||
|
||||
|
||||
|
|
|
@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI
|
|||
when the required modules are installed.
|
||||
|
||||
Activating this module also has the positive side effect of disabling SSL/TLS
|
||||
encryption in Python 2 (see `CRIME attack`_).
|
||||
compression in Python 2 (see `CRIME attack`_).
|
||||
|
||||
If you want to configure the default list of supported cipher suites, you can
|
||||
set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
|
||||
|
@ -70,9 +70,14 @@ HAS_SNI = SUBJ_ALT_NAME_SUPPORT
|
|||
# Map from urllib3 to PyOpenSSL compatible parameter-values.
|
||||
_openssl_versions = {
|
||||
ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
|
||||
ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD,
|
||||
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
|
||||
}
|
||||
|
||||
try:
|
||||
_openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
_openssl_verify = {
|
||||
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
|
||||
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
|
||||
|
@ -199,8 +204,21 @@ class WrappedSocket(object):
|
|||
def settimeout(self, timeout):
|
||||
return self.socket.settimeout(timeout)
|
||||
|
||||
def _send_until_done(self, data):
|
||||
while True:
|
||||
try:
|
||||
return self.connection.send(data)
|
||||
except OpenSSL.SSL.WantWriteError:
|
||||
_, wlist, _ = select.select([], [self.socket], [],
|
||||
self.socket.gettimeout())
|
||||
if not wlist:
|
||||
raise timeout()
|
||||
continue
|
||||
|
||||
def sendall(self, data):
|
||||
return self.connection.sendall(data)
|
||||
while len(data):
|
||||
sent = self._send_until_done(data)
|
||||
data = data[sent:]
|
||||
|
||||
def close(self):
|
||||
if self._makefile_refs < 1:
|
||||
|
@ -248,6 +266,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
|
|||
ssl_version=None):
|
||||
ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
|
||||
if certfile:
|
||||
keyfile = keyfile or certfile # Match behaviour of the normal python ssl library
|
||||
ctx.use_certificate_file(certfile)
|
||||
if keyfile:
|
||||
ctx.use_privatekey_file(keyfile)
|
||||
|
|
|
@ -72,11 +72,8 @@ class MaxRetryError(RequestError):
|
|||
def __init__(self, pool, url, reason=None):
|
||||
self.reason = reason
|
||||
|
||||
message = "Max retries exceeded with url: %s" % url
|
||||
if reason:
|
||||
message += " (Caused by %r)" % reason
|
||||
else:
|
||||
message += " (Caused by redirect)"
|
||||
message = "Max retries exceeded with url: %s (Caused by %r)" % (
|
||||
url, reason)
|
||||
|
||||
RequestError.__init__(self, pool, url, message)
|
||||
|
||||
|
@ -141,6 +138,12 @@ class LocationParseError(LocationValueError):
|
|||
self.location = location
|
||||
|
||||
|
||||
class ResponseError(HTTPError):
|
||||
"Used as a container for an error reason supplied in a MaxRetryError."
|
||||
GENERIC_ERROR = 'too many error responses'
|
||||
SPECIFIC_ERROR = 'too many {status_code} error responses'
|
||||
|
||||
|
||||
class SecurityWarning(HTTPWarning):
|
||||
"Warned when perfoming security reducing actions"
|
||||
pass
|
||||
|
|
|
@ -118,18 +118,24 @@ class RequestMethods(object):
|
|||
which is used to compose the body of the request. The random boundary
|
||||
string can be explicitly set with the ``multipart_boundary`` parameter.
|
||||
"""
|
||||
if encode_multipart:
|
||||
body, content_type = encode_multipart_formdata(
|
||||
fields or {}, boundary=multipart_boundary)
|
||||
else:
|
||||
body, content_type = (urlencode(fields or {}),
|
||||
'application/x-www-form-urlencoded')
|
||||
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
headers_ = {'Content-Type': content_type}
|
||||
headers_.update(headers)
|
||||
extra_kw = {'headers': {}}
|
||||
|
||||
return self.urlopen(method, url, body=body, headers=headers_,
|
||||
**urlopen_kw)
|
||||
if fields:
|
||||
if 'body' in urlopen_kw:
|
||||
raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.')
|
||||
|
||||
if encode_multipart:
|
||||
body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
|
||||
else:
|
||||
body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'
|
||||
|
||||
extra_kw['body'] = body
|
||||
extra_kw['headers'] = {'Content-Type': content_type}
|
||||
|
||||
extra_kw['headers'].update(headers)
|
||||
extra_kw.update(urlopen_kw)
|
||||
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
|
|
|
@ -2,10 +2,11 @@ import time
|
|||
import logging
|
||||
|
||||
from ..exceptions import (
|
||||
ProtocolError,
|
||||
ConnectTimeoutError,
|
||||
ReadTimeoutError,
|
||||
MaxRetryError,
|
||||
ProtocolError,
|
||||
ReadTimeoutError,
|
||||
ResponseError,
|
||||
)
|
||||
from ..packages import six
|
||||
|
||||
|
@ -36,7 +37,6 @@ class Retry(object):
|
|||
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
|
||||
retries are disabled, in which case the causing exception will be raised.
|
||||
|
||||
|
||||
:param int total:
|
||||
Total number of retries to allow. Takes precedence over other counts.
|
||||
|
||||
|
@ -184,8 +184,8 @@ class Retry(object):
|
|||
return isinstance(err, ConnectTimeoutError)
|
||||
|
||||
def _is_read_error(self, err):
|
||||
""" Errors that occur after the request has been started, so we can't
|
||||
assume that the server did not process any of it.
|
||||
""" Errors that occur after the request has been started, so we should
|
||||
assume that the server began processing it.
|
||||
"""
|
||||
return isinstance(err, (ReadTimeoutError, ProtocolError))
|
||||
|
||||
|
@ -198,8 +198,7 @@ class Retry(object):
|
|||
return self.status_forcelist and status_code in self.status_forcelist
|
||||
|
||||
def is_exhausted(self):
|
||||
""" Are we out of retries?
|
||||
"""
|
||||
""" Are we out of retries? """
|
||||
retry_counts = (self.total, self.connect, self.read, self.redirect)
|
||||
retry_counts = list(filter(None, retry_counts))
|
||||
if not retry_counts:
|
||||
|
@ -230,6 +229,7 @@ class Retry(object):
|
|||
connect = self.connect
|
||||
read = self.read
|
||||
redirect = self.redirect
|
||||
cause = 'unknown'
|
||||
|
||||
if error and self._is_connection_error(error):
|
||||
# Connect retry?
|
||||
|
@ -251,10 +251,16 @@ class Retry(object):
|
|||
# Redirect retry?
|
||||
if redirect is not None:
|
||||
redirect -= 1
|
||||
cause = 'too many redirects'
|
||||
|
||||
else:
|
||||
# FIXME: Nothing changed, scenario doesn't make sense.
|
||||
# Incrementing because of a server error like a 500 in
|
||||
# status_forcelist and a the given method is in the whitelist
|
||||
_observed_errors += 1
|
||||
cause = ResponseError.GENERIC_ERROR
|
||||
if response and response.status:
|
||||
cause = ResponseError.SPECIFIC_ERROR.format(
|
||||
status_code=response.status)
|
||||
|
||||
new_retry = self.new(
|
||||
total=total,
|
||||
|
@ -262,7 +268,7 @@ class Retry(object):
|
|||
_observed_errors=_observed_errors)
|
||||
|
||||
if new_retry.is_exhausted():
|
||||
raise MaxRetryError(_pool, url, error)
|
||||
raise MaxRetryError(_pool, url, error or ResponseError(cause))
|
||||
|
||||
log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry))
|
||||
|
||||
|
|
|
@ -4,18 +4,84 @@ from hashlib import md5, sha1
|
|||
from ..exceptions import SSLError
|
||||
|
||||
|
||||
try: # Test for SSL features
|
||||
SSLContext = None
|
||||
HAS_SNI = False
|
||||
SSLContext = None
|
||||
HAS_SNI = False
|
||||
create_default_context = None
|
||||
|
||||
import ssl
|
||||
import errno
|
||||
import ssl
|
||||
|
||||
try: # Test for SSL features
|
||||
from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
|
||||
from ssl import SSLContext # Modern SSL?
|
||||
from ssl import HAS_SNI # Has SNI?
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
|
||||
except ImportError:
|
||||
OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
|
||||
OP_NO_COMPRESSION = 0x20000
|
||||
|
||||
try:
|
||||
from ssl import _DEFAULT_CIPHERS
|
||||
except ImportError:
|
||||
_DEFAULT_CIPHERS = (
|
||||
'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
|
||||
'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:ECDH+RC4:'
|
||||
'DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5'
|
||||
)
|
||||
|
||||
try:
|
||||
from ssl import SSLContext # Modern SSL?
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
class SSLContext(object): # Platform-specific: Python 2 & 3.1
|
||||
supports_set_ciphers = sys.version_info >= (2, 7)
|
||||
|
||||
def __init__(self, protocol_version):
|
||||
self.protocol = protocol_version
|
||||
# Use default values from a real SSLContext
|
||||
self.check_hostname = False
|
||||
self.verify_mode = ssl.CERT_NONE
|
||||
self.ca_certs = None
|
||||
self.options = 0
|
||||
self.certfile = None
|
||||
self.keyfile = None
|
||||
self.ciphers = None
|
||||
|
||||
def load_cert_chain(self, certfile, keyfile):
|
||||
self.certfile = certfile
|
||||
self.keyfile = keyfile
|
||||
|
||||
def load_verify_locations(self, location):
|
||||
self.ca_certs = location
|
||||
|
||||
def set_ciphers(self, cipher_suite):
|
||||
if not self.supports_set_ciphers:
|
||||
raise TypeError(
|
||||
'Your version of Python does not support setting '
|
||||
'a custom cipher suite. Please upgrade to Python '
|
||||
'2.7, 3.2, or later if you need this functionality.'
|
||||
)
|
||||
self.ciphers = cipher_suite
|
||||
|
||||
def wrap_socket(self, socket, server_hostname=None):
|
||||
kwargs = {
|
||||
'keyfile': self.keyfile,
|
||||
'certfile': self.certfile,
|
||||
'ca_certs': self.ca_certs,
|
||||
'cert_reqs': self.verify_mode,
|
||||
'ssl_version': self.protocol,
|
||||
}
|
||||
if self.supports_set_ciphers: # Platform-specific: Python 2.7+
|
||||
return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
|
||||
else: # Platform-specific: Python 2.6
|
||||
return wrap_socket(socket, **kwargs)
|
||||
|
||||
|
||||
def assert_fingerprint(cert, fingerprint):
|
||||
"""
|
||||
Checks if given fingerprint matches the supplied certificate.
|
||||
|
@ -91,42 +157,98 @@ def resolve_ssl_version(candidate):
|
|||
return candidate
|
||||
|
||||
|
||||
if SSLContext is not None: # Python 3.2+
|
||||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
|
||||
ca_certs=None, server_hostname=None,
|
||||
ssl_version=None):
|
||||
"""
|
||||
All arguments except `server_hostname` have the same meaning as for
|
||||
:func:`ssl.wrap_socket`
|
||||
def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
|
||||
options=None, ciphers=None):
|
||||
"""All arguments have the same meaning as ``ssl_wrap_socket``.
|
||||
|
||||
:param server_hostname:
|
||||
Hostname of the expected certificate
|
||||
"""
|
||||
context = SSLContext(ssl_version)
|
||||
context.verify_mode = cert_reqs
|
||||
By default, this function does a lot of the same work that
|
||||
``ssl.create_default_context`` does on Python 3.4+. It:
|
||||
|
||||
# Disable TLS compression to migitate CRIME attack (issue #309)
|
||||
OP_NO_COMPRESSION = 0x20000
|
||||
context.options |= OP_NO_COMPRESSION
|
||||
- Disables SSLv2, SSLv3, and compression
|
||||
- Sets a restricted set of server ciphers
|
||||
|
||||
if ca_certs:
|
||||
try:
|
||||
context.load_verify_locations(ca_certs)
|
||||
# Py32 raises IOError
|
||||
# Py33 raises FileNotFoundError
|
||||
except Exception as e: # Reraise as SSLError
|
||||
If you wish to enable SSLv3, you can do::
|
||||
|
||||
from urllib3.util import ssl_
|
||||
context = ssl_.create_urllib3_context()
|
||||
context.options &= ~ssl_.OP_NO_SSLv3
|
||||
|
||||
You can do the same to enable compression (substituting ``COMPRESSION``
|
||||
for ``SSLv3`` in the last line above).
|
||||
|
||||
:param ssl_version:
|
||||
The desired protocol version to use. This will default to
|
||||
PROTOCOL_SSLv23 which will negotiate the highest protocol that both
|
||||
the server and your installation of OpenSSL support.
|
||||
:param cert_reqs:
|
||||
Whether to require the certificate verification. This defaults to
|
||||
``ssl.CERT_REQUIRED``.
|
||||
:param options:
|
||||
Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
|
||||
``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
|
||||
:param ciphers:
|
||||
Which cipher suites to allow the server to select.
|
||||
:returns:
|
||||
Constructed SSLContext object with specified options
|
||||
:rtype: SSLContext
|
||||
"""
|
||||
context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
|
||||
|
||||
if options is None:
|
||||
options = 0
|
||||
# SSLv2 is easily broken and is considered harmful and dangerous
|
||||
options |= OP_NO_SSLv2
|
||||
# SSLv3 has several problems and is now dangerous
|
||||
options |= OP_NO_SSLv3
|
||||
# Disable compression to prevent CRIME attacks for OpenSSL 1.0+
|
||||
# (issue #309)
|
||||
options |= OP_NO_COMPRESSION
|
||||
|
||||
context.options |= options
|
||||
|
||||
if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6
|
||||
context.set_ciphers(ciphers or _DEFAULT_CIPHERS)
|
||||
|
||||
context.verify_mode = cert_reqs
|
||||
if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2
|
||||
context.check_hostname = (context.verify_mode == ssl.CERT_REQUIRED)
|
||||
return context
|
||||
|
||||
|
||||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
|
||||
ca_certs=None, server_hostname=None,
|
||||
ssl_version=None, ciphers=None, ssl_context=None):
|
||||
"""
|
||||
All arguments except for server_hostname and ssl_context have the same
|
||||
meaning as they do when using :func:`ssl.wrap_socket`.
|
||||
|
||||
:param server_hostname:
|
||||
When SNI is supported, the expected hostname of the certificate
|
||||
:param ssl_context:
|
||||
A pre-made :class:`SSLContext` object. If none is provided, one will
|
||||
be created using :func:`create_urllib3_context`.
|
||||
:param ciphers:
|
||||
A string of ciphers we wish the client to support. This is not
|
||||
supported on Python 2.6 as the ssl module does not support it.
|
||||
"""
|
||||
context = ssl_context
|
||||
if context is None:
|
||||
context = create_urllib3_context(ssl_version, cert_reqs,
|
||||
ciphers=ciphers)
|
||||
|
||||
if ca_certs:
|
||||
try:
|
||||
context.load_verify_locations(ca_certs)
|
||||
except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2
|
||||
raise SSLError(e)
|
||||
# Py33 raises FileNotFoundError which subclasses OSError
|
||||
# These are not equivalent unless we check the errno attribute
|
||||
except OSError as e: # Platform-specific: Python 3.3 and beyond
|
||||
if e.errno == errno.ENOENT:
|
||||
raise SSLError(e)
|
||||
if certfile:
|
||||
# FIXME: This block needs a test.
|
||||
context.load_cert_chain(certfile, keyfile)
|
||||
if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI
|
||||
return context.wrap_socket(sock, server_hostname=server_hostname)
|
||||
return context.wrap_socket(sock)
|
||||
|
||||
else: # Python 3.1 and earlier
|
||||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
|
||||
ca_certs=None, server_hostname=None,
|
||||
ssl_version=None):
|
||||
return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
|
||||
ca_certs=ca_certs, cert_reqs=cert_reqs,
|
||||
ssl_version=ssl_version)
|
||||
raise
|
||||
if certfile:
|
||||
context.load_cert_chain(certfile, keyfile)
|
||||
if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI
|
||||
return context.wrap_socket(sock, server_hostname=server_hostname)
|
||||
return context.wrap_socket(sock)
|
||||
|
|
|
@ -40,6 +40,48 @@ class Url(namedtuple('Url', url_attrs)):
|
|||
return '%s:%d' % (self.host, self.port)
|
||||
return self.host
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
"""
|
||||
Convert self into a url
|
||||
|
||||
This function should more or less round-trip with :func:`.parse_url`. The
|
||||
returned url may not be exactly the same as the url inputted to
|
||||
:func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
|
||||
with a blank port will have : removed).
|
||||
|
||||
Example: ::
|
||||
|
||||
>>> U = parse_url('http://google.com/mail/')
|
||||
>>> U.url
|
||||
'http://google.com/mail/'
|
||||
>>> Url('http', 'username:password', 'host.com', 80,
|
||||
... '/path', 'query', 'fragment').url
|
||||
'http://username:password@host.com:80/path?query#fragment'
|
||||
"""
|
||||
scheme, auth, host, port, path, query, fragment = self
|
||||
url = ''
|
||||
|
||||
# We use "is not None" we want things to happen with empty strings (or 0 port)
|
||||
if scheme is not None:
|
||||
url += scheme + '://'
|
||||
if auth is not None:
|
||||
url += auth + '@'
|
||||
if host is not None:
|
||||
url += host
|
||||
if port is not None:
|
||||
url += ':' + str(port)
|
||||
if path is not None:
|
||||
url += path
|
||||
if query is not None:
|
||||
url += '?' + query
|
||||
if fragment is not None:
|
||||
url += '#' + fragment
|
||||
|
||||
return url
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
|
||||
def split_first(s, delims):
|
||||
"""
|
||||
|
@ -84,7 +126,7 @@ def parse_url(url):
|
|||
Example::
|
||||
|
||||
>>> parse_url('http://google.com/mail/')
|
||||
Url(scheme='http', host='google.com', port=None, path='/', ...)
|
||||
Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
|
||||
>>> parse_url('google.com:80')
|
||||
Url(scheme=None, host='google.com', port=80, path=None, ...)
|
||||
>>> parse_url('/foo?bar')
|
||||
|
@ -162,7 +204,6 @@ def parse_url(url):
|
|||
|
||||
return Url(scheme, auth, host, port, path, query, fragment)
|
||||
|
||||
|
||||
def get_host(url):
|
||||
"""
|
||||
Deprecated. Use :func:`.parse_url` instead.
|
||||
|
|
|
@ -13,7 +13,7 @@ from collections import Mapping
|
|||
from datetime import datetime
|
||||
|
||||
from .auth import _basic_auth_str
|
||||
from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str
|
||||
from .compat import cookielib, OrderedDict, urljoin, urlparse
|
||||
from .cookies import (
|
||||
cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
|
||||
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
|
||||
|
@ -21,6 +21,7 @@ from .hooks import default_hooks, dispatch_hook
|
|||
from .utils import to_key_val_list, default_headers, to_native_string
|
||||
from .exceptions import (
|
||||
TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
|
||||
from .packages.urllib3._collections import RecentlyUsedContainer
|
||||
from .structures import CaseInsensitiveDict
|
||||
|
||||
from .adapters import HTTPAdapter
|
||||
|
@ -35,6 +36,8 @@ from .status_codes import codes
|
|||
# formerly defined here, reexposed here for backward compatibility
|
||||
from .models import REDIRECT_STATI
|
||||
|
||||
REDIRECT_CACHE_SIZE = 1000
|
||||
|
||||
|
||||
def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
|
||||
"""
|
||||
|
@ -128,7 +131,7 @@ class SessionRedirectMixin(object):
|
|||
# Facilitate relative 'location' headers, as allowed by RFC 7231.
|
||||
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
|
||||
# Compliant with RFC3986, we percent encode the url.
|
||||
if not urlparse(url).netloc:
|
||||
if not parsed.netloc:
|
||||
url = urljoin(resp.url, requote_uri(url))
|
||||
else:
|
||||
url = requote_uri(url)
|
||||
|
@ -271,9 +274,10 @@ class Session(SessionRedirectMixin):
|
|||
"""
|
||||
|
||||
__attrs__ = [
|
||||
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
|
||||
'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream',
|
||||
'trust_env', 'max_redirects', 'redirect_cache']
|
||||
'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
|
||||
'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
|
||||
'max_redirects',
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
|
||||
|
@ -326,7 +330,8 @@ class Session(SessionRedirectMixin):
|
|||
self.mount('https://', HTTPAdapter())
|
||||
self.mount('http://', HTTPAdapter())
|
||||
|
||||
self.redirect_cache = {}
|
||||
# Only store 1000 redirects to prevent using infinite memory
|
||||
self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
@ -365,6 +370,7 @@ class Session(SessionRedirectMixin):
|
|||
url=request.url,
|
||||
files=request.files,
|
||||
data=request.data,
|
||||
json=request.json,
|
||||
headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
|
||||
params=merge_setting(request.params, self.params),
|
||||
auth=merge_setting(auth, self.auth),
|
||||
|
@ -386,7 +392,8 @@ class Session(SessionRedirectMixin):
|
|||
hooks=None,
|
||||
stream=None,
|
||||
verify=None,
|
||||
cert=None):
|
||||
cert=None,
|
||||
json=None):
|
||||
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
|
||||
Returns :class:`Response <Response>` object.
|
||||
|
||||
|
@ -396,6 +403,8 @@ class Session(SessionRedirectMixin):
|
|||
string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the
|
||||
:class:`Request`.
|
||||
:param json: (optional) json to send in the body of the
|
||||
:class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the
|
||||
:class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the
|
||||
|
@ -420,7 +429,7 @@ class Session(SessionRedirectMixin):
|
|||
If Tuple, ('cert', 'key') pair.
|
||||
"""
|
||||
|
||||
method = builtin_str(method)
|
||||
method = to_native_string(method)
|
||||
|
||||
# Create the Request.
|
||||
req = Request(
|
||||
|
@ -429,6 +438,7 @@ class Session(SessionRedirectMixin):
|
|||
headers = headers,
|
||||
files = files,
|
||||
data = data or {},
|
||||
json = json,
|
||||
params = params or {},
|
||||
auth = auth,
|
||||
cookies = cookies,
|
||||
|
@ -482,15 +492,16 @@ class Session(SessionRedirectMixin):
|
|||
kwargs.setdefault('allow_redirects', False)
|
||||
return self.request('HEAD', url, **kwargs)
|
||||
|
||||
def post(self, url, data=None, **kwargs):
|
||||
def post(self, url, data=None, json=None, **kwargs):
|
||||
"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return self.request('POST', url, data=data, **kwargs)
|
||||
return self.request('POST', url, data=data, json=json, **kwargs)
|
||||
|
||||
def put(self, url, data=None, **kwargs):
|
||||
"""Sends a PUT request. Returns :class:`Response` object.
|
||||
|
@ -535,8 +546,13 @@ class Session(SessionRedirectMixin):
|
|||
if not isinstance(request, PreparedRequest):
|
||||
raise ValueError('You can only send PreparedRequests.')
|
||||
|
||||
checked_urls = set()
|
||||
while request.url in self.redirect_cache:
|
||||
request.url = self.redirect_cache.get(request.url)
|
||||
checked_urls.add(request.url)
|
||||
new_url = self.redirect_cache.get(request.url)
|
||||
if new_url in checked_urls:
|
||||
break
|
||||
request.url = new_url
|
||||
|
||||
# Set up variables needed for resolve_redirects and dispatching of hooks
|
||||
allow_redirects = kwargs.pop('allow_redirects', True)
|
||||
|
@ -646,12 +662,19 @@ class Session(SessionRedirectMixin):
|
|||
self.adapters[key] = self.adapters.pop(key)
|
||||
|
||||
def __getstate__(self):
|
||||
return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
|
||||
state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
|
||||
state['redirect_cache'] = dict(self.redirect_cache)
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
redirect_cache = state.pop('redirect_cache', {})
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
|
||||
for redirect, to in redirect_cache.items():
|
||||
self.redirect_cache[redirect] = to
|
||||
|
||||
|
||||
def session():
|
||||
"""Returns a :class:`Session` for context-management."""
|
||||
|
|
|
@ -19,6 +19,7 @@ import re
|
|||
import sys
|
||||
import socket
|
||||
import struct
|
||||
import warnings
|
||||
|
||||
from . import __version__
|
||||
from . import certs
|
||||
|
@ -287,6 +288,11 @@ def get_encodings_from_content(content):
|
|||
|
||||
:param content: bytestring to extract encodings from.
|
||||
"""
|
||||
warnings.warn((
|
||||
'In requests 3.0, get_encodings_from_content will be removed. For '
|
||||
'more information, please see the discussion on issue #2266. (This'
|
||||
' warning should only appear once.)'),
|
||||
DeprecationWarning)
|
||||
|
||||
charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
|
||||
pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
|
||||
|
@ -351,12 +357,14 @@ def get_unicode_from_response(r):
|
|||
Tried:
|
||||
|
||||
1. charset from content-type
|
||||
|
||||
2. every encodings from ``<meta ... charset=XXX>``
|
||||
|
||||
3. fall back and replace all unicode characters
|
||||
2. fall back and replace all unicode characters
|
||||
|
||||
"""
|
||||
warnings.warn((
|
||||
'In requests 3.0, get_unicode_from_response will be removed. For '
|
||||
'more information, please see the discussion on issue #2266. (This'
|
||||
' warning should only appear once.)'),
|
||||
DeprecationWarning)
|
||||
|
||||
tried_encodings = []
|
||||
|
||||
|
@ -570,7 +578,7 @@ def parse_header_links(value):
|
|||
|
||||
replace_chars = " '\""
|
||||
|
||||
for val in value.split(","):
|
||||
for val in re.split(", *<", value):
|
||||
try:
|
||||
url, params = val.split(";", 1)
|
||||
except ValueError:
|
||||
|
@ -672,3 +680,18 @@ def to_native_string(string, encoding='ascii'):
|
|||
out = string.decode(encoding)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def urldefragauth(url):
|
||||
"""
|
||||
Given a url remove the fragment and the authentication part
|
||||
"""
|
||||
scheme, netloc, path, params, query, fragment = urlparse(url)
|
||||
|
||||
# see func:`prepend_scheme_if_needed`
|
||||
if not netloc:
|
||||
netloc, path = path, netloc
|
||||
|
||||
netloc = netloc.rsplit('@', 1)[-1]
|
||||
|
||||
return urlunparse((scheme, netloc, path, params, query, ''))
|
||||
|
|
|
@ -2,7 +2,7 @@ distlib==0.1.9
|
|||
html5lib==0.999 # See: https://github.com/html5lib/html5lib-python/issues/161
|
||||
six==1.8.0
|
||||
colorama==0.3.2
|
||||
requests==2.4.1
|
||||
requests==2.5.0
|
||||
certifi==14.05.14
|
||||
CacheControl==0.10.4
|
||||
lockfile==0.9.1
|
||||
|
|
|
@ -46,8 +46,13 @@ def test_pip_version_check(monkeypatch, stored_time, newver, check, warn):
|
|||
monkeypatch.setattr(outdated.logger, 'debug',
|
||||
pretend.call_recorder(lambda s, exc_info=None: None))
|
||||
|
||||
with freezegun.freeze_time("1970-01-09 10:00:00",
|
||||
ignore=["pip._vendor.six.moves"]):
|
||||
with freezegun.freeze_time(
|
||||
"1970-01-09 10:00:00",
|
||||
ignore=[
|
||||
"six.moves",
|
||||
"pip._vendor.six.moves",
|
||||
"pip._vendor.requests.packages.urllib3.packages.six.moves",
|
||||
]):
|
||||
outdated.pip_version_check(session)
|
||||
|
||||
assert not outdated.logger.debug.calls
|
||||
|
|
Loading…
Reference in a new issue