From 8078d9948068f13e1af2b99a1b2b324c4c941650 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Wed, 16 May 2018 17:42:52 -0400 Subject: [PATCH 001/129] fix un-vendored support; add missing entry for urllib3 --- src/pip/_vendor/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 1387dba3e..a0aae81ae 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -107,3 +107,4 @@ if DEBUNDLED: vendored("requests.packages.urllib3.util.ssl_") vendored("requests.packages.urllib3.util.timeout") vendored("requests.packages.urllib3.util.url") + vendored("urllib3") From ecc7e574dd0f70845a8a2c8189949065f743f185 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:26:56 +0530 Subject: [PATCH 002/129] Update distro to 1.3.0 --- news/distro.vendor | 1 + src/pip/_vendor/distro.py | 187 +++++++++++++++++++++++++++---------- src/pip/_vendor/vendor.txt | 2 +- 3 files changed, 142 insertions(+), 48 deletions(-) create mode 100644 news/distro.vendor diff --git a/news/distro.vendor b/news/distro.vendor new file mode 100644 index 000000000..a21167d38 --- /dev/null +++ b/news/distro.vendor @@ -0,0 +1 @@ +Update distro to 1.3.0 diff --git a/src/pip/_vendor/distro.py b/src/pip/_vendor/distro.py index 39bfce797..aa4defc3b 100644 --- a/src/pip/_vendor/distro.py +++ b/src/pip/_vendor/distro.py @@ -1,4 +1,4 @@ -# Copyright 2015,2016 Nir Cohen +# Copyright 2015,2016,2017 Nir Cohen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ functionality. An alternative implementation became necessary because Python 3.5 deprecated this function, and Python 3.7 is expected to remove it altogether. Its predecessor function :py:func:`platform.dist` was already deprecated since Python 2.6 and is also expected to be removed in Python 3.7. -Still, there are many cases in which access to Linux distribution information +Still, there are many cases in which access to OS distribution information is needed. See `Python issue 1322 `_ for more information. """ @@ -94,7 +94,7 @@ _DISTRO_RELEASE_IGNORE_BASENAMES = ( def linux_distribution(full_distribution_name=True): """ - Return information about the current Linux distribution as a tuple + Return information about the current OS distribution as a tuple ``(id_name, version, codename)`` with items as follows: * ``id_name``: If *full_distribution_name* is false, the result of @@ -110,22 +110,22 @@ def linux_distribution(full_distribution_name=True): The data it returns may not exactly be the same, because it uses more data sources than the original function, and that may lead to different data if - the Linux distribution is not consistent across multiple data sources it + the OS distribution is not consistent across multiple data sources it provides (there are indeed such distributions ...). Another reason for differences is the fact that the :func:`distro.id` method normalizes the distro ID string to a reliable machine-readable value - for a number of popular Linux distributions. + for a number of popular OS distributions. """ return _distro.linux_distribution(full_distribution_name) def id(): """ - Return the distro ID of the current Linux distribution, as a + Return the distro ID of the current distribution, as a machine-readable string. - For a number of Linux distributions, the returned distro ID value is + For a number of OS distributions, the returned distro ID value is *reliable*, in the sense that it is documented and that it does not change across releases of the distribution. @@ -158,6 +158,9 @@ def id(): "scientific" Scientific Linux "slackware" Slackware "xenserver" XenServer + "openbsd" OpenBSD + "netbsd" NetBSD + "freebsd" FreeBSD ============== ========================================= If you have a need to get distros for reliable IDs added into this set, @@ -187,7 +190,7 @@ def id(): * a normalization of the ID is performed, based upon `normalization tables`_. The purpose of this normalization is to ensure that the ID is as reliable as possible, even across incompatible changes - in the Linux distributions. A common reason for an incompatible change is + in the OS distributions. A common reason for an incompatible change is the addition of an os-release file, or the addition of the lsb_release command, with ID values that differ from what was previously determined from the distro release file name. @@ -197,7 +200,7 @@ def id(): def name(pretty=False): """ - Return the name of the current Linux distribution, as a human-readable + Return the name of the current OS distribution, as a human-readable string. If *pretty* is false, the name is returned without version or codename. @@ -236,7 +239,7 @@ def name(pretty=False): def version(pretty=False, best=False): """ - Return the version of the current Linux distribution, as a human-readable + Return the version of the current OS distribution, as a human-readable string. If *pretty* is false, the version is returned without codename (e.g. @@ -280,7 +283,7 @@ def version(pretty=False, best=False): def version_parts(best=False): """ - Return the version of the current Linux distribution as a tuple + Return the version of the current OS distribution as a tuple ``(major, minor, build_number)`` with items as follows: * ``major``: The result of :func:`distro.major_version`. @@ -297,7 +300,7 @@ def version_parts(best=False): def major_version(best=False): """ - Return the major version of the current Linux distribution, as a string, + Return the major version of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The major version is the first part of the dot-separated version string. @@ -310,7 +313,7 @@ def major_version(best=False): def minor_version(best=False): """ - Return the minor version of the current Linux distribution, as a string, + Return the minor version of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The minor version is the second part of the dot-separated version string. @@ -323,7 +326,7 @@ def minor_version(best=False): def build_number(best=False): """ - Return the build number of the current Linux distribution, as a string, + Return the build number of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The build number is the third part of the dot-separated version string. @@ -337,7 +340,7 @@ def build_number(best=False): def like(): """ Return a space-separated list of distro IDs of distributions that are - closely related to the current Linux distribution in regards to packaging + closely related to the current OS distribution in regards to packaging and programming interfaces, for example distributions the current distribution is a derivative from. @@ -353,7 +356,7 @@ def like(): def codename(): """ - Return the codename for the release of the current Linux distribution, + Return the codename for the release of the current OS distribution, as a string. If the distribution does not have a codename, an empty string is returned. @@ -377,7 +380,7 @@ def codename(): def info(pretty=False, best=False): """ - Return certain machine-readable information items about the current Linux + Return certain machine-readable information items about the current OS distribution in a dictionary, as shown in the following example: .. sourcecode:: python @@ -422,7 +425,7 @@ def info(pretty=False, best=False): def os_release_info(): """ Return a dictionary containing key-value pairs for the information items - from the os-release file data source of the current Linux distribution. + from the os-release file data source of the current OS distribution. See `os-release file`_ for details about these information items. """ @@ -432,7 +435,7 @@ def os_release_info(): def lsb_release_info(): """ Return a dictionary containing key-value pairs for the information items - from the lsb_release command data source of the current Linux distribution. + from the lsb_release command data source of the current OS distribution. See `lsb_release command output`_ for details about these information items. @@ -443,17 +446,25 @@ def lsb_release_info(): def distro_release_info(): """ Return a dictionary containing key-value pairs for the information items - from the distro release file data source of the current Linux distribution. + from the distro release file data source of the current OS distribution. See `distro release file`_ for details about these information items. """ return _distro.distro_release_info() +def uname_info(): + """ + Return a dictionary containing key-value pairs for the information items + from the distro release file data source of the current OS distribution. + """ + return _distro.uname_info() + + def os_release_attr(attribute): """ Return a single named information item from the os-release file data source - of the current Linux distribution. + of the current OS distribution. Parameters: @@ -472,7 +483,7 @@ def os_release_attr(attribute): def lsb_release_attr(attribute): """ Return a single named information item from the lsb_release command output - data source of the current Linux distribution. + data source of the current OS distribution. Parameters: @@ -492,7 +503,7 @@ def lsb_release_attr(attribute): def distro_release_attr(attribute): """ Return a single named information item from the distro release file - data source of the current Linux distribution. + data source of the current OS distribution. Parameters: @@ -508,6 +519,23 @@ def distro_release_attr(attribute): return _distro.distro_release_attr(attribute) +def uname_attr(attribute): + """ + Return a single named information item from the distro release file + data source of the current OS distribution. + + Parameters: + + * ``attribute`` (string): Key of the information item. + + Returns: + + * (string): Value of the information item, if the item exists. + The empty string, if the item does not exist. + """ + return _distro.uname_attr(attribute) + + class cached_property(object): """A version of @property which caches the value. On access, it calls the underlying function and sets the value in `__dict__` so future accesses @@ -525,13 +553,13 @@ class cached_property(object): class LinuxDistribution(object): """ - Provides information about a Linux distribution. + Provides information about a OS distribution. This package creates a private module-global instance of this class with default initialization arguments, that is used by the `consolidated accessor functions`_ and `single source accessor functions`_. By using default initialization arguments, that module-global instance - returns data about the current Linux distribution (i.e. the distro this + returns data about the current OS distribution (i.e. the distro this package runs on). Normally, it is not necessary to create additional instances of this class. @@ -544,7 +572,8 @@ class LinuxDistribution(object): def __init__(self, include_lsb=True, os_release_file='', - distro_release_file=''): + distro_release_file='', + include_uname=True): """ The initialization method of this class gathers information from the available data sources, and stores that in private instance attributes. @@ -578,6 +607,11 @@ class LinuxDistribution(object): distro release file can be found, the data source for the distro release file will be empty. + * ``include_name`` (bool): Controls whether uname command output is + included as a data source. If the uname command is not available in + the program execution path the data source for the uname command will + be empty. + Public instance attributes: * ``os_release_file`` (string): The path name of the @@ -591,6 +625,10 @@ class LinuxDistribution(object): * ``include_lsb`` (bool): The result of the ``include_lsb`` parameter. This controls whether the lsb information will be loaded. + * ``include_uname`` (bool): The result of the ``include_uname`` + parameter. This controls whether the uname information will + be loaded. + Raises: * :py:exc:`IOError`: Some I/O issue with an os-release file or distro @@ -607,6 +645,7 @@ class LinuxDistribution(object): os.path.join(_UNIXCONFDIR, _OS_RELEASE_BASENAME) self.distro_release_file = distro_release_file or '' # updated later self.include_lsb = include_lsb + self.include_uname = include_uname def __repr__(self): """Return repr of all info @@ -616,14 +655,16 @@ class LinuxDistribution(object): "os_release_file={self.os_release_file!r}, " \ "distro_release_file={self.distro_release_file!r}, " \ "include_lsb={self.include_lsb!r}, " \ + "include_uname={self.include_uname!r}, " \ "_os_release_info={self._os_release_info!r}, " \ "_lsb_release_info={self._lsb_release_info!r}, " \ - "_distro_release_info={self._distro_release_info!r})".format( + "_distro_release_info={self._distro_release_info!r}, " \ + "_uname_info={self._uname_info!r})".format( self=self) def linux_distribution(self, full_distribution_name=True): """ - Return information about the Linux distribution that is compatible + Return information about the OS distribution that is compatible with Python's :func:`platform.linux_distribution`, supporting a subset of its parameters. @@ -636,7 +677,7 @@ class LinuxDistribution(object): ) def id(self): - """Return the distro ID of the Linux distribution, as a string. + """Return the distro ID of the OS distribution, as a string. For details, see :func:`distro.id`. """ @@ -656,22 +697,28 @@ class LinuxDistribution(object): if distro_id: return normalize(distro_id, NORMALIZED_DISTRO_ID) + distro_id = self.uname_attr('id') + if distro_id: + return normalize(distro_id, NORMALIZED_DISTRO_ID) + return '' def name(self, pretty=False): """ - Return the name of the Linux distribution, as a string. + Return the name of the OS distribution, as a string. For details, see :func:`distro.name`. """ name = self.os_release_attr('name') \ or self.lsb_release_attr('distributor_id') \ - or self.distro_release_attr('name') + or self.distro_release_attr('name') \ + or self.uname_attr('name') if pretty: name = self.os_release_attr('pretty_name') \ or self.lsb_release_attr('description') if not name: - name = self.distro_release_attr('name') + name = self.distro_release_attr('name') \ + or self.uname_attr('name') version = self.version(pretty=True) if version: name = name + ' ' + version @@ -679,7 +726,7 @@ class LinuxDistribution(object): def version(self, pretty=False, best=False): """ - Return the version of the Linux distribution, as a string. + Return the version of the OS distribution, as a string. For details, see :func:`distro.version`. """ @@ -690,7 +737,8 @@ class LinuxDistribution(object): self._parse_distro_release_content( self.os_release_attr('pretty_name')).get('version_id', ''), self._parse_distro_release_content( - self.lsb_release_attr('description')).get('version_id', '') + self.lsb_release_attr('description')).get('version_id', ''), + self.uname_attr('release') ] version = '' if best: @@ -712,7 +760,7 @@ class LinuxDistribution(object): def version_parts(self, best=False): """ - Return the version of the Linux distribution, as a tuple of version + Return the version of the OS distribution, as a tuple of version numbers. For details, see :func:`distro.version_parts`. @@ -736,7 +784,7 @@ class LinuxDistribution(object): def minor_version(self, best=False): """ - Return the minor version number of the Linux distribution. + Return the minor version number of the current distribution. For details, see :func:`distro.minor_version`. """ @@ -744,7 +792,7 @@ class LinuxDistribution(object): def build_number(self, best=False): """ - Return the build number of the Linux distribution. + Return the build number of the current distribution. For details, see :func:`distro.build_number`. """ @@ -752,7 +800,7 @@ class LinuxDistribution(object): def like(self): """ - Return the IDs of distributions that are like the Linux distribution. + Return the IDs of distributions that are like the OS distribution. For details, see :func:`distro.like`. """ @@ -760,7 +808,7 @@ class LinuxDistribution(object): def codename(self): """ - Return the codename of the Linux distribution. + Return the codename of the OS distribution. For details, see :func:`distro.codename`. """ @@ -771,7 +819,7 @@ class LinuxDistribution(object): def info(self, pretty=False, best=False): """ - Return certain machine-readable information about the Linux + Return certain machine-readable information about the OS distribution. For details, see :func:`distro.info`. @@ -791,7 +839,7 @@ class LinuxDistribution(object): def os_release_info(self): """ Return a dictionary containing key-value pairs for the information - items from the os-release file data source of the Linux distribution. + items from the os-release file data source of the OS distribution. For details, see :func:`distro.os_release_info`. """ @@ -800,7 +848,7 @@ class LinuxDistribution(object): def lsb_release_info(self): """ Return a dictionary containing key-value pairs for the information - items from the lsb_release command data source of the Linux + items from the lsb_release command data source of the OS distribution. For details, see :func:`distro.lsb_release_info`. @@ -810,17 +858,25 @@ class LinuxDistribution(object): def distro_release_info(self): """ Return a dictionary containing key-value pairs for the information - items from the distro release file data source of the Linux + items from the distro release file data source of the OS distribution. For details, see :func:`distro.distro_release_info`. """ return self._distro_release_info + def uname_info(self): + """ + Return a dictionary containing key-value pairs for the information + items from the uname command data source of the OS distribution. + + For details, see :func:`distro.uname_info`. + """ + def os_release_attr(self, attribute): """ Return a single named information item from the os-release file data - source of the Linux distribution. + source of the OS distribution. For details, see :func:`distro.os_release_attr`. """ @@ -829,7 +885,7 @@ class LinuxDistribution(object): def lsb_release_attr(self, attribute): """ Return a single named information item from the lsb_release command - output data source of the Linux distribution. + output data source of the OS distribution. For details, see :func:`distro.lsb_release_attr`. """ @@ -838,12 +894,21 @@ class LinuxDistribution(object): def distro_release_attr(self, attribute): """ Return a single named information item from the distro release file - data source of the Linux distribution. + data source of the OS distribution. For details, see :func:`distro.distro_release_attr`. """ return self._distro_release_info.get(attribute, '') + def uname_attr(self, attribute): + """ + Return a single named information item from the uname command + output data source of the OS distribution. + + For details, see :func:`distro.uname_release_attr`. + """ + return self._uname_info.get(attribute, '') + @cached_property def _os_release_info(self): """ @@ -960,6 +1025,34 @@ class LinuxDistribution(object): props.update({k.replace(' ', '_').lower(): v.strip()}) return props + @cached_property + def _uname_info(self): + with open(os.devnull, 'w') as devnull: + try: + cmd = ('uname', '-rs') + stdout = subprocess.check_output(cmd, stderr=devnull) + except OSError: + return {} + content = stdout.decode(sys.getfilesystemencoding()).splitlines() + return self._parse_uname_content(content) + + @staticmethod + def _parse_uname_content(lines): + props = {} + match = re.search(r'^([^\s]+)\s+([\d\.]+)', lines[0].strip()) + if match: + name, version = match.groups() + + # This is to prevent the Linux kernel version from + # appearing as the 'best' version on otherwise + # identifiable distributions. + if name == 'Linux': + return {} + props['id'] = name.lower() + props['name'] = name + props['release'] = version + return props + @cached_property def _distro_release_info(self): """ @@ -1082,7 +1175,7 @@ def main(): logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) - parser = argparse.ArgumentParser(description="Linux distro info tool") + parser = argparse.ArgumentParser(description="OS distro info tool") parser.add_argument( '--json', '-j', diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 4a937e8d7..7fc9d118b 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,6 +1,6 @@ appdirs==1.4.3 distlib==0.2.7 -distro==1.2.0 +distro==1.3.0 html5lib==1.0.1 six==1.11.0 colorama==0.3.9 From 38fc85241749a89255719d8a8a973ee72f1bd4b1 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:29:26 +0530 Subject: [PATCH 003/129] Update CacheControl to 0.12.5 --- news/cachecontrol.vendor | 1 + src/pip/_vendor/cachecontrol/__init__.py | 6 +- src/pip/_vendor/cachecontrol/_cmd.py | 17 +- src/pip/_vendor/cachecontrol/adapter.py | 53 +++-- src/pip/_vendor/cachecontrol/cache.py | 6 +- .../_vendor/cachecontrol/caches/file_cache.py | 31 ++- .../cachecontrol/caches/redis_cache.py | 12 +- src/pip/_vendor/cachecontrol/controller.py | 206 +++++++++--------- src/pip/_vendor/cachecontrol/filewrapper.py | 6 +- src/pip/_vendor/cachecontrol/heuristics.py | 37 ++-- src/pip/_vendor/cachecontrol/serialize.py | 38 ++-- src/pip/_vendor/cachecontrol/wrapper.py | 24 +- src/pip/_vendor/vendor.txt | 2 +- 13 files changed, 213 insertions(+), 226 deletions(-) create mode 100644 news/cachecontrol.vendor diff --git a/news/cachecontrol.vendor b/news/cachecontrol.vendor new file mode 100644 index 000000000..1affedb83 --- /dev/null +++ b/news/cachecontrol.vendor @@ -0,0 +1 @@ +Update CacheControl to 0.12.5 diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py index f386d4929..8fdee66ff 100644 --- a/src/pip/_vendor/cachecontrol/__init__.py +++ b/src/pip/_vendor/cachecontrol/__init__.py @@ -2,9 +2,9 @@ Make it easy to import from cachecontrol without long namespaces. """ -__author__ = 'Eric Larson' -__email__ = 'eric@ionrock.org' -__version__ = '0.12.4' +__author__ = "Eric Larson" +__email__ = "eric@ionrock.org" +__version__ = "0.12.5" from .wrapper import CacheControl from .adapter import CacheControlAdapter diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py index afdcc88c2..f1e0ad94a 100644 --- a/src/pip/_vendor/cachecontrol/_cmd.py +++ b/src/pip/_vendor/cachecontrol/_cmd.py @@ -17,14 +17,11 @@ def setup_logging(): def get_session(): adapter = CacheControlAdapter( - DictCache(), - cache_etags=True, - serializer=None, - heuristic=None, + DictCache(), cache_etags=True, serializer=None, heuristic=None ) sess = requests.Session() - sess.mount('http://', adapter) - sess.mount('https://', adapter) + sess.mount("http://", adapter) + sess.mount("https://", adapter) sess.cache_controller = adapter.controller return sess @@ -32,7 +29,7 @@ def get_session(): def get_args(): parser = ArgumentParser() - parser.add_argument('url', help='The URL to try and cache') + parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() @@ -51,10 +48,10 @@ def main(args=None): # Now try to get it if sess.cache_controller.cached_request(resp.request): - print('Cached!') + print("Cached!") else: - print('Not cached :(') + print("Not cached :(") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py index ecb34a6db..780eb2883 100644 --- a/src/pip/_vendor/cachecontrol/adapter.py +++ b/src/pip/_vendor/cachecontrol/adapter.py @@ -10,25 +10,27 @@ from .filewrapper import CallbackFileWrapper class CacheControlAdapter(HTTPAdapter): - invalidating_methods = set(['PUT', 'DELETE']) + invalidating_methods = {"PUT", "DELETE"} - def __init__(self, cache=None, - cache_etags=True, - controller_class=None, - serializer=None, - heuristic=None, - cacheable_methods=None, - *args, **kw): + def __init__( + self, + cache=None, + cache_etags=True, + controller_class=None, + serializer=None, + heuristic=None, + cacheable_methods=None, + *args, + **kw + ): super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = cache or DictCache() self.heuristic = heuristic - self.cacheable_methods = cacheable_methods or ('GET',) + self.cacheable_methods = cacheable_methods or ("GET",) controller_factory = controller_class or CacheController self.controller = controller_factory( - self.cache, - cache_etags=cache_etags, - serializer=serializer, + self.cache, cache_etags=cache_etags, serializer=serializer ) def send(self, request, cacheable_methods=None, **kw): @@ -43,20 +45,18 @@ class CacheControlAdapter(HTTPAdapter): except zlib.error: cached_response = None if cached_response: - return self.build_response(request, cached_response, - from_cache=True) + return self.build_response(request, cached_response, from_cache=True) # check for etags and add headers if appropriate - request.headers.update( - self.controller.conditional_headers(request) - ) + request.headers.update(self.controller.conditional_headers(request)) resp = super(CacheControlAdapter, self).send(request, **kw) return resp - def build_response(self, request, response, from_cache=False, - cacheable_methods=None): + def build_response( + self, request, response, from_cache=False, cacheable_methods=None + ): """ Build a response by making a request or using the cache. @@ -101,10 +101,8 @@ class CacheControlAdapter(HTTPAdapter): response._fp = CallbackFileWrapper( response._fp, functools.partial( - self.controller.cache_response, - request, - response, - ) + self.controller.cache_response, request, response + ), ) if response.chunked: super_update_chunk_length = response._update_chunk_length @@ -113,11 +111,12 @@ class CacheControlAdapter(HTTPAdapter): super_update_chunk_length() if self.chunk_left == 0: self._fp._close() - response._update_chunk_length = types.MethodType(_update_chunk_length, response) - resp = super(CacheControlAdapter, self).build_response( - request, response - ) + response._update_chunk_length = types.MethodType( + _update_chunk_length, response + ) + + resp = super(CacheControlAdapter, self).build_response(request, response) # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py index 7389a73f8..94e07732d 100644 --- a/src/pip/_vendor/cachecontrol/cache.py +++ b/src/pip/_vendor/cachecontrol/cache.py @@ -8,13 +8,13 @@ from threading import Lock class BaseCache(object): def get(self, key): - raise NotImplemented() + raise NotImplementedError() def set(self, key, value): - raise NotImplemented() + raise NotImplementedError() def delete(self, key): - raise NotImplemented() + raise NotImplementedError() def close(self): pass diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py index 885c8a653..1ba00806c 100644 --- a/src/pip/_vendor/cachecontrol/caches/file_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py @@ -9,7 +9,7 @@ try: FileNotFoundError except NameError: # py2.X - FileNotFoundError = OSError + FileNotFoundError = (IOError, OSError) def _secure_open_write(filename, fmode): @@ -46,6 +46,7 @@ def _secure_open_write(filename, fmode): fd = os.open(filename, flags, fmode) try: return os.fdopen(fd, "wb") + except: # An error occurred wrapping our FD in a file object os.close(fd) @@ -53,8 +54,16 @@ def _secure_open_write(filename, fmode): class FileCache(BaseCache): - def __init__(self, directory, forever=False, filemode=0o0600, - dirmode=0o0700, use_dir_lock=None, lock_class=None): + + def __init__( + self, + directory, + forever=False, + filemode=0o0600, + dirmode=0o0700, + use_dir_lock=None, + lock_class=None, + ): if use_dir_lock is not None and lock_class is not None: raise ValueError("Cannot use use_dir_lock and lock_class together") @@ -63,12 +72,15 @@ class FileCache(BaseCache): from pip._vendor.lockfile import LockFile from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile except ImportError: - notice = dedent(""" + notice = dedent( + """ NOTE: In order to use the FileCache you must have lockfile installed. You can install it via pip: pip install lockfile - """) + """ + ) raise ImportError(notice) + else: if use_dir_lock: lock_class = MkdirLockFile @@ -95,11 +107,12 @@ class FileCache(BaseCache): def get(self, key): name = self._fn(key) - if not os.path.exists(name): - return None + try: + with open(name, "rb") as fh: + return fh.read() - with open(name, 'rb') as fh: - return fh.read() + except FileNotFoundError: + return None def set(self, key, value): name = self._fn(key) diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py index b6285e9f1..ed705ce7d 100644 --- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py @@ -4,16 +4,6 @@ from datetime import datetime from pip._vendor.cachecontrol.cache import BaseCache -def total_seconds(td): - """Python 2.6 compatability""" - if hasattr(td, 'total_seconds'): - return int(td.total_seconds()) - - ms = td.microseconds - secs = (td.seconds + td.days * 24 * 3600) - return int((ms + secs * 10**6) / 10**6) - - class RedisCache(BaseCache): def __init__(self, conn): @@ -27,7 +17,7 @@ class RedisCache(BaseCache): self.conn.set(key, value) else: expires = expires - datetime.utcnow() - self.conn.setex(key, total_seconds(expires), value) + self.conn.setex(key, int(expires.total_seconds()), value) def delete(self, key): self.conn.delete(key) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 0e2eb3c4f..1b2b943cb 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -30,8 +30,10 @@ def parse_uri(uri): class CacheController(object): """An interface to see if request should cached or not. """ - def __init__(self, cache=None, cache_etags=True, serializer=None, - status_codes=None): + + def __init__( + self, cache=None, cache_etags=True, serializer=None, status_codes=None + ): self.cache = cache or DictCache() self.cache_etags = cache_etags self.serializer = serializer or Serializer() @@ -64,34 +66,35 @@ class CacheController(object): def parse_cache_control(self, headers): known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 - 'max-age': (int, True,), - 'max-stale': (int, False,), - 'min-fresh': (int, True,), - 'no-cache': (None, False,), - 'no-store': (None, False,), - 'no-transform': (None, False,), - 'only-if-cached' : (None, False,), - 'must-revalidate': (None, False,), - 'public': (None, False,), - 'private': (None, False,), - 'proxy-revalidate': (None, False,), - 's-maxage': (int, True,) + "max-age": (int, True), + "max-stale": (int, False), + "min-fresh": (int, True), + "no-cache": (None, False), + "no-store": (None, False), + "no-transform": (None, False), + "only-if-cached": (None, False), + "must-revalidate": (None, False), + "public": (None, False), + "private": (None, False), + "proxy-revalidate": (None, False), + "s-maxage": (int, True), } - cc_headers = headers.get('cache-control', - headers.get('Cache-Control', '')) + cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) retval = {} - for cc_directive in cc_headers.split(','): - parts = cc_directive.split('=', 1) + for cc_directive in cc_headers.split(","): + if not cc_directive.strip(): + continue + + parts = cc_directive.split("=", 1) directive = parts[0].strip() try: typ, required = known_directives[directive] except KeyError: - logger.debug('Ignoring unknown cache-control directive: %s', - directive) + logger.debug("Ignoring unknown cache-control directive: %s", directive) continue if not typ or not required: @@ -101,11 +104,16 @@ class CacheController(object): retval[directive] = typ(parts[1].strip()) except IndexError: if required: - logger.debug('Missing value for cache-control ' - 'directive: %s', directive) + logger.debug( + "Missing value for cache-control " "directive: %s", + directive, + ) except ValueError: - logger.debug('Invalid value for cache-control directive ' - '%s, must be %s', directive, typ.__name__) + logger.debug( + "Invalid value for cache-control directive " "%s, must be %s", + directive, + typ.__name__, + ) return retval @@ -119,24 +127,24 @@ class CacheController(object): cc = self.parse_cache_control(request.headers) # Bail out if the request insists on fresh data - if 'no-cache' in cc: + if "no-cache" in cc: logger.debug('Request header has "no-cache", cache bypassed') return False - if 'max-age' in cc and cc['max-age'] == 0: + if "max-age" in cc and cc["max-age"] == 0: logger.debug('Request header has "max_age" as 0, cache bypassed') return False # Request allows serving from the cache, let's see if we find something cache_data = self.cache.get(cache_url) if cache_data is None: - logger.debug('No cache entry available') + logger.debug("No cache entry available") return False # Check whether it can be deserialized resp = self.serializer.loads(request, cache_data) if not resp: - logger.warning('Cache entry deserialization failed, entry ignored') + logger.warning("Cache entry deserialization failed, entry ignored") return False # If we have a cached 301, return it immediately. We don't @@ -148,27 +156,27 @@ class CacheController(object): # Client can try to refresh the value by repeating the request # with cache busting headers as usual (ie no-cache). if resp.status == 301: - msg = ('Returning cached "301 Moved Permanently" response ' - '(ignoring date and etag information)') + msg = ( + 'Returning cached "301 Moved Permanently" response ' + "(ignoring date and etag information)" + ) logger.debug(msg) return resp headers = CaseInsensitiveDict(resp.headers) - if not headers or 'date' not in headers: - if 'etag' not in headers: + if not headers or "date" not in headers: + if "etag" not in headers: # Without date or etag, the cached response can never be used # and should be deleted. - logger.debug('Purging cached response: no date or etag') + logger.debug("Purging cached response: no date or etag") self.cache.delete(cache_url) - logger.debug('Ignoring cached response: no date') + logger.debug("Ignoring cached response: no date") return False now = time.time() - date = calendar.timegm( - parsedate_tz(headers['date']) - ) + date = calendar.timegm(parsedate_tz(headers["date"])) current_age = max(0, now - date) - logger.debug('Current age based on date: %i', current_age) + logger.debug("Current age based on date: %i", current_age) # TODO: There is an assumption that the result will be a # urllib3 response object. This may not be best since we @@ -180,45 +188,41 @@ class CacheController(object): freshness_lifetime = 0 # Check the max-age pragma in the cache control header - if 'max-age' in resp_cc: - freshness_lifetime = resp_cc['max-age'] - logger.debug('Freshness lifetime from max-age: %i', - freshness_lifetime) + if "max-age" in resp_cc: + freshness_lifetime = resp_cc["max-age"] + logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime) # If there isn't a max-age, check for an expires header - elif 'expires' in headers: - expires = parsedate_tz(headers['expires']) + elif "expires" in headers: + expires = parsedate_tz(headers["expires"]) if expires is not None: expire_time = calendar.timegm(expires) - date freshness_lifetime = max(0, expire_time) - logger.debug("Freshness lifetime from expires: %i", - freshness_lifetime) + logger.debug("Freshness lifetime from expires: %i", freshness_lifetime) # Determine if we are setting freshness limit in the # request. Note, this overrides what was in the response. - if 'max-age' in cc: - freshness_lifetime = cc['max-age'] - logger.debug('Freshness lifetime from request max-age: %i', - freshness_lifetime) + if "max-age" in cc: + freshness_lifetime = cc["max-age"] + logger.debug( + "Freshness lifetime from request max-age: %i", freshness_lifetime + ) - if 'min-fresh' in cc: - min_fresh = cc['min-fresh'] + if "min-fresh" in cc: + min_fresh = cc["min-fresh"] # adjust our current age by our min fresh current_age += min_fresh - logger.debug('Adjusted current age from min-fresh: %i', - current_age) + logger.debug("Adjusted current age from min-fresh: %i", current_age) # Return entry if it is fresh enough if freshness_lifetime > current_age: logger.debug('The response is "fresh", returning cached response') - logger.debug('%i > %i', freshness_lifetime, current_age) + logger.debug("%i > %i", freshness_lifetime, current_age) return resp # we're not fresh. If we don't have an Etag, clear it out - if 'etag' not in headers: - logger.debug( - 'The cached response is "stale" with no etag, purging' - ) + if "etag" not in headers: + logger.debug('The cached response is "stale" with no etag, purging') self.cache.delete(cache_url) # return the original handler @@ -232,16 +236,15 @@ class CacheController(object): if resp: headers = CaseInsensitiveDict(resp.headers) - if 'etag' in headers: - new_headers['If-None-Match'] = headers['ETag'] + if "etag" in headers: + new_headers["If-None-Match"] = headers["ETag"] - if 'last-modified' in headers: - new_headers['If-Modified-Since'] = headers['Last-Modified'] + if "last-modified" in headers: + new_headers["If-Modified-Since"] = headers["Last-Modified"] return new_headers - def cache_response(self, request, response, body=None, - status_codes=None): + def cache_response(self, request, response, body=None, status_codes=None): """ Algorithm for caching requests. @@ -252,9 +255,7 @@ class CacheController(object): cacheable_status_codes = status_codes or self.cacheable_status_codes if response.status not in cacheable_status_codes: logger.debug( - 'Status code %s not in %s', - response.status, - cacheable_status_codes + "Status code %s not in %s", response.status, cacheable_status_codes ) return @@ -264,10 +265,12 @@ class CacheController(object): # Content-Length is valid then we can check to see if the body we've # been given matches the expected size, and if it doesn't we'll just # skip trying to cache it. - if (body is not None and - "content-length" in response_headers and - response_headers["content-length"].isdigit() and - int(response_headers["content-length"]) != len(body)): + if ( + body is not None + and "content-length" in response_headers + and response_headers["content-length"].isdigit() + and int(response_headers["content-length"]) != len(body) + ): return cc_req = self.parse_cache_control(request.headers) @@ -278,53 +281,49 @@ class CacheController(object): # Delete it from the cache if we happen to have it stored there no_store = False - if 'no-store' in cc: + if "no-store" in cc: no_store = True logger.debug('Response header has "no-store"') - if 'no-store' in cc_req: + if "no-store" in cc_req: no_store = True logger.debug('Request header has "no-store"') if no_store and self.cache.get(cache_url): logger.debug('Purging existing cache entry to honor "no-store"') self.cache.delete(cache_url) + if no_store: + return # If we've been given an etag, then keep the response - if self.cache_etags and 'etag' in response_headers: - logger.debug('Caching due to etag') + if self.cache_etags and "etag" in response_headers: + logger.debug("Caching due to etag") self.cache.set( - cache_url, - self.serializer.dumps(request, response, body=body), + cache_url, self.serializer.dumps(request, response, body=body) ) # Add to the cache any 301s. We do this before looking that # the Date headers. elif response.status == 301: - logger.debug('Caching permanant redirect') - self.cache.set( - cache_url, - self.serializer.dumps(request, response) - ) + logger.debug("Caching permanant redirect") + self.cache.set(cache_url, self.serializer.dumps(request, response)) # Add to the cache if the response headers demand it. If there # is no date header then we can't do anything about expiring # the cache. - elif 'date' in response_headers: + elif "date" in response_headers: # cache when there is a max-age > 0 - if 'max-age' in cc and cc['max-age'] > 0: - logger.debug('Caching b/c date exists and max-age > 0') + if "max-age" in cc and cc["max-age"] > 0: + logger.debug("Caching b/c date exists and max-age > 0") self.cache.set( - cache_url, - self.serializer.dumps(request, response, body=body), + cache_url, self.serializer.dumps(request, response, body=body) ) # If the request can expire, it means we should cache it # in the meantime. - elif 'expires' in response_headers: - if response_headers['expires']: - logger.debug('Caching b/c of expires header') + elif "expires" in response_headers: + if response_headers["expires"]: + logger.debug("Caching b/c of expires header") self.cache.set( - cache_url, - self.serializer.dumps(request, response, body=body), + cache_url, self.serializer.dumps(request, response, body=body) ) def update_cached_response(self, request, response): @@ -336,10 +335,7 @@ class CacheController(object): """ cache_url = self.cache_url(request.url) - cached_response = self.serializer.loads( - request, - self.cache.get(cache_url) - ) + cached_response = self.serializer.loads(request, self.cache.get(cache_url)) if not cached_response: # we didn't have a cached response @@ -352,22 +348,20 @@ class CacheController(object): # the cached body invalid. But... just in case, we'll be sure # to strip out ones we know that might be problmatic due to # typical assumptions. - excluded_headers = [ - "content-length", - ] + excluded_headers = ["content-length"] cached_response.headers.update( - dict((k, v) for k, v in response.headers.items() - if k.lower() not in excluded_headers) + dict( + (k, v) + for k, v in response.headers.items() + if k.lower() not in excluded_headers + ) ) # we want a 200 b/c we have content via the cache cached_response.status = 200 # update our cache - self.cache.set( - cache_url, - self.serializer.dumps(request, cached_response), - ) + self.cache.set(cache_url, self.serializer.dumps(request, cached_response)) return cached_response diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py index f1e1ce055..30ed4c5a6 100644 --- a/src/pip/_vendor/cachecontrol/filewrapper.py +++ b/src/pip/_vendor/cachecontrol/filewrapper.py @@ -27,17 +27,19 @@ class CallbackFileWrapper(object): # self.__fp hasn't been set. # # [0] https://docs.python.org/2/reference/expressions.html#atom-identifiers - fp = self.__getattribute__('_CallbackFileWrapper__fp') + fp = self.__getattribute__("_CallbackFileWrapper__fp") return getattr(fp, name) def __is_fp_closed(self): try: return self.__fp.fp is None + except AttributeError: pass try: return self.__fp.closed + except AttributeError: pass @@ -66,7 +68,7 @@ class CallbackFileWrapper(object): def _safe_read(self, amt): data = self.__fp._safe_read(amt) - if amt == 2 and data == b'\r\n': + if amt == 2 and data == b"\r\n": # urllib executes this read to toss the CRLF at the end # of the chunk. return data diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py index f182ff026..6c0e9790d 100644 --- a/src/pip/_vendor/cachecontrol/heuristics.py +++ b/src/pip/_vendor/cachecontrol/heuristics.py @@ -46,7 +46,7 @@ class BaseHeuristic(object): response.headers.update(updated_headers) warning_header_value = self.warning(response) if warning_header_value is not None: - response.headers.update({'Warning': warning_header_value}) + response.headers.update({"Warning": warning_header_value}) return response @@ -56,15 +56,15 @@ class OneDayCache(BaseHeuristic): Cache the response by providing an expires 1 day in the future. """ + def update_headers(self, response): headers = {} - if 'expires' not in response.headers: - date = parsedate(response.headers['date']) - expires = expire_after(timedelta(days=1), - date=datetime(*date[:6])) - headers['expires'] = datetime_to_header(expires) - headers['cache-control'] = 'public' + if "expires" not in response.headers: + date = parsedate(response.headers["date"]) + expires = expire_after(timedelta(days=1), date=datetime(*date[:6])) + headers["expires"] = datetime_to_header(expires) + headers["cache-control"] = "public" return headers @@ -78,13 +78,10 @@ class ExpiresAfter(BaseHeuristic): def update_headers(self, response): expires = expire_after(self.delta) - return { - 'expires': datetime_to_header(expires), - 'cache-control': 'public', - } + return {"expires": datetime_to_header(expires), "cache-control": "public"} def warning(self, response): - tmpl = '110 - Automatically cached for %s. Response might be stale' + tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -100,27 +97,27 @@ class LastModified(BaseHeuristic): http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397 Unlike mozilla we limit this to 24-hr. """ - cacheable_by_default_statuses = set([ + cacheable_by_default_statuses = { 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501 - ]) + } def update_headers(self, resp): headers = resp.headers - if 'expires' in headers: + if "expires" in headers: return {} - if 'cache-control' in headers and headers['cache-control'] != 'public': + if "cache-control" in headers and headers["cache-control"] != "public": return {} if resp.status not in self.cacheable_by_default_statuses: return {} - if 'date' not in headers or 'last-modified' not in headers: + if "date" not in headers or "last-modified" not in headers: return {} - date = calendar.timegm(parsedate_tz(headers['date'])) - last_modified = parsedate(headers['last-modified']) + date = calendar.timegm(parsedate_tz(headers["date"])) + last_modified = parsedate(headers["last-modified"]) if date is None or last_modified is None: return {} @@ -132,7 +129,7 @@ class LastModified(BaseHeuristic): return {} expires = date + freshness_lifetime - return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))} + return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} def warning(self, resp): return None diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py index 05b6e2474..ec43ff27a 100644 --- a/src/pip/_vendor/cachecontrol/serialize.py +++ b/src/pip/_vendor/cachecontrol/serialize.py @@ -48,23 +48,22 @@ class Serializer(object): u"response": { u"body": body, u"headers": dict( - (text_type(k), text_type(v)) - for k, v in response.headers.items() + (text_type(k), text_type(v)) for k, v in response.headers.items() ), u"status": response.status, u"version": response.version, u"reason": text_type(response.reason), u"strict": response.strict, u"decode_content": response.decode_content, - }, + } } # Construct our vary headers data[u"vary"] = {} if u"vary" in response_headers: - varied_headers = response_headers[u'vary'].split(',') + varied_headers = response_headers[u"vary"].split(",") for header in varied_headers: - header = header.strip() + header = text_type(header).strip() header_value = request.headers.get(header, None) if header_value is not None: header_value = text_type(header_value) @@ -95,7 +94,8 @@ class Serializer(object): # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{0}".format(ver))(request, data) + return getattr(self, "_loads_v{}".format(ver))(request, data) + except AttributeError: # This is a version we don't have a loads function for, so we'll # just treat it as a miss and return None @@ -118,11 +118,11 @@ class Serializer(object): body_raw = cached["response"].pop("body") - headers = CaseInsensitiveDict(data=cached['response']['headers']) - if headers.get('transfer-encoding', '') == 'chunked': - headers.pop('transfer-encoding') + headers = CaseInsensitiveDict(data=cached["response"]["headers"]) + if headers.get("transfer-encoding", "") == "chunked": + headers.pop("transfer-encoding") - cached['response']['headers'] = headers + cached["response"]["headers"] = headers try: body = io.BytesIO(body_raw) @@ -133,13 +133,9 @@ class Serializer(object): # fail with: # # TypeError: 'str' does not support the buffer interface - body = io.BytesIO(body_raw.encode('utf8')) + body = io.BytesIO(body_raw.encode("utf8")) - return HTTPResponse( - body=body, - preload_content=False, - **cached["response"] - ) + return HTTPResponse(body=body, preload_content=False, **cached["response"]) def _loads_v0(self, request, data): # The original legacy cache data. This doesn't contain enough @@ -162,16 +158,12 @@ class Serializer(object): return # We need to decode the items that we've base64 encoded - cached["response"]["body"] = _b64_decode_bytes( - cached["response"]["body"] - ) + cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) cached["response"]["headers"] = dict( (_b64_decode_str(k), _b64_decode_str(v)) for k, v in cached["response"]["headers"].items() ) - cached["response"]["reason"] = _b64_decode_str( - cached["response"]["reason"], - ) + cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"]) cached["vary"] = dict( (_b64_decode_str(k), _b64_decode_str(v) if v is not None else v) for k, v in cached["vary"].items() @@ -187,7 +179,7 @@ class Serializer(object): def _loads_v4(self, request, data): try: - cached = msgpack.loads(data, encoding='utf-8') + cached = msgpack.loads(data, encoding="utf-8") except ValueError: return diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py index b50a6e27d..265bfc8bc 100644 --- a/src/pip/_vendor/cachecontrol/wrapper.py +++ b/src/pip/_vendor/cachecontrol/wrapper.py @@ -2,14 +2,16 @@ from .adapter import CacheControlAdapter from .cache import DictCache -def CacheControl(sess, - cache=None, - cache_etags=True, - serializer=None, - heuristic=None, - controller_class=None, - adapter_class=None, - cacheable_methods=None): +def CacheControl( + sess, + cache=None, + cache_etags=True, + serializer=None, + heuristic=None, + controller_class=None, + adapter_class=None, + cacheable_methods=None, +): cache = cache or DictCache() adapter_class = adapter_class or CacheControlAdapter @@ -19,9 +21,9 @@ def CacheControl(sess, serializer=serializer, heuristic=heuristic, controller_class=controller_class, - cacheable_methods=cacheable_methods + cacheable_methods=cacheable_methods, ) - sess.mount('http://', adapter) - sess.mount('https://', adapter) + sess.mount("http://", adapter) + sess.mount("https://", adapter) return sess diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 7fc9d118b..4399d2686 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -4,7 +4,7 @@ distro==1.3.0 html5lib==1.0.1 six==1.11.0 colorama==0.3.9 -CacheControl==0.12.4 +CacheControl==0.12.5 msgpack-python==0.5.6 lockfile==0.12.2 progress==1.3 From 5af09fc8b00a12928d8580bcf83881d00994cad8 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:30:46 +0530 Subject: [PATCH 004/129] Update ipaddress to 1.0.22 --- news/ipaddress.vendor | 1 + src/pip/_vendor/ipaddress.py | 2 +- src/pip/_vendor/vendor.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 news/ipaddress.vendor diff --git a/news/ipaddress.vendor b/news/ipaddress.vendor new file mode 100644 index 000000000..4df2e8374 --- /dev/null +++ b/news/ipaddress.vendor @@ -0,0 +1 @@ +Update ipaddress to 1.0.22 diff --git a/src/pip/_vendor/ipaddress.py b/src/pip/_vendor/ipaddress.py index 8cfdd58ac..f2d076684 100644 --- a/src/pip/_vendor/ipaddress.py +++ b/src/pip/_vendor/ipaddress.py @@ -14,7 +14,7 @@ from __future__ import unicode_literals import itertools import struct -__version__ = '1.0.19' +__version__ = '1.0.22' # Compatibility functions _compat_int_types = (int,) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 4399d2686..32a94c906 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -8,7 +8,7 @@ CacheControl==0.12.5 msgpack-python==0.5.6 lockfile==0.12.2 progress==1.3 -ipaddress==1.0.19 # Only needed on 2.6 and 2.7 +ipaddress==1.0.22 # Only needed on 2.6 and 2.7 packaging==17.1 pyparsing==2.2.0 pytoml==0.1.14 From 26a2ffdb522f71fb8ea91a8586dd4a1eef561e37 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:32:42 +0530 Subject: [PATCH 005/129] Update pytoml to 0.1.16 --- news/pytoml.vendor | 1 + src/pip/_vendor/pytoml/parser.py | 52 ++++++++++++++++---------------- src/pip/_vendor/vendor.txt | 2 +- 3 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 news/pytoml.vendor diff --git a/news/pytoml.vendor b/news/pytoml.vendor new file mode 100644 index 000000000..122491173 --- /dev/null +++ b/news/pytoml.vendor @@ -0,0 +1 @@ +Update pytoml to 0.1.16 diff --git a/src/pip/_vendor/pytoml/parser.py b/src/pip/_vendor/pytoml/parser.py index 7fc3d34d5..e03a03fbd 100644 --- a/src/pip/_vendor/pytoml/parser.py +++ b/src/pip/_vendor/pytoml/parser.py @@ -6,35 +6,35 @@ if sys.version_info[0] == 2: else: _chr = chr -def load(fin, translate=lambda t, x, v: v): - return loads(fin.read(), translate=translate, filename=getattr(fin, 'name', repr(fin))) +def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): + return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) -def loads(s, filename='', translate=lambda t, x, v: v): +def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): if isinstance(s, bytes): s = s.decode('utf-8') s = s.replace('\r\n', '\n') - root = {} - tables = {} + root = object_pairs_hook() + tables = object_pairs_hook() scope = root src = _Source(s, filename=filename) - ast = _p_toml(src) + ast = _p_toml(src, object_pairs_hook=object_pairs_hook) def error(msg): raise TomlError(msg, pos[0], pos[1], filename) - def process_value(v): + def process_value(v, object_pairs_hook): kind, text, value, pos = v if kind == 'str' and value.startswith('\n'): value = value[1:] if kind == 'array': if value and any(k != value[0][0] for k, t, v, p in value[1:]): error('array-type-mismatch') - value = [process_value(item) for item in value] + value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] elif kind == 'table': - value = dict([(k, process_value(value[k])) for k in value]) + value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) return translate(kind, text, value) for kind, value, pos in ast: @@ -42,7 +42,7 @@ def loads(s, filename='', translate=lambda t, x, v: v): k, v = value if k in scope: error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) - scope[k] = process_value(v) + scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) else: is_table_array = (kind == 'table_array') cur = tables @@ -50,19 +50,19 @@ def loads(s, filename='', translate=lambda t, x, v: v): if isinstance(cur.get(name), list): d, cur = cur[name][-1] else: - d, cur = cur.setdefault(name, (None, {})) + d, cur = cur.setdefault(name, (None, object_pairs_hook())) - scope = {} + scope = object_pairs_hook() name = value[-1] if name not in cur: if is_table_array: - cur[name] = [(scope, {})] + cur[name] = [(scope, object_pairs_hook())] else: - cur[name] = (scope, {}) + cur[name] = (scope, object_pairs_hook()) elif isinstance(cur[name], list): if not is_table_array: error('table_type_mismatch') - cur[name].append((scope, {})) + cur[name].append((scope, object_pairs_hook())) else: if is_table_array: error('table_type_mismatch') @@ -73,7 +73,7 @@ def loads(s, filename='', translate=lambda t, x, v: v): def merge_tables(scope, tables): if scope is None: - scope = {} + scope = object_pairs_hook() for k in tables: if k in scope: error('key_table_conflict') @@ -225,7 +225,7 @@ _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') _litstr_re = re.compile(r"[^'\000-\037]*") _litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") -def _p_value(s): +def _p_value(s, object_pairs_hook): pos = s.pos() if s.consume('true'): @@ -283,7 +283,7 @@ def _p_value(s): with s: while True: _p_ews(s) - items.append(_p_value(s)) + items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) s.commit() _p_ews(s) s.expect(',') @@ -294,13 +294,13 @@ def _p_value(s): if s.consume('{'): _p_ws(s) - items = {} + items = object_pairs_hook() if not s.consume('}'): k = _p_key(s) _p_ws(s) s.expect('=') _p_ws(s) - items[k] = _p_value(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) _p_ws(s) while s.consume(','): _p_ws(s) @@ -308,14 +308,14 @@ def _p_value(s): _p_ws(s) s.expect('=') _p_ws(s) - items[k] = _p_value(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) _p_ws(s) s.expect('}') return 'table', None, items, pos s.fail() -def _p_stmt(s): +def _p_stmt(s, object_pairs_hook): pos = s.pos() if s.consume( '['): is_array = s.consume('[') @@ -335,19 +335,19 @@ def _p_stmt(s): _p_ws(s) s.expect('=') _p_ws(s) - value = _p_value(s) + value = _p_value(s, object_pairs_hook=object_pairs_hook) return 'kv', (key, value), pos _stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') -def _p_toml(s): +def _p_toml(s, object_pairs_hook): stmts = [] _p_ews(s) with s: - stmts.append(_p_stmt(s)) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) while True: s.commit() s.expect_re(_stmtsep_re) - stmts.append(_p_stmt(s)) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) _p_ews(s) s.expect_eof() return stmts diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 32a94c906..b32585e08 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -11,7 +11,7 @@ progress==1.3 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 packaging==17.1 pyparsing==2.2.0 -pytoml==0.1.14 +pytoml==0.1.16 retrying==1.3.3 requests==2.18.4 chardet==3.0.4 From 2532a492a9fc46445f3177337825a70410f49f83 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:34:53 +0530 Subject: [PATCH 006/129] Update pkg_resources to 39.2.0 (via setuptools) --- news/pkg_resources.vendor | 1 + src/pip/_vendor/pkg_resources/__init__.py | 29 ++++++++++++++--------- src/pip/_vendor/vendor.txt | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 news/pkg_resources.vendor diff --git a/news/pkg_resources.vendor b/news/pkg_resources.vendor new file mode 100644 index 000000000..ebde0d1dd --- /dev/null +++ b/news/pkg_resources.vendor @@ -0,0 +1 @@ +Update pkg_resources to 39.2.0 (via setuptools) diff --git a/src/pip/_vendor/pkg_resources/__init__.py b/src/pip/_vendor/pkg_resources/__init__.py index 19d3d0562..f2815cc6b 100644 --- a/src/pip/_vendor/pkg_resources/__init__.py +++ b/src/pip/_vendor/pkg_resources/__init__.py @@ -377,11 +377,7 @@ def get_build_platform(): XXX Currently this is the same as ``distutils.util.get_platform()``, but it needs some hacks for Linux and Mac OS X. """ - try: - # Python 2.7 or >=3.2 - from sysconfig import get_platform - except ImportError: - from distutils.util import get_platform + from sysconfig import get_platform plat = get_platform() if sys.platform == "darwin" and not plat.startswith('macosx-'): @@ -1518,12 +1514,10 @@ class DefaultProvider(EggProvider): @classmethod def _register(cls): - loader_cls = getattr( - importlib_machinery, - 'SourceFileLoader', - type(None), - ) - register_loader_type(loader_cls, cls) + loader_names = 'SourceFileLoader', 'SourcelessFileLoader', + for name in loader_names: + loader_cls = getattr(importlib_machinery, name, type(None)) + register_loader_type(loader_cls, cls) DefaultProvider._register() @@ -2669,6 +2663,19 @@ class Distribution(object): raise AttributeError(attr) return getattr(self._provider, attr) + def __dir__(self): + return list( + set(super(Distribution, self).__dir__()) + | set( + attr for attr in self._provider.__dir__() + if not attr.startswith('_') + ) + ) + + if not hasattr(object, '__dir__'): + # python 2.7 not supported + del __dir__ + @classmethod def from_filename(cls, filename, metadata=None, **kw): return cls.from_location( diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index b32585e08..63d6ad06e 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -18,5 +18,5 @@ requests==2.18.4 idna==2.6 urllib3==1.22 certifi==2018.1.18 -setuptools==39.0.1 +setuptools==39.2.0 webencodings==0.5.1 From acd51b2838ed2da7ecdb0c34637a35ae7813a76b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:36:30 +0530 Subject: [PATCH 007/129] Update certifi to 2018.4.16 --- news/certifi.vendor | 1 + src/pip/_vendor/certifi/__init__.py | 2 +- src/pip/_vendor/certifi/cacert.pem | 33 ----------------------------- src/pip/_vendor/vendor.txt | 2 +- 4 files changed, 3 insertions(+), 35 deletions(-) create mode 100644 news/certifi.vendor diff --git a/news/certifi.vendor b/news/certifi.vendor new file mode 100644 index 000000000..e03c3ae1c --- /dev/null +++ b/news/certifi.vendor @@ -0,0 +1 @@ +Update certifi to 2018.4.16 diff --git a/src/pip/_vendor/certifi/__init__.py b/src/pip/_vendor/certifi/__init__.py index 556193cef..0c4963ef6 100644 --- a/src/pip/_vendor/certifi/__init__.py +++ b/src/pip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.01.18" +__version__ = "2018.04.16" diff --git a/src/pip/_vendor/certifi/cacert.pem b/src/pip/_vendor/certifi/cacert.pem index 101ac98fa..2713f541c 100644 --- a/src/pip/_vendor/certifi/cacert.pem +++ b/src/pip/_vendor/certifi/cacert.pem @@ -3483,39 +3483,6 @@ AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su -----END CERTIFICATE----- -# Issuer: CN=T\xdcRKTRUST Elektronik Sertifika Hizmet Sa\u011flay\u0131c\u0131s\u0131 H5 O=T\xdcRKTRUST Bilgi \u0130leti\u015fim ve Bili\u015fim G\xfcvenli\u011fi Hizmetleri A.\u015e. -# Subject: CN=T\xdcRKTRUST Elektronik Sertifika Hizmet Sa\u011flay\u0131c\u0131s\u0131 H5 O=T\xdcRKTRUST Bilgi \u0130leti\u015fim ve Bili\u015fim G\xfcvenli\u011fi Hizmetleri A.\u015e. -# Label: "T\xdcRKTRUST Elektronik Sertifika Hizmet Sa\u011flay\u0131c\u0131s\u0131 H5" -# Serial: 156233699172481 -# MD5 Fingerprint: da:70:8e:f0:22:df:93:26:f6:5f:9f:d3:15:06:52:4e -# SHA1 Fingerprint: c4:18:f6:4d:46:d1:df:00:3d:27:30:13:72:43:a9:12:11:c6:75:fb -# SHA256 Fingerprint: 49:35:1b:90:34:44:c1:85:cc:dc:5c:69:3d:24:d8:55:5c:b2:08:d6:a8:14:13:07:69:9f:4a:f0:63:19:9d:78 ------BEGIN CERTIFICATE----- -MIIEJzCCAw+gAwIBAgIHAI4X/iQggTANBgkqhkiG9w0BAQsFADCBsTELMAkGA1UE -BhMCVFIxDzANBgNVBAcMBkFua2FyYTFNMEsGA1UECgxEVMOcUktUUlVTVCBCaWxn -aSDEsGxldGnFn2ltIHZlIEJpbGnFn2ltIEfDvHZlbmxpxJ9pIEhpem1ldGxlcmkg -QS7Fni4xQjBABgNVBAMMOVTDnFJLVFJVU1QgRWxla3Ryb25payBTZXJ0aWZpa2Eg -SGl6bWV0IFNhxJ9sYXnEsWPEsXPEsSBINTAeFw0xMzA0MzAwODA3MDFaFw0yMzA0 -MjgwODA3MDFaMIGxMQswCQYDVQQGEwJUUjEPMA0GA1UEBwwGQW5rYXJhMU0wSwYD -VQQKDERUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUgQmlsacWfaW0gR8O8 -dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLjFCMEAGA1UEAww5VMOcUktUUlVTVCBF -bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxIEg1MIIB -IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApCUZ4WWe60ghUEoI5RHwWrom -/4NZzkQqL/7hzmAD/I0Dpe3/a6i6zDQGn1k19uwsu537jVJp45wnEFPzpALFp/kR -Gml1bsMdi9GYjZOHp3GXDSHHmflS0yxjXVW86B8BSLlg/kJK9siArs1mep5Fimh3 -4khon6La8eHBEJ/rPCmBp+EyCNSgBbGM+42WAA4+Jd9ThiI7/PS98wl+d+yG6w8z -5UNP9FR1bSmZLmZaQ9/LXMrI5Tjxfjs1nQ/0xVqhzPMggCTTV+wVunUlm+hkS7M0 -hO8EuPbJbKoCPrZV4jI3X/xml1/N1p7HIL9Nxqw/dV8c7TKcfGkAaZHjIxhT6QID -AQABo0IwQDAdBgNVHQ4EFgQUVpkHHtOsDGlktAxQR95DLL4gwPswDgYDVR0PAQH/ -BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJ5FdnsX -SDLyOIspve6WSk6BGLFRRyDN0GSxDsnZAdkJzsiZ3GglE9Rc8qPoBP5yCccLqh0l -VX6Wmle3usURehnmp349hQ71+S4pL+f5bFgWV1Al9j4uPqrtd3GqqpmWRgqujuwq -URawXs3qZwQcWDD1YIq9pr1N5Za0/EKJAWv2cMhQOQwt1WbZyNKzMrcbGW3LM/nf -peYVhDfwwvJllpKQd/Ct9JDpEXjXk4nAPQu6KfTomZ1yju2dL+6SfaHx/126M2CF -Yv4HAqGEVka+lgqaE9chTLd8B59OTj+RdPsnnRHM3eaxynFNExc5JsUpISuTKWqW -+qtB4Uu2NQvAmxU= ------END CERTIFICATE----- - # Issuer: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 # Subject: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 # Label: "Certinomis - Root CA" diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 63d6ad06e..c0af28813 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -17,6 +17,6 @@ requests==2.18.4 chardet==3.0.4 idna==2.6 urllib3==1.22 - certifi==2018.1.18 + certifi==2018.4.16 setuptools==39.2.0 webencodings==0.5.1 From b1aee4b3876bbdc319a75726363c809953d71ede Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:38:05 +0530 Subject: [PATCH 008/129] Update urllib3 to 1.23 --- news/urllib3.vendor | 1 + src/pip/_vendor/urllib3/__init__.py | 2 +- src/pip/_vendor/urllib3/_collections.py | 23 +- src/pip/_vendor/urllib3/connection.py | 40 +- src/pip/_vendor/urllib3/connectionpool.py | 27 +- src/pip/_vendor/urllib3/contrib/__init__.py | 0 .../contrib/_securetransport/__init__.py | 0 .../contrib/_securetransport/bindings.py | 0 .../contrib/_securetransport/low_level.py | 7 +- src/pip/_vendor/urllib3/contrib/appengine.py | 11 +- src/pip/_vendor/urllib3/contrib/ntlmpool.py | 0 src/pip/_vendor/urllib3/contrib/pyopenssl.py | 22 +- .../urllib3/contrib/securetransport.py | 32 +- src/pip/_vendor/urllib3/contrib/socks.py | 4 + src/pip/_vendor/urllib3/exceptions.py | 2 +- src/pip/_vendor/urllib3/fields.py | 2 +- src/pip/_vendor/urllib3/filepost.py | 10 +- src/pip/_vendor/urllib3/packages/__init__.py | 0 .../urllib3/packages/backports/__init__.py | 0 .../urllib3/packages/backports/makefile.py | 0 .../_vendor/urllib3/packages/ordered_dict.py | 0 src/pip/_vendor/urllib3/packages/six.py | 0 .../packages/ssl_match_hostname/__init__.py | 0 .../ssl_match_hostname/_implementation.py | 0 src/pip/_vendor/urllib3/poolmanager.py | 13 +- src/pip/_vendor/urllib3/request.py | 8 +- src/pip/_vendor/urllib3/response.py | 98 ++- src/pip/_vendor/urllib3/util/__init__.py | 0 src/pip/_vendor/urllib3/util/connection.py | 14 +- src/pip/_vendor/urllib3/util/queue.py | 21 + src/pip/_vendor/urllib3/util/request.py | 0 src/pip/_vendor/urllib3/util/response.py | 0 src/pip/_vendor/urllib3/util/retry.py | 12 +- src/pip/_vendor/urllib3/util/selectors.py | 581 ------------------ src/pip/_vendor/urllib3/util/ssl_.py | 81 ++- src/pip/_vendor/urllib3/util/timeout.py | 0 src/pip/_vendor/urllib3/util/url.py | 0 src/pip/_vendor/urllib3/util/wait.py | 179 +++++- src/pip/_vendor/vendor.txt | 2 +- 39 files changed, 464 insertions(+), 728 deletions(-) create mode 100644 news/urllib3.vendor mode change 100644 => 100755 src/pip/_vendor/urllib3/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/_collections.py mode change 100644 => 100755 src/pip/_vendor/urllib3/connection.py mode change 100644 => 100755 src/pip/_vendor/urllib3/connectionpool.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/appengine.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/ntlmpool.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/pyopenssl.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/securetransport.py mode change 100644 => 100755 src/pip/_vendor/urllib3/contrib/socks.py mode change 100644 => 100755 src/pip/_vendor/urllib3/exceptions.py mode change 100644 => 100755 src/pip/_vendor/urllib3/fields.py mode change 100644 => 100755 src/pip/_vendor/urllib3/filepost.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/backports/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/backports/makefile.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/ordered_dict.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/six.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py mode change 100644 => 100755 src/pip/_vendor/urllib3/poolmanager.py mode change 100644 => 100755 src/pip/_vendor/urllib3/request.py mode change 100644 => 100755 src/pip/_vendor/urllib3/response.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/__init__.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/connection.py create mode 100755 src/pip/_vendor/urllib3/util/queue.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/request.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/response.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/retry.py delete mode 100644 src/pip/_vendor/urllib3/util/selectors.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/ssl_.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/timeout.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/url.py mode change 100644 => 100755 src/pip/_vendor/urllib3/util/wait.py diff --git a/news/urllib3.vendor b/news/urllib3.vendor new file mode 100644 index 000000000..d854f4703 --- /dev/null +++ b/news/urllib3.vendor @@ -0,0 +1 @@ +Update urllib3 to 1.23 diff --git a/src/pip/_vendor/urllib3/__init__.py b/src/pip/_vendor/urllib3/__init__.py old mode 100644 new mode 100755 index aaa6b1c6d..4bd533b5b --- a/src/pip/_vendor/urllib3/__init__.py +++ b/src/pip/_vendor/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.22' +__version__ = '1.23' __all__ = ( 'HTTPConnectionPool', diff --git a/src/pip/_vendor/urllib3/_collections.py b/src/pip/_vendor/urllib3/_collections.py old mode 100644 new mode 100755 index 5df2372c4..6e36b84e5 --- a/src/pip/_vendor/urllib3/_collections.py +++ b/src/pip/_vendor/urllib3/_collections.py @@ -1,5 +1,8 @@ from __future__ import absolute_import -from collections import Mapping, MutableMapping +try: + from collections.abc import Mapping, MutableMapping +except ImportError: + from collections import Mapping, MutableMapping try: from threading import RLock except ImportError: # Platform-specific: No threads available @@ -15,6 +18,7 @@ try: # Python 2.7+ from collections import OrderedDict except ImportError: from .packages.ordered_dict import OrderedDict +from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 @@ -305,13 +309,22 @@ class HTTPHeaderDict(MutableMapping): # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. + obs_fold_continued_leaders = (' ', '\t') headers = [] for line in message.headers: - if line.startswith((' ', '\t')): - key, value = headers[-1] - headers[-1] = (key, value + '\r\n' + line.rstrip()) - continue + if line.startswith(obs_fold_continued_leaders): + if not headers: + # We received a header line that starts with OWS as described + # in RFC-7230 S3.2.4. This indicates a multiline header, but + # there exists no previous header to which we can attach it. + raise InvalidHeader( + 'Header continuation with no previous header: %s' % line + ) + else: + key, value = headers[-1] + headers[-1] = (key, value + ' ' + line.strip()) + continue key, value = line.split(':', 1) headers.append((key, value.strip())) diff --git a/src/pip/_vendor/urllib3/connection.py b/src/pip/_vendor/urllib3/connection.py old mode 100644 new mode 100755 index c0d832998..a03b573f0 --- a/src/pip/_vendor/urllib3/connection.py +++ b/src/pip/_vendor/urllib3/connection.py @@ -56,10 +56,11 @@ port_by_scheme = { 'https': 443, } -# When updating RECENT_DATE, move it to -# within two years of the current date, and no -# earlier than 6 months ago. -RECENT_DATE = datetime.date(2016, 1, 1) +# When updating RECENT_DATE, move it to within two years of the current date, +# and not less than 6 months ago. +# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or +# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months) +RECENT_DATE = datetime.date(2017, 6, 30) class DummyConnection(object): @@ -124,6 +125,35 @@ class HTTPConnection(_HTTPConnection, object): # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) + @property + def host(self): + """ + Getter method to remove any trailing dots that indicate the hostname is an FQDN. + + In general, SSL certificates don't include the trailing dot indicating a + fully-qualified domain name, and thus, they don't validate properly when + checked against a domain name that includes the dot. In addition, some + servers may not expect to receive the trailing dot when provided. + + However, the hostname with trailing dot is critical to DNS resolution; doing a + lookup with the trailing dot will properly only resolve the appropriate FQDN, + whereas a lookup without a trailing dot will search the system's search domain + list. Thus, it's important to keep the original host around for use only in + those cases where it's appropriate (i.e., when doing DNS lookup to establish the + actual TCP connection across which we're going to send HTTP requests). + """ + return self._dns_host.rstrip('.') + + @host.setter + def host(self, value): + """ + Setter for the `host` property. + + We assume that only urllib3 uses the _dns_host attribute; httplib itself + only uses `host`, and it seems reasonable that other libraries follow suit. + """ + self._dns_host = value + def _new_conn(self): """ Establish a socket connection and set nodelay settings on it. @@ -138,7 +168,7 @@ class HTTPConnection(_HTTPConnection, object): try: conn = connection.create_connection( - (self.host, self.port), self.timeout, **extra_kw) + (self._dns_host, self.port), self.timeout, **extra_kw) except SocketTimeout as e: raise ConnectTimeoutError( diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py old mode 100644 new mode 100755 index ec9600f8f..8fcb0bce7 --- a/src/pip/_vendor/urllib3/connectionpool.py +++ b/src/pip/_vendor/urllib3/connectionpool.py @@ -40,13 +40,10 @@ from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout -from .util.url import get_host, Url +from .util.url import get_host, Url, NORMALIZABLE_SCHEMES +from .util.queue import LifoQueue -if six.PY2: - # Queue is imported for side effects on MS Windows - import Queue as _unused_module_Queue # noqa: F401 - xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -62,13 +59,13 @@ class ConnectionPool(object): """ scheme = None - QueueCls = queue.LifoQueue + QueueCls = LifoQueue def __init__(self, host, port=None): if not host: raise LocationValueError("No host specified.") - self.host = _ipv6_host(host).lower() + self.host = _ipv6_host(host, self.scheme) self._proxy_host = host.lower() self.port = port @@ -204,8 +201,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTP connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTP connection (%d): %s:%s", + self.num_connections, self.host, self.port or "80") conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -411,6 +408,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ Close all pooled connections and disable the pool. """ + if self.pool is None: + return # Disable access to the pool old_pool, self.pool = self.pool, None @@ -434,7 +433,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - host = _ipv6_host(host).lower() + host = _ipv6_host(host, self.scheme) # Use explicit default port for comparison when none is given if self.port and not port: @@ -820,8 +819,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTPS connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTPS connection (%d): %s:%s", + self.num_connections, self.host, self.port or "443") if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -886,7 +885,7 @@ def connection_from_url(url, **kw): return HTTPConnectionPool(host, port=port, **kw) -def _ipv6_host(host): +def _ipv6_host(host, scheme): """ Process IPv6 address literals """ @@ -902,4 +901,6 @@ def _ipv6_host(host): # percent sign might be URIencoded, convert it back into ASCII if host.startswith('[') and host.endswith(']'): host = host.replace('%25', '%').strip('[]') + if scheme in NORMALIZABLE_SCHEMES: + host = host.lower() return host diff --git a/src/pip/_vendor/urllib3/contrib/__init__.py b/src/pip/_vendor/urllib3/contrib/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py b/src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py b/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py b/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py old mode 100644 new mode 100755 index 5e3494bce..b13cd9e72 --- a/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py +++ b/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py @@ -111,6 +111,9 @@ def _cert_array_from_pem(pem_bundle): Given a bundle of certs in PEM format, turns them into a CFArray of certs that can be used to validate a cert chain. """ + # Normalize the PEM bundle's line endings. + pem_bundle = pem_bundle.replace(b"\r\n", b"\n") + der_certs = [ base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) @@ -183,8 +186,8 @@ def _temporary_keychain(): # some random bytes to password-protect the keychain we're creating, so we # ask for 40 random bytes. random_bytes = os.urandom(40) - filename = base64.b64encode(random_bytes[:8]).decode('utf-8') - password = base64.b64encode(random_bytes[8:]) # Must be valid UTF-8 + filename = base64.b16encode(random_bytes[:8]).decode('utf-8') + password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 tempdirectory = tempfile.mkdtemp() keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') diff --git a/src/pip/_vendor/urllib3/contrib/appengine.py b/src/pip/_vendor/urllib3/contrib/appengine.py old mode 100644 new mode 100755 index ce17e83f5..59f2a617c --- a/src/pip/_vendor/urllib3/contrib/appengine.py +++ b/src/pip/_vendor/urllib3/contrib/appengine.py @@ -236,15 +236,24 @@ class AppEngineManager(RequestMethods): encodings.remove('chunked') urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) - return HTTPResponse( + original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. body=BytesIO(urlfetch_resp.content), + msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, **response_kw ) + return HTTPResponse( + body=BytesIO(urlfetch_resp.content), + headers=urlfetch_resp.headers, + status=urlfetch_resp.status_code, + original_response=original_response, + **response_kw + ) + def _get_absolute_timeout(self, timeout): if timeout is Timeout.DEFAULT_TIMEOUT: return None # Defer to URLFetch's default. diff --git a/src/pip/_vendor/urllib3/contrib/ntlmpool.py b/src/pip/_vendor/urllib3/contrib/ntlmpool.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/pyopenssl.py b/src/pip/_vendor/urllib3/contrib/pyopenssl.py old mode 100644 new mode 100755 index 1bb3787d3..6dd3a01e2 --- a/src/pip/_vendor/urllib3/contrib/pyopenssl.py +++ b/src/pip/_vendor/urllib3/contrib/pyopenssl.py @@ -47,6 +47,12 @@ import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend from cryptography.hazmat.backends.openssl.x509 import _Certificate +try: + from cryptography.x509 import UnsupportedExtension +except ImportError: + # UnsupportedExtension is gone in cryptography >= 2.1.0 + class UnsupportedExtension(Exception): + pass from socket import timeout, error as SocketError from io import BytesIO @@ -199,7 +205,7 @@ def get_subj_alt_name(peer_cert): except x509.ExtensionNotFound: # No such extension, return the empty list. return [] - except (x509.DuplicateExtension, x509.UnsupportedExtension, + except (x509.DuplicateExtension, UnsupportedExtension, x509.UnsupportedGeneralNameType, UnicodeError) as e: # A problem has been found with the quality of the certificate. Assume # no SAN field is present. @@ -267,8 +273,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd = util.wait_for_read(self.socket, self.socket.gettimeout()) - if not rd: + if not util.wait_for_read(self.socket, self.socket.gettimeout()): raise timeout('The read operation timed out') else: return self.recv(*args, **kwargs) @@ -289,8 +294,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd = util.wait_for_read(self.socket, self.socket.gettimeout()) - if not rd: + if not util.wait_for_read(self.socket, self.socket.gettimeout()): raise timeout('The read operation timed out') else: return self.recv_into(*args, **kwargs) @@ -303,8 +307,7 @@ class WrappedSocket(object): try: return self.connection.send(data) except OpenSSL.SSL.WantWriteError: - wr = util.wait_for_write(self.socket, self.socket.gettimeout()) - if not wr: + if not util.wait_for_write(self.socket, self.socket.gettimeout()): raise timeout() continue except OpenSSL.SSL.SysCallError as e: @@ -418,7 +421,7 @@ class PyOpenSSLContext(object): self._ctx.load_verify_locations(BytesIO(cadata)) def load_cert_chain(self, certfile, keyfile=None, password=None): - self._ctx.use_certificate_file(certfile) + self._ctx.use_certificate_chain_file(certfile) if password is not None: self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) self._ctx.use_privatekey_file(keyfile or certfile) @@ -440,8 +443,7 @@ class PyOpenSSLContext(object): try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - rd = util.wait_for_read(sock, sock.gettimeout()) - if not rd: + if not util.wait_for_read(sock, sock.gettimeout()): raise timeout('select timed out') continue except OpenSSL.SSL.Error as e: diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py old mode 100644 new mode 100755 index 2cac70f7f..77cb59ed7 --- a/src/pip/_vendor/urllib3/contrib/securetransport.py +++ b/src/pip/_vendor/urllib3/contrib/securetransport.py @@ -51,11 +51,6 @@ except ImportError: # Platform-specific: Python 3 _fileobject = None from ..packages.backports.makefile import backport_makefile -try: - memoryview(b'') -except NameError: - raise ImportError("SecureTransport only works on Pythons with memoryview") - __all__ = ['inject_into_urllib3', 'extract_from_urllib3'] # SNI always works @@ -88,7 +83,7 @@ _connection_ref_lock = threading.Lock() SSL_WRITE_BLOCKSIZE = 16384 # This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to -# individual cipher suites. We need to do this becuase this is how +# individual cipher suites. We need to do this because this is how # SecureTransport wants them. CIPHER_SUITES = [ SecurityConst.TLS_AES_256_GCM_SHA384, @@ -195,21 +190,18 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): timeout = wrapped_socket.gettimeout() error = None read_count = 0 - buffer = (ctypes.c_char * requested_length).from_address(data_buffer) - buffer_view = memoryview(buffer) try: while read_count < requested_length: if timeout is None or timeout >= 0: - readables = util.wait_for_read([base_socket], timeout) - if not readables: + if not util.wait_for_read(base_socket, timeout): raise socket.error(errno.EAGAIN, 'timed out') - # We need to tell ctypes that we have a buffer that can be - # written to. Upsettingly, we do that like this: - chunk_size = base_socket.recv_into( - buffer_view[read_count:requested_length] + remaining = requested_length - read_count + buffer = (ctypes.c_char * remaining).from_address( + data_buffer + read_count ) + chunk_size = base_socket.recv_into(buffer, remaining) read_count += chunk_size if not chunk_size: if not read_count: @@ -219,7 +211,8 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): error = e.errno if error is not None and error != errno.EAGAIN: - if error == errno.ECONNRESET: + data_length_pointer[0] = read_count + if error == errno.ECONNRESET or error == errno.EPIPE: return SecurityConst.errSSLClosedAbort raise @@ -257,8 +250,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): try: while sent < bytes_to_write: if timeout is None or timeout >= 0: - writables = util.wait_for_write([base_socket], timeout) - if not writables: + if not util.wait_for_write(base_socket, timeout): raise socket.error(errno.EAGAIN, 'timed out') chunk_sent = base_socket.send(data) sent += chunk_sent @@ -270,11 +262,13 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): error = e.errno if error is not None and error != errno.EAGAIN: - if error == errno.ECONNRESET: + data_length_pointer[0] = sent + if error == errno.ECONNRESET or error == errno.EPIPE: return SecurityConst.errSSLClosedAbort raise data_length_pointer[0] = sent + if sent != bytes_to_write: return SecurityConst.errSSLWouldBlock @@ -399,7 +393,7 @@ class WrappedSocket(object): if trust: CoreFoundation.CFRelease(trust) - if cert_array is None: + if cert_array is not None: CoreFoundation.CFRelease(cert_array) # Ok, now we can look at what the result was. diff --git a/src/pip/_vendor/urllib3/contrib/socks.py b/src/pip/_vendor/urllib3/contrib/socks.py old mode 100644 new mode 100755 index 39e92fde1..811e312ec --- a/src/pip/_vendor/urllib3/contrib/socks.py +++ b/src/pip/_vendor/urllib3/contrib/socks.py @@ -152,6 +152,10 @@ class SOCKSProxyManager(PoolManager): num_pools=10, headers=None, **connection_pool_kw): parsed = parse_url(proxy_url) + if username is None and password is None and parsed.auth is not None: + split = parsed.auth.split(':') + if len(split) == 2: + username, password = split if parsed.scheme == 'socks5': socks_version = socks.PROXY_TYPE_SOCKS5 rdns = False diff --git a/src/pip/_vendor/urllib3/exceptions.py b/src/pip/_vendor/urllib3/exceptions.py old mode 100644 new mode 100755 index 6c4be5810..7bbaa9871 --- a/src/pip/_vendor/urllib3/exceptions.py +++ b/src/pip/_vendor/urllib3/exceptions.py @@ -154,7 +154,7 @@ class ResponseError(HTTPError): class SecurityWarning(HTTPWarning): - "Warned when perfoming security reducing actions" + "Warned when performing security reducing actions" pass diff --git a/src/pip/_vendor/urllib3/fields.py b/src/pip/_vendor/urllib3/fields.py old mode 100644 new mode 100755 index 19b0ae0c8..37fe64a3e --- a/src/pip/_vendor/urllib3/fields.py +++ b/src/pip/_vendor/urllib3/fields.py @@ -121,7 +121,7 @@ class RequestField(object): 'Content-Disposition' fields. :param header_parts: - A sequence of (k, v) typles or a :class:`dict` of (k, v) to format + A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format as `k1="v1"; k2="v2"; ...`. """ parts = [] diff --git a/src/pip/_vendor/urllib3/filepost.py b/src/pip/_vendor/urllib3/filepost.py old mode 100644 new mode 100755 index cd11cee46..78f1e19b0 --- a/src/pip/_vendor/urllib3/filepost.py +++ b/src/pip/_vendor/urllib3/filepost.py @@ -1,7 +1,8 @@ from __future__ import absolute_import +import binascii import codecs +import os -from uuid import uuid4 from io import BytesIO from .packages import six @@ -15,7 +16,10 @@ def choose_boundary(): """ Our embarrassingly-simple replacement for mimetools.choose_boundary. """ - return uuid4().hex + boundary = binascii.hexlify(os.urandom(16)) + if six.PY3: + boundary = boundary.decode('ascii') + return boundary def iter_field_objects(fields): @@ -65,7 +69,7 @@ def encode_multipart_formdata(fields, boundary=None): :param boundary: If not specified, then a random boundary will be generated using - :func:`mimetools.choose_boundary`. + :func:`urllib3.filepost.choose_boundary`. """ body = BytesIO() if boundary is None: diff --git a/src/pip/_vendor/urllib3/packages/__init__.py b/src/pip/_vendor/urllib3/packages/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/backports/__init__.py b/src/pip/_vendor/urllib3/packages/backports/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/backports/makefile.py b/src/pip/_vendor/urllib3/packages/backports/makefile.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/ordered_dict.py b/src/pip/_vendor/urllib3/packages/ordered_dict.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/six.py b/src/pip/_vendor/urllib3/packages/six.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/poolmanager.py b/src/pip/_vendor/urllib3/poolmanager.py old mode 100644 new mode 100755 index 4ae91744d..506a3c9b8 --- a/src/pip/_vendor/urllib3/poolmanager.py +++ b/src/pip/_vendor/urllib3/poolmanager.py @@ -312,8 +312,9 @@ class PoolManager(RequestMethods): kw['assert_same_host'] = False kw['redirect'] = False + if 'headers' not in kw: - kw['headers'] = self.headers + kw['headers'] = self.headers.copy() if self.proxy is not None and u.scheme == "http": response = conn.urlopen(method, url, **kw) @@ -335,6 +336,14 @@ class PoolManager(RequestMethods): if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) + # Strip headers marked as unsafe to forward to the redirected location. + # Check remove_headers_on_redirect to avoid a potential network call within + # conn.is_same_host() which may use socket.gethostbyname() in the future. + if (retries.remove_headers_on_redirect + and not conn.is_same_host(redirect_location)): + for header in retries.remove_headers_on_redirect: + kw['headers'].pop(header, None) + try: retries = retries.increment(method, url, response=response, _pool=conn) except MaxRetryError: @@ -358,7 +367,7 @@ class ProxyManager(PoolManager): The URL of the proxy to be used. :param proxy_headers: - A dictionary contaning headers that will be sent to the proxy. In case + A dictionary containing headers that will be sent to the proxy. In case of HTTP they are being sent with each request, while in the HTTPS/CONNECT case they are sent only once. Could be used for proxy authentication. diff --git a/src/pip/_vendor/urllib3/request.py b/src/pip/_vendor/urllib3/request.py old mode 100644 new mode 100755 index c0fddff04..1be333411 --- a/src/pip/_vendor/urllib3/request.py +++ b/src/pip/_vendor/urllib3/request.py @@ -44,8 +44,8 @@ class RequestMethods(object): def urlopen(self, method, url, body=None, headers=None, encode_multipart=True, multipart_boundary=None, **kw): # Abstract - raise NotImplemented("Classes extending RequestMethods must implement " - "their own ``urlopen`` method.") + raise NotImplementedError("Classes extending RequestMethods must implement " + "their own ``urlopen`` method.") def request(self, method, url, fields=None, headers=None, **urlopen_kw): """ @@ -60,6 +60,8 @@ class RequestMethods(object): """ method = method.upper() + urlopen_kw['request_url'] = url + if method in self._encode_url_methods: return self.request_encode_url(method, url, fields=fields, headers=headers, @@ -117,7 +119,7 @@ class RequestMethods(object): } When uploading a file, providing a filename (the first parameter of the - tuple) is optional but recommended to best mimick behavior of browsers. + tuple) is optional but recommended to best mimic behavior of browsers. Note that if ``headers`` are supplied, the 'Content-Type' header will be overwritten because it depends on the dynamic random boundary string diff --git a/src/pip/_vendor/urllib3/response.py b/src/pip/_vendor/urllib3/response.py old mode 100644 new mode 100755 index d3e5a1e60..9873cb942 --- a/src/pip/_vendor/urllib3/response.py +++ b/src/pip/_vendor/urllib3/response.py @@ -52,18 +52,42 @@ class DeflateDecoder(object): self._data = None +class GzipDecoderState(object): + + FIRST_MEMBER = 0 + OTHER_MEMBERS = 1 + SWALLOW_DATA = 2 + + class GzipDecoder(object): def __init__(self): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + self._state = GzipDecoderState.FIRST_MEMBER def __getattr__(self, name): return getattr(self._obj, name) def decompress(self, data): - if not data: - return data - return self._obj.decompress(data) + ret = binary_type() + if self._state == GzipDecoderState.SWALLOW_DATA or not data: + return ret + while True: + try: + ret += self._obj.decompress(data) + except zlib.error: + previous_state = self._state + # Ignore data after the first error + self._state = GzipDecoderState.SWALLOW_DATA + if previous_state == GzipDecoderState.OTHER_MEMBERS: + # Allow trailing garbage acceptable in other gzip clients + return ret + raise + data = self._obj.unused_data + if not data: + return ret + self._state = GzipDecoderState.OTHER_MEMBERS + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) def _get_decoder(mode): @@ -89,9 +113,8 @@ class HTTPResponse(io.IOBase): If True, the response's body will be preloaded during construction. :param decode_content: - If True, attempts to decode specific content-encoding's based on headers - (like 'gzip' and 'deflate') will be skipped and raw data will be used - instead. + If True, will attempt to decode the body based on the + 'content-encoding' header. :param original_response: When this HTTPResponse wrapper is generated from an httplib.HTTPResponse @@ -112,8 +135,9 @@ class HTTPResponse(io.IOBase): def __init__(self, body='', headers=None, status=0, version=0, reason=None, strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None, - retries=None, enforce_content_length=False, request_method=None): + original_response=None, pool=None, connection=None, msg=None, + retries=None, enforce_content_length=False, + request_method=None, request_url=None): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -132,6 +156,8 @@ class HTTPResponse(io.IOBase): self._fp = None self._original_response = original_response self._fp_bytes_read = 0 + self.msg = msg + self._request_url = request_url if body and isinstance(body, (basestring, binary_type)): self._body = body @@ -191,6 +217,9 @@ class HTTPResponse(io.IOBase): def connection(self): return self._connection + def isclosed(self): + return is_fp_closed(self._fp) + def tell(self): """ Obtain the number of bytes pulled over the wire so far. May differ from @@ -205,18 +234,18 @@ class HTTPResponse(io.IOBase): """ length = self.headers.get('content-length') - if length is not None and self.chunked: - # This Response will fail with an IncompleteRead if it can't be - # received as chunked. This method falls back to attempt reading - # the response before raising an exception. - log.warning("Received response with both Content-Length and " - "Transfer-Encoding set. This is expressly forbidden " - "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " - "attempting to process response as Transfer-Encoding: " - "chunked.") - return None + if length is not None: + if self.chunked: + # This Response will fail with an IncompleteRead if it can't be + # received as chunked. This method falls back to attempt reading + # the response before raising an exception. + log.warning("Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked.") + return None - elif length is not None: try: # RFC 7230 section 3.3.2 specifies multiple content lengths can # be sent in a single Content-Length header @@ -573,6 +602,11 @@ class HTTPResponse(io.IOBase): Similar to :meth:`HTTPResponse.read`, but with an additional parameter: ``decode_content``. + :param amt: + How much of the content to read. If specified, caching is skipped + because it doesn't make sense to cache partial content as the full + response. + :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. @@ -588,12 +622,17 @@ class HTTPResponse(io.IOBase): "Body should be httplib.HTTPResponse like. " "It should have have an fp attribute which returns raw chunks.") - # Don't bother reading the body of a HEAD request. - if self._original_response and is_response_to_head(self._original_response): - self._original_response.close() - return - with self._error_catcher(): + # Don't bother reading the body of a HEAD request. + if self._original_response and is_response_to_head(self._original_response): + self._original_response.close() + return + + # If a response is already read and closed + # then return immediately. + if self._fp.fp is None: + return + while True: self._update_chunk_length() if self.chunk_left == 0: @@ -624,3 +663,14 @@ class HTTPResponse(io.IOBase): # We read everything; close the "file". if self._original_response: self._original_response.close() + + def geturl(self): + """ + Returns the URL that was the source of this response. + If the request that generated this response redirected, this method + will return the final redirect location. + """ + if self.retries is not None and len(self.retries.history): + return self.retries.history[-1].redirect_location + else: + return self._request_url diff --git a/src/pip/_vendor/urllib3/util/__init__.py b/src/pip/_vendor/urllib3/util/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/connection.py b/src/pip/_vendor/urllib3/util/connection.py old mode 100644 new mode 100755 index bf699cfd0..5cf488f4b --- a/src/pip/_vendor/urllib3/util/connection.py +++ b/src/pip/_vendor/urllib3/util/connection.py @@ -1,7 +1,6 @@ from __future__ import absolute_import import socket -from .wait import wait_for_read -from .selectors import HAS_SELECT, SelectorError +from .wait import NoWayToWaitForSocketError, wait_for_read def is_connection_dropped(conn): # Platform-specific @@ -19,14 +18,11 @@ def is_connection_dropped(conn): # Platform-specific return False if sock is None: # Connection already closed (such as by httplib). return True - - if not HAS_SELECT: - return False - try: - return bool(wait_for_read(sock, timeout=0.0)) - except SelectorError: - return True + # Returns True if readable, which here means it's been dropped + return wait_for_read(sock, timeout=0.0) + except NoWayToWaitForSocketError: # Platform-specific: AppEngine + return False # This function is copied from socket.py in the Python 2.7 standard diff --git a/src/pip/_vendor/urllib3/util/queue.py b/src/pip/_vendor/urllib3/util/queue.py new file mode 100755 index 000000000..d3d379a19 --- /dev/null +++ b/src/pip/_vendor/urllib3/util/queue.py @@ -0,0 +1,21 @@ +import collections +from ..packages import six +from ..packages.six.moves import queue + +if six.PY2: + # Queue is imported for side effects on MS Windows. See issue #229. + import Queue as _unused_module_Queue # noqa: F401 + + +class LifoQueue(queue.Queue): + def _init(self, _): + self.queue = collections.deque() + + def _qsize(self, len=len): + return len(self.queue) + + def _put(self, item): + self.queue.append(item) + + def _get(self): + return self.queue.pop() diff --git a/src/pip/_vendor/urllib3/util/request.py b/src/pip/_vendor/urllib3/util/request.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/response.py b/src/pip/_vendor/urllib3/util/response.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py old mode 100644 new mode 100755 index c603cb490..7ad3dc660 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -19,6 +19,7 @@ from ..packages import six log = logging.getLogger(__name__) + # Data structure for representing the metadata of requests that result in a retry. RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", "status", "redirect_location"]) @@ -139,6 +140,10 @@ class Retry(object): Whether to respect Retry-After header on status codes defined as :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. + :param iterable remove_headers_on_redirect: + Sequence of headers to remove from the request when a response + indicating a redirect is returned before firing off the redirected + request. """ DEFAULT_METHOD_WHITELIST = frozenset([ @@ -146,13 +151,16 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) + DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) + #: Maximum backoff time. BACKOFF_MAX = 120 def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - history=None, respect_retry_after_header=True): + history=None, respect_retry_after_header=True, + remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): self.total = total self.connect = connect @@ -171,6 +179,7 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header + self.remove_headers_on_redirect = remove_headers_on_redirect def new(self, **kw): params = dict( @@ -182,6 +191,7 @@ class Retry(object): raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, + remove_headers_on_redirect=self.remove_headers_on_redirect ) params.update(kw) return type(self)(**params) diff --git a/src/pip/_vendor/urllib3/util/selectors.py b/src/pip/_vendor/urllib3/util/selectors.py deleted file mode 100644 index d75cb266b..000000000 --- a/src/pip/_vendor/urllib3/util/selectors.py +++ /dev/null @@ -1,581 +0,0 @@ -# Backport of selectors.py from Python 3.5+ to support Python < 3.4 -# Also has the behavior specified in PEP 475 which is to retry syscalls -# in the case of an EINTR error. This module is required because selectors34 -# does not follow this behavior and instead returns that no dile descriptor -# events have occurred rather than retry the syscall. The decision to drop -# support for select.devpoll is made to maintain 100% test coverage. - -import errno -import math -import select -import socket -import sys -import time -from collections import namedtuple, Mapping - -try: - monotonic = time.monotonic -except (AttributeError, ImportError): # Python 3.3< - monotonic = time.time - -EVENT_READ = (1 << 0) -EVENT_WRITE = (1 << 1) - -HAS_SELECT = True # Variable that shows whether the platform has a selector. -_SYSCALL_SENTINEL = object() # Sentinel in case a system call returns None. -_DEFAULT_SELECTOR = None - - -class SelectorError(Exception): - def __init__(self, errcode): - super(SelectorError, self).__init__() - self.errno = errcode - - def __repr__(self): - return "".format(self.errno) - - def __str__(self): - return self.__repr__() - - -def _fileobj_to_fd(fileobj): - """ Return a file descriptor from a file object. If - given an integer will simply return that integer back. """ - if isinstance(fileobj, int): - fd = fileobj - else: - try: - fd = int(fileobj.fileno()) - except (AttributeError, TypeError, ValueError): - raise ValueError("Invalid file object: {0!r}".format(fileobj)) - if fd < 0: - raise ValueError("Invalid file descriptor: {0}".format(fd)) - return fd - - -# Determine which function to use to wrap system calls because Python 3.5+ -# already handles the case when system calls are interrupted. -if sys.version_info >= (3, 5): - def _syscall_wrapper(func, _, *args, **kwargs): - """ This is the short-circuit version of the below logic - because in Python 3.5+ all system calls automatically restart - and recalculate their timeouts. """ - try: - return func(*args, **kwargs) - except (OSError, IOError, select.error) as e: - errcode = None - if hasattr(e, "errno"): - errcode = e.errno - raise SelectorError(errcode) -else: - def _syscall_wrapper(func, recalc_timeout, *args, **kwargs): - """ Wrapper function for syscalls that could fail due to EINTR. - All functions should be retried if there is time left in the timeout - in accordance with PEP 475. """ - timeout = kwargs.get("timeout", None) - if timeout is None: - expires = None - recalc_timeout = False - else: - timeout = float(timeout) - if timeout < 0.0: # Timeout less than 0 treated as no timeout. - expires = None - else: - expires = monotonic() + timeout - - args = list(args) - if recalc_timeout and "timeout" not in kwargs: - raise ValueError( - "Timeout must be in args or kwargs to be recalculated") - - result = _SYSCALL_SENTINEL - while result is _SYSCALL_SENTINEL: - try: - result = func(*args, **kwargs) - # OSError is thrown by select.select - # IOError is thrown by select.epoll.poll - # select.error is thrown by select.poll.poll - # Aren't we thankful for Python 3.x rework for exceptions? - except (OSError, IOError, select.error) as e: - # select.error wasn't a subclass of OSError in the past. - errcode = None - if hasattr(e, "errno"): - errcode = e.errno - elif hasattr(e, "args"): - errcode = e.args[0] - - # Also test for the Windows equivalent of EINTR. - is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and - errcode == errno.WSAEINTR)) - - if is_interrupt: - if expires is not None: - current_time = monotonic() - if current_time > expires: - raise OSError(errno=errno.ETIMEDOUT) - if recalc_timeout: - if "timeout" in kwargs: - kwargs["timeout"] = expires - current_time - continue - if errcode: - raise SelectorError(errcode) - else: - raise - return result - - -SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) - - -class _SelectorMapping(Mapping): - """ Mapping of file objects to selector keys """ - - def __init__(self, selector): - self._selector = selector - - def __len__(self): - return len(self._selector._fd_to_key) - - def __getitem__(self, fileobj): - try: - fd = self._selector._fileobj_lookup(fileobj) - return self._selector._fd_to_key[fd] - except KeyError: - raise KeyError("{0!r} is not registered.".format(fileobj)) - - def __iter__(self): - return iter(self._selector._fd_to_key) - - -class BaseSelector(object): - """ Abstract Selector class - - A selector supports registering file objects to be monitored - for specific I/O events. - - A file object is a file descriptor or any object with a - `fileno()` method. An arbitrary object can be attached to the - file object which can be used for example to store context info, - a callback, etc. - - A selector can use various implementations (select(), poll(), epoll(), - and kqueue()) depending on the platform. The 'DefaultSelector' class uses - the most efficient implementation for the current platform. - """ - def __init__(self): - # Maps file descriptors to keys. - self._fd_to_key = {} - - # Read-only mapping returned by get_map() - self._map = _SelectorMapping(self) - - def _fileobj_lookup(self, fileobj): - """ Return a file descriptor from a file object. - This wraps _fileobj_to_fd() to do an exhaustive - search in case the object is invalid but we still - have it in our map. Used by unregister() so we can - unregister an object that was previously registered - even if it is closed. It is also used by _SelectorMapping - """ - try: - return _fileobj_to_fd(fileobj) - except ValueError: - - # Search through all our mapped keys. - for key in self._fd_to_key.values(): - if key.fileobj is fileobj: - return key.fd - - # Raise ValueError after all. - raise - - def register(self, fileobj, events, data=None): - """ Register a file object for a set of events to monitor. """ - if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): - raise ValueError("Invalid events: {0!r}".format(events)) - - key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) - - if key.fd in self._fd_to_key: - raise KeyError("{0!r} (FD {1}) is already registered" - .format(fileobj, key.fd)) - - self._fd_to_key[key.fd] = key - return key - - def unregister(self, fileobj): - """ Unregister a file object from being monitored. """ - try: - key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) - except KeyError: - raise KeyError("{0!r} is not registered".format(fileobj)) - - # Getting the fileno of a closed socket on Windows errors with EBADF. - except socket.error as e: # Platform-specific: Windows. - if e.errno != errno.EBADF: - raise - else: - for key in self._fd_to_key.values(): - if key.fileobj is fileobj: - self._fd_to_key.pop(key.fd) - break - else: - raise KeyError("{0!r} is not registered".format(fileobj)) - return key - - def modify(self, fileobj, events, data=None): - """ Change a registered file object monitored events and data. """ - # NOTE: Some subclasses optimize this operation even further. - try: - key = self._fd_to_key[self._fileobj_lookup(fileobj)] - except KeyError: - raise KeyError("{0!r} is not registered".format(fileobj)) - - if events != key.events: - self.unregister(fileobj) - key = self.register(fileobj, events, data) - - elif data != key.data: - # Use a shortcut to update the data. - key = key._replace(data=data) - self._fd_to_key[key.fd] = key - - return key - - def select(self, timeout=None): - """ Perform the actual selection until some monitored file objects - are ready or the timeout expires. """ - raise NotImplementedError() - - def close(self): - """ Close the selector. This must be called to ensure that all - underlying resources are freed. """ - self._fd_to_key.clear() - self._map = None - - def get_key(self, fileobj): - """ Return the key associated with a registered file object. """ - mapping = self.get_map() - if mapping is None: - raise RuntimeError("Selector is closed") - try: - return mapping[fileobj] - except KeyError: - raise KeyError("{0!r} is not registered".format(fileobj)) - - def get_map(self): - """ Return a mapping of file objects to selector keys """ - return self._map - - def _key_from_fd(self, fd): - """ Return the key associated to a given file descriptor - Return None if it is not found. """ - try: - return self._fd_to_key[fd] - except KeyError: - return None - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - -# Almost all platforms have select.select() -if hasattr(select, "select"): - class SelectSelector(BaseSelector): - """ Select-based selector. """ - def __init__(self): - super(SelectSelector, self).__init__() - self._readers = set() - self._writers = set() - - def register(self, fileobj, events, data=None): - key = super(SelectSelector, self).register(fileobj, events, data) - if events & EVENT_READ: - self._readers.add(key.fd) - if events & EVENT_WRITE: - self._writers.add(key.fd) - return key - - def unregister(self, fileobj): - key = super(SelectSelector, self).unregister(fileobj) - self._readers.discard(key.fd) - self._writers.discard(key.fd) - return key - - def _select(self, r, w, timeout=None): - """ Wrapper for select.select because timeout is a positional arg """ - return select.select(r, w, [], timeout) - - def select(self, timeout=None): - # Selecting on empty lists on Windows errors out. - if not len(self._readers) and not len(self._writers): - return [] - - timeout = None if timeout is None else max(timeout, 0.0) - ready = [] - r, w, _ = _syscall_wrapper(self._select, True, self._readers, - self._writers, timeout) - r = set(r) - w = set(w) - for fd in r | w: - events = 0 - if fd in r: - events |= EVENT_READ - if fd in w: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - -if hasattr(select, "poll"): - class PollSelector(BaseSelector): - """ Poll-based selector """ - def __init__(self): - super(PollSelector, self).__init__() - self._poll = select.poll() - - def register(self, fileobj, events, data=None): - key = super(PollSelector, self).register(fileobj, events, data) - event_mask = 0 - if events & EVENT_READ: - event_mask |= select.POLLIN - if events & EVENT_WRITE: - event_mask |= select.POLLOUT - self._poll.register(key.fd, event_mask) - return key - - def unregister(self, fileobj): - key = super(PollSelector, self).unregister(fileobj) - self._poll.unregister(key.fd) - return key - - def _wrap_poll(self, timeout=None): - """ Wrapper function for select.poll.poll() so that - _syscall_wrapper can work with only seconds. """ - if timeout is not None: - if timeout <= 0: - timeout = 0 - else: - # select.poll.poll() has a resolution of 1 millisecond, - # round away from zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) - - result = self._poll.poll(timeout) - return result - - def select(self, timeout=None): - ready = [] - fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout) - for fd, event_mask in fd_events: - events = 0 - if event_mask & ~select.POLLIN: - events |= EVENT_WRITE - if event_mask & ~select.POLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - - return ready - - -if hasattr(select, "epoll"): - class EpollSelector(BaseSelector): - """ Epoll-based selector """ - def __init__(self): - super(EpollSelector, self).__init__() - self._epoll = select.epoll() - - def fileno(self): - return self._epoll.fileno() - - def register(self, fileobj, events, data=None): - key = super(EpollSelector, self).register(fileobj, events, data) - events_mask = 0 - if events & EVENT_READ: - events_mask |= select.EPOLLIN - if events & EVENT_WRITE: - events_mask |= select.EPOLLOUT - _syscall_wrapper(self._epoll.register, False, key.fd, events_mask) - return key - - def unregister(self, fileobj): - key = super(EpollSelector, self).unregister(fileobj) - try: - _syscall_wrapper(self._epoll.unregister, False, key.fd) - except SelectorError: - # This can occur when the fd was closed since registry. - pass - return key - - def select(self, timeout=None): - if timeout is not None: - if timeout <= 0: - timeout = 0.0 - else: - # select.epoll.poll() has a resolution of 1 millisecond - # but luckily takes seconds so we don't need a wrapper - # like PollSelector. Just for better rounding. - timeout = math.ceil(timeout * 1e3) * 1e-3 - timeout = float(timeout) - else: - timeout = -1.0 # epoll.poll() must have a float. - - # We always want at least 1 to ensure that select can be called - # with no file descriptors registered. Otherwise will fail. - max_events = max(len(self._fd_to_key), 1) - - ready = [] - fd_events = _syscall_wrapper(self._epoll.poll, True, - timeout=timeout, - maxevents=max_events) - for fd, event_mask in fd_events: - events = 0 - if event_mask & ~select.EPOLLIN: - events |= EVENT_WRITE - if event_mask & ~select.EPOLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - def close(self): - self._epoll.close() - super(EpollSelector, self).close() - - -if hasattr(select, "kqueue"): - class KqueueSelector(BaseSelector): - """ Kqueue / Kevent-based selector """ - def __init__(self): - super(KqueueSelector, self).__init__() - self._kqueue = select.kqueue() - - def fileno(self): - return self._kqueue.fileno() - - def register(self, fileobj, events, data=None): - key = super(KqueueSelector, self).register(fileobj, events, data) - if events & EVENT_READ: - kevent = select.kevent(key.fd, - select.KQ_FILTER_READ, - select.KQ_EV_ADD) - - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - - if events & EVENT_WRITE: - kevent = select.kevent(key.fd, - select.KQ_FILTER_WRITE, - select.KQ_EV_ADD) - - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - - return key - - def unregister(self, fileobj): - key = super(KqueueSelector, self).unregister(fileobj) - if key.events & EVENT_READ: - kevent = select.kevent(key.fd, - select.KQ_FILTER_READ, - select.KQ_EV_DELETE) - try: - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - except SelectorError: - pass - if key.events & EVENT_WRITE: - kevent = select.kevent(key.fd, - select.KQ_FILTER_WRITE, - select.KQ_EV_DELETE) - try: - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - except SelectorError: - pass - - return key - - def select(self, timeout=None): - if timeout is not None: - timeout = max(timeout, 0) - - max_events = len(self._fd_to_key) * 2 - ready_fds = {} - - kevent_list = _syscall_wrapper(self._kqueue.control, True, - None, max_events, timeout) - - for kevent in kevent_list: - fd = kevent.ident - event_mask = kevent.filter - events = 0 - if event_mask == select.KQ_FILTER_READ: - events |= EVENT_READ - if event_mask == select.KQ_FILTER_WRITE: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) - if key: - if key.fd not in ready_fds: - ready_fds[key.fd] = (key, events & key.events) - else: - old_events = ready_fds[key.fd][1] - ready_fds[key.fd] = (key, (events | old_events) & key.events) - - return list(ready_fds.values()) - - def close(self): - self._kqueue.close() - super(KqueueSelector, self).close() - - -if not hasattr(select, 'select'): # Platform-specific: AppEngine - HAS_SELECT = False - - -def _can_allocate(struct): - """ Checks that select structs can be allocated by the underlying - operating system, not just advertised by the select module. We don't - check select() because we'll be hopeful that most platforms that - don't have it available will not advertise it. (ie: GAE) """ - try: - # select.poll() objects won't fail until used. - if struct == 'poll': - p = select.poll() - p.poll(0) - - # All others will fail on allocation. - else: - getattr(select, struct)().close() - return True - except (OSError, AttributeError) as e: - return False - - -# Choose the best implementation, roughly: -# kqueue == epoll > poll > select. Devpoll not supported. (See above) -# select() also can't accept a FD > FD_SETSIZE (usually around 1024) -def DefaultSelector(): - """ This function serves as a first call for DefaultSelector to - detect if the select module is being monkey-patched incorrectly - by eventlet, greenlet, and preserve proper behavior. """ - global _DEFAULT_SELECTOR - if _DEFAULT_SELECTOR is None: - if _can_allocate('kqueue'): - _DEFAULT_SELECTOR = KqueueSelector - elif _can_allocate('epoll'): - _DEFAULT_SELECTOR = EpollSelector - elif _can_allocate('poll'): - _DEFAULT_SELECTOR = PollSelector - elif hasattr(select, 'select'): - _DEFAULT_SELECTOR = SelectSelector - else: # Platform-specific: AppEngine - raise ValueError('Platform does not have a selector') - return _DEFAULT_SELECTOR() diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py old mode 100644 new mode 100755 index dafc75b5b..325428052 --- a/src/pip/_vendor/urllib3/util/ssl_.py +++ b/src/pip/_vendor/urllib3/util/ssl_.py @@ -2,11 +2,13 @@ from __future__ import absolute_import import errno import warnings import hmac +import socket from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning +from ..packages import six SSLContext = None @@ -53,6 +55,27 @@ except ImportError: OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 OP_NO_COMPRESSION = 0x20000 + +# Python 2.7 and earlier didn't have inet_pton on non-Linux +# so we fallback on inet_aton in those cases. This means that +# we can only detect IPv4 addresses in this case. +if hasattr(socket, 'inet_pton'): + inet_pton = socket.inet_pton +else: + # Maybe we can use ipaddress if the user has urllib3[secure]? + try: + from pip._vendor import ipaddress + + def inet_pton(_, host): + if isinstance(host, six.binary_type): + host = host.decode('ascii') + return ipaddress.ip_address(host) + + except ImportError: # Platform-specific: Non-Linux + def inet_pton(_, host): + return socket.inet_aton(host) + + # A secure default. # Sources for more information on TLS ciphers: # @@ -183,7 +206,7 @@ def resolve_cert_reqs(candidate): the wrap_socket function/method from the ssl module. Defaults to :data:`ssl.CERT_NONE`. If given a string it is assumed to be the name of the constant in the - :mod:`ssl` module or its abbrevation. + :mod:`ssl` module or its abbreviation. (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. If it's neither `None` nor a string we assume it is already the numeric constant which can directly be passed to wrap_socket. @@ -325,17 +348,49 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if certfile: context.load_cert_chain(certfile, keyfile) - if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI - return context.wrap_socket(sock, server_hostname=server_hostname) - warnings.warn( - 'An HTTPS request has been made, but the SNI (Subject Name ' - 'Indication) extension to TLS is not available on this platform. ' - 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. You can upgrade to ' - 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - SNIMissingWarning - ) + # If we detect server_hostname is an IP address then the SNI + # extension should not be used according to RFC3546 Section 3.1 + # We shouldn't warn the user if SNI isn't available but we would + # not be using SNI anyways due to IP address for server_hostname. + if ((server_hostname is not None and not is_ipaddress(server_hostname)) + or IS_SECURETRANSPORT): + if HAS_SNI and server_hostname is not None: + return context.wrap_socket(sock, server_hostname=server_hostname) + + warnings.warn( + 'An HTTPS request has been made, but the SNI (Server Name ' + 'Indication) extension to TLS is not available on this platform. ' + 'This may cause the server to present an incorrect TLS ' + 'certificate, which can cause validation failures. You can upgrade to ' + 'a newer version of Python to solve this. For more information, see ' + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', + SNIMissingWarning + ) + return context.wrap_socket(sock) + + +def is_ipaddress(hostname): + """Detects whether the hostname given is an IP address. + + :param str hostname: Hostname to examine. + :return: True if the hostname is an IP address, False otherwise. + """ + if six.PY3 and isinstance(hostname, six.binary_type): + # IDN A-label bytes are ASCII compatible. + hostname = hostname.decode('ascii') + + families = [socket.AF_INET] + if hasattr(socket, 'AF_INET6'): + families.append(socket.AF_INET6) + + for af in families: + try: + inet_pton(af, hostname) + except (socket.error, ValueError, OSError): + pass + else: + return True + return False diff --git a/src/pip/_vendor/urllib3/util/timeout.py b/src/pip/_vendor/urllib3/util/timeout.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/url.py b/src/pip/_vendor/urllib3/util/url.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/wait.py b/src/pip/_vendor/urllib3/util/wait.py old mode 100644 new mode 100755 index cb396e508..fa686eff4 --- a/src/pip/_vendor/urllib3/util/wait.py +++ b/src/pip/_vendor/urllib3/util/wait.py @@ -1,40 +1,153 @@ -from .selectors import ( - HAS_SELECT, - DefaultSelector, - EVENT_READ, - EVENT_WRITE -) +import errno +from functools import partial +import select +import sys +try: + from time import monotonic +except ImportError: + from time import time as monotonic + +__all__ = ["NoWayToWaitForSocketError", "wait_for_read", "wait_for_write"] -def _wait_for_io_events(socks, events, timeout=None): - """ Waits for IO events to be available from a list of sockets - or optionally a single socket if passed in. Returns a list of - sockets that can be interacted with immediately. """ - if not HAS_SELECT: - raise ValueError('Platform does not have a selector') - if not isinstance(socks, list): - # Probably just a single socket. - if hasattr(socks, "fileno"): - socks = [socks] - # Otherwise it might be a non-list iterable. +class NoWayToWaitForSocketError(Exception): + pass + + +# How should we wait on sockets? +# +# There are two types of APIs you can use for waiting on sockets: the fancy +# modern stateful APIs like epoll/kqueue, and the older stateless APIs like +# select/poll. The stateful APIs are more efficient when you have a lots of +# sockets to keep track of, because you can set them up once and then use them +# lots of times. But we only ever want to wait on a single socket at a time +# and don't want to keep track of state, so the stateless APIs are actually +# more efficient. So we want to use select() or poll(). +# +# Now, how do we choose between select() and poll()? On traditional Unixes, +# select() has a strange calling convention that makes it slow, or fail +# altogether, for high-numbered file descriptors. The point of poll() is to fix +# that, so on Unixes, we prefer poll(). +# +# On Windows, there is no poll() (or at least Python doesn't provide a wrapper +# for it), but that's OK, because on Windows, select() doesn't have this +# strange calling convention; plain select() works fine. +# +# So: on Windows we use select(), and everywhere else we use poll(). We also +# fall back to select() in case poll() is somehow broken or missing. + +if sys.version_info >= (3, 5): + # Modern Python, that retries syscalls by default + def _retry_on_intr(fn, timeout): + return fn(timeout) +else: + # Old and broken Pythons. + def _retry_on_intr(fn, timeout): + if timeout is not None and timeout <= 0: + return fn(timeout) + + if timeout is None: + deadline = float("inf") else: - socks = list(socks) - with DefaultSelector() as selector: - for sock in socks: - selector.register(sock, events) - return [key[0].fileobj for key in - selector.select(timeout) if key[1] & events] + deadline = monotonic() + timeout + + while True: + try: + return fn(timeout) + # OSError for 3 <= pyver < 3.5, select.error for pyver <= 2.7 + except (OSError, select.error) as e: + # 'e.args[0]' incantation works for both OSError and select.error + if e.args[0] != errno.EINTR: + raise + else: + timeout = deadline - monotonic() + if timeout < 0: + timeout = 0 + if timeout == float("inf"): + timeout = None + continue -def wait_for_read(socks, timeout=None): - """ Waits for reading to be available from a list of sockets - or optionally a single socket if passed in. Returns a list of - sockets that can be read from immediately. """ - return _wait_for_io_events(socks, EVENT_READ, timeout) +def select_wait_for_socket(sock, read=False, write=False, timeout=None): + if not read and not write: + raise RuntimeError("must specify at least one of read=True, write=True") + rcheck = [] + wcheck = [] + if read: + rcheck.append(sock) + if write: + wcheck.append(sock) + # When doing a non-blocking connect, most systems signal success by + # marking the socket writable. Windows, though, signals success by marked + # it as "exceptional". We paper over the difference by checking the write + # sockets for both conditions. (The stdlib selectors module does the same + # thing.) + fn = partial(select.select, rcheck, wcheck, wcheck) + rready, wready, xready = _retry_on_intr(fn, timeout) + return bool(rready or wready or xready) -def wait_for_write(socks, timeout=None): - """ Waits for writing to be available from a list of sockets - or optionally a single socket if passed in. Returns a list of - sockets that can be written to immediately. """ - return _wait_for_io_events(socks, EVENT_WRITE, timeout) +def poll_wait_for_socket(sock, read=False, write=False, timeout=None): + if not read and not write: + raise RuntimeError("must specify at least one of read=True, write=True") + mask = 0 + if read: + mask |= select.POLLIN + if write: + mask |= select.POLLOUT + poll_obj = select.poll() + poll_obj.register(sock, mask) + + # For some reason, poll() takes timeout in milliseconds + def do_poll(t): + if t is not None: + t *= 1000 + return poll_obj.poll(t) + + return bool(_retry_on_intr(do_poll, timeout)) + + +def null_wait_for_socket(*args, **kwargs): + raise NoWayToWaitForSocketError("no select-equivalent available") + + +def _have_working_poll(): + # Apparently some systems have a select.poll that fails as soon as you try + # to use it, either due to strange configuration or broken monkeypatching + # from libraries like eventlet/greenlet. + try: + poll_obj = select.poll() + poll_obj.poll(0) + except (AttributeError, OSError): + return False + else: + return True + + +def wait_for_socket(*args, **kwargs): + # We delay choosing which implementation to use until the first time we're + # called. We could do it at import time, but then we might make the wrong + # decision if someone goes wild with monkeypatching select.poll after + # we're imported. + global wait_for_socket + if _have_working_poll(): + wait_for_socket = poll_wait_for_socket + elif hasattr(select, "select"): + wait_for_socket = select_wait_for_socket + else: # Platform-specific: Appengine. + wait_for_socket = null_wait_for_socket + return wait_for_socket(*args, **kwargs) + + +def wait_for_read(sock, timeout=None): + """ Waits for reading to be available on a given socket. + Returns True if the socket is readable, or False if the timeout expired. + """ + return wait_for_socket(sock, read=True, timeout=timeout) + + +def wait_for_write(sock, timeout=None): + """ Waits for writing to be available on a given socket. + Returns True if the socket is readable, or False if the timeout expired. + """ + return wait_for_socket(sock, write=True, timeout=timeout) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index c0af28813..f3adf70f7 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -16,7 +16,7 @@ retrying==1.3.3 requests==2.18.4 chardet==3.0.4 idna==2.6 - urllib3==1.22 + urllib3==1.23 certifi==2018.4.16 setuptools==39.2.0 webencodings==0.5.1 From 7b97a38c6bf91e45fac8872cbc3fbd0acf8f57e2 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:39:29 +0530 Subject: [PATCH 009/129] Update idna to 2.7 --- news/idna.vendor | 1 + src/pip/_vendor/idna/__init__.py | 0 src/pip/_vendor/idna/codec.py | 0 src/pip/_vendor/idna/compat.py | 0 src/pip/_vendor/idna/core.py | 40 +- src/pip/_vendor/idna/idnadata.py | 398 +++++++- src/pip/_vendor/idna/intranges.py | 0 src/pip/_vendor/idna/package_data.py | 2 +- src/pip/_vendor/idna/uts46data.py | 1279 ++++++++++++++++++-------- src/pip/_vendor/vendor.txt | 2 +- 10 files changed, 1294 insertions(+), 428 deletions(-) create mode 100644 news/idna.vendor mode change 100644 => 100755 src/pip/_vendor/idna/__init__.py mode change 100644 => 100755 src/pip/_vendor/idna/codec.py mode change 100644 => 100755 src/pip/_vendor/idna/compat.py mode change 100644 => 100755 src/pip/_vendor/idna/core.py mode change 100644 => 100755 src/pip/_vendor/idna/idnadata.py mode change 100644 => 100755 src/pip/_vendor/idna/intranges.py mode change 100644 => 100755 src/pip/_vendor/idna/package_data.py mode change 100644 => 100755 src/pip/_vendor/idna/uts46data.py diff --git a/news/idna.vendor b/news/idna.vendor new file mode 100644 index 000000000..e0491b2d6 --- /dev/null +++ b/news/idna.vendor @@ -0,0 +1 @@ +Update idna to 2.7 diff --git a/src/pip/_vendor/idna/__init__.py b/src/pip/_vendor/idna/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/codec.py b/src/pip/_vendor/idna/codec.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/compat.py b/src/pip/_vendor/idna/compat.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/core.py b/src/pip/_vendor/idna/core.py old mode 100644 new mode 100755 index b55b66456..090c2c18d --- a/src/pip/_vendor/idna/core.py +++ b/src/pip/_vendor/idna/core.py @@ -34,7 +34,11 @@ class InvalidCodepointContext(IDNAError): def _combining_class(cp): - return unicodedata.combining(unichr(cp)) + v = unicodedata.combining(unichr(cp)) + if v == 0: + if not unicodedata.name(unichr(cp)): + raise ValueError("Unknown character in unicodedata") + return v def _is_script(cp, script): return intranges_contain(ord(cp), idnadata.scripts[script]) @@ -71,7 +75,6 @@ def check_bidi(label, check_ltr=False): raise IDNABidiError('Unknown directionality in label {0} at position {1}'.format(repr(label), idx)) if direction in ['R', 'AL', 'AN']: bidi_label = True - break if not bidi_label and not check_ltr: return True @@ -244,8 +247,13 @@ def check_label(label): if intranges_contain(cp_value, idnadata.codepoint_classes['PVALID']): continue elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']): - if not valid_contextj(label, pos): - raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label))) + try: + if not valid_contextj(label, pos): + raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format( + _unot(cp_value), pos+1, repr(label))) + except ValueError: + raise IDNAError('Unknown codepoint adjacent to joiner {0} at position {1} in {2}'.format( + _unot(cp_value), pos+1, repr(label))) elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']): if not valid_contexto(label, pos): raise InvalidCodepointContext('Codepoint {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label))) @@ -317,10 +325,10 @@ def uts46_remap(domain, std3_rules=True, transitional=False): replacement = uts46row[2] if len(uts46row) == 3 else None if (status == "V" or (status == "D" and not transitional) or - (status == "3" and std3_rules and replacement is None)): + (status == "3" and not std3_rules and replacement is None)): output += char elif replacement is not None and (status == "M" or - (status == "3" and std3_rules) or + (status == "3" and not std3_rules) or (status == "D" and transitional)): output += replacement elif status != "I": @@ -344,15 +352,17 @@ def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False): labels = s.split('.') else: labels = _unicode_dots_re.split(s) - while labels and not labels[0]: - del labels[0] - if not labels: + if not labels or labels == ['']: raise IDNAError('Empty domain') if labels[-1] == '': del labels[-1] trailing_dot = True for label in labels: - result.append(alabel(label)) + s = alabel(label) + if s: + result.append(s) + else: + raise IDNAError('Empty label') if trailing_dot: result.append(b'') s = b'.'.join(result) @@ -373,15 +383,17 @@ def decode(s, strict=False, uts46=False, std3_rules=False): labels = _unicode_dots_re.split(s) else: labels = s.split(u'.') - while labels and not labels[0]: - del labels[0] - if not labels: + if not labels or labels == ['']: raise IDNAError('Empty domain') if not labels[-1]: del labels[-1] trailing_dot = True for label in labels: - result.append(ulabel(label)) + s = ulabel(label) + if s: + result.append(s) + else: + raise IDNAError('Empty label') if trailing_dot: result.append(u'') return u'.'.join(result) diff --git a/src/pip/_vendor/idna/idnadata.py b/src/pip/_vendor/idna/idnadata.py old mode 100644 new mode 100755 index c48f1b504..17974e233 --- a/src/pip/_vendor/idna/idnadata.py +++ b/src/pip/_vendor/idna/idnadata.py @@ -1,11 +1,12 @@ # This file is automatically generated by tools/idna-data -__version__ = "6.3.0" +__version__ = "10.0.0" scripts = { 'Greek': ( 0x37000000374, 0x37500000378, 0x37a0000037e, + 0x37f00000380, 0x38400000385, 0x38600000387, 0x3880000038b, @@ -34,7 +35,9 @@ scripts = { 0x1ff200001ff5, 0x1ff600001fff, 0x212600002127, - 0x101400001018b, + 0xab650000ab66, + 0x101400001018f, + 0x101a0000101a1, 0x1d2000001d246, ), 'Han': ( @@ -46,12 +49,14 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009fcd, + 0x4e0000009feb, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, 0x2a7000002b735, 0x2b7400002b81e, + 0x2b8200002cea2, + 0x2ceb00002ebe1, 0x2f8000002fa1e, ), 'Hebrew': ( @@ -68,7 +73,7 @@ scripts = { 'Hiragana': ( 0x304100003097, 0x309d000030a0, - 0x1b0010001b002, + 0x1b0010001b11f, 0x1f2000001f201, ), 'Katakana': ( @@ -88,6 +93,7 @@ joining_types = { 0x602: 85, 0x603: 85, 0x604: 85, + 0x605: 85, 0x608: 85, 0x60b: 85, 0x620: 68, @@ -365,7 +371,7 @@ joining_types = { 0x844: 68, 0x845: 68, 0x846: 82, - 0x847: 68, + 0x847: 82, 0x848: 68, 0x849: 82, 0x84a: 68, @@ -373,7 +379,7 @@ joining_types = { 0x84c: 68, 0x84d: 68, 0x84e: 68, - 0x84f: 82, + 0x84f: 68, 0x850: 68, 0x851: 68, 0x852: 68, @@ -383,7 +389,19 @@ joining_types = { 0x856: 85, 0x857: 85, 0x858: 85, + 0x860: 68, + 0x861: 85, + 0x862: 68, + 0x863: 68, + 0x864: 68, + 0x865: 68, + 0x866: 85, + 0x867: 82, + 0x868: 68, + 0x869: 82, + 0x86a: 82, 0x8a0: 68, + 0x8a1: 68, 0x8a2: 68, 0x8a3: 68, 0x8a4: 68, @@ -395,6 +413,23 @@ joining_types = { 0x8aa: 82, 0x8ab: 82, 0x8ac: 82, + 0x8ad: 85, + 0x8ae: 82, + 0x8af: 68, + 0x8b0: 68, + 0x8b1: 82, + 0x8b2: 82, + 0x8b3: 68, + 0x8b4: 68, + 0x8b6: 68, + 0x8b7: 68, + 0x8b8: 68, + 0x8b9: 82, + 0x8ba: 68, + 0x8bb: 68, + 0x8bc: 68, + 0x8bd: 68, + 0x8e2: 85, 0x1806: 85, 0x1807: 68, 0x180a: 67, @@ -492,8 +527,8 @@ joining_types = { 0x1882: 85, 0x1883: 85, 0x1884: 85, - 0x1885: 85, - 0x1886: 85, + 0x1885: 84, + 0x1886: 84, 0x1887: 68, 0x1888: 68, 0x1889: 68, @@ -531,6 +566,7 @@ joining_types = { 0x18aa: 68, 0x200c: 85, 0x200d: 67, + 0x202f: 85, 0x2066: 85, 0x2067: 85, 0x2068: 85, @@ -587,6 +623,141 @@ joining_types = { 0xa871: 68, 0xa872: 76, 0xa873: 85, + 0x10ac0: 68, + 0x10ac1: 68, + 0x10ac2: 68, + 0x10ac3: 68, + 0x10ac4: 68, + 0x10ac5: 82, + 0x10ac6: 85, + 0x10ac7: 82, + 0x10ac8: 85, + 0x10ac9: 82, + 0x10aca: 82, + 0x10acb: 85, + 0x10acc: 85, + 0x10acd: 76, + 0x10ace: 82, + 0x10acf: 82, + 0x10ad0: 82, + 0x10ad1: 82, + 0x10ad2: 82, + 0x10ad3: 68, + 0x10ad4: 68, + 0x10ad5: 68, + 0x10ad6: 68, + 0x10ad7: 76, + 0x10ad8: 68, + 0x10ad9: 68, + 0x10ada: 68, + 0x10adb: 68, + 0x10adc: 68, + 0x10add: 82, + 0x10ade: 68, + 0x10adf: 68, + 0x10ae0: 68, + 0x10ae1: 82, + 0x10ae2: 85, + 0x10ae3: 85, + 0x10ae4: 82, + 0x10aeb: 68, + 0x10aec: 68, + 0x10aed: 68, + 0x10aee: 68, + 0x10aef: 82, + 0x10b80: 68, + 0x10b81: 82, + 0x10b82: 68, + 0x10b83: 82, + 0x10b84: 82, + 0x10b85: 82, + 0x10b86: 68, + 0x10b87: 68, + 0x10b88: 68, + 0x10b89: 82, + 0x10b8a: 68, + 0x10b8b: 68, + 0x10b8c: 82, + 0x10b8d: 68, + 0x10b8e: 82, + 0x10b8f: 82, + 0x10b90: 68, + 0x10b91: 82, + 0x10ba9: 82, + 0x10baa: 82, + 0x10bab: 82, + 0x10bac: 82, + 0x10bad: 68, + 0x10bae: 68, + 0x10baf: 85, + 0x1e900: 68, + 0x1e901: 68, + 0x1e902: 68, + 0x1e903: 68, + 0x1e904: 68, + 0x1e905: 68, + 0x1e906: 68, + 0x1e907: 68, + 0x1e908: 68, + 0x1e909: 68, + 0x1e90a: 68, + 0x1e90b: 68, + 0x1e90c: 68, + 0x1e90d: 68, + 0x1e90e: 68, + 0x1e90f: 68, + 0x1e910: 68, + 0x1e911: 68, + 0x1e912: 68, + 0x1e913: 68, + 0x1e914: 68, + 0x1e915: 68, + 0x1e916: 68, + 0x1e917: 68, + 0x1e918: 68, + 0x1e919: 68, + 0x1e91a: 68, + 0x1e91b: 68, + 0x1e91c: 68, + 0x1e91d: 68, + 0x1e91e: 68, + 0x1e91f: 68, + 0x1e920: 68, + 0x1e921: 68, + 0x1e922: 68, + 0x1e923: 68, + 0x1e924: 68, + 0x1e925: 68, + 0x1e926: 68, + 0x1e927: 68, + 0x1e928: 68, + 0x1e929: 68, + 0x1e92a: 68, + 0x1e92b: 68, + 0x1e92c: 68, + 0x1e92d: 68, + 0x1e92e: 68, + 0x1e92f: 68, + 0x1e930: 68, + 0x1e931: 68, + 0x1e932: 68, + 0x1e933: 68, + 0x1e934: 68, + 0x1e935: 68, + 0x1e936: 68, + 0x1e937: 68, + 0x1e938: 68, + 0x1e939: 68, + 0x1e93a: 68, + 0x1e93b: 68, + 0x1e93c: 68, + 0x1e93d: 68, + 0x1e93e: 68, + 0x1e93f: 68, + 0x1e940: 68, + 0x1e941: 68, + 0x1e942: 68, + 0x1e943: 68, } codepoint_classes = { 'PVALID': ( @@ -858,6 +1029,10 @@ codepoint_classes = { 0x52300000524, 0x52500000526, 0x52700000528, + 0x5290000052a, + 0x52b0000052c, + 0x52d0000052e, + 0x52f00000530, 0x5590000055a, 0x56100000587, 0x591000005be, @@ -881,15 +1056,14 @@ codepoint_classes = { 0x7c0000007f6, 0x8000000082e, 0x8400000085c, - 0x8a0000008a1, - 0x8a2000008ad, - 0x8e4000008ff, - 0x90000000958, + 0x8600000086b, + 0x8a0000008b5, + 0x8b6000008be, + 0x8d4000008e2, + 0x8e300000958, 0x96000000964, 0x96600000970, - 0x97100000978, - 0x97900000980, - 0x98100000984, + 0x97100000984, 0x9850000098d, 0x98f00000991, 0x993000009a9, @@ -902,6 +1076,7 @@ codepoint_classes = { 0x9d7000009d8, 0x9e0000009e4, 0x9e6000009f2, + 0x9fc000009fd, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -930,6 +1105,7 @@ codepoint_classes = { 0xad000000ad1, 0xae000000ae4, 0xae600000af0, + 0xaf900000b00, 0xb0100000b04, 0xb0500000b0d, 0xb0f00000b11, @@ -960,20 +1136,19 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0100000c04, + 0xc0000000c04, 0xc0500000c0d, 0xc0e00000c11, 0xc1200000c29, - 0xc2a00000c34, - 0xc3500000c3a, + 0xc2a00000c3a, 0xc3d00000c45, 0xc4600000c49, 0xc4a00000c4e, 0xc5500000c57, - 0xc5800000c5a, + 0xc5800000c5b, 0xc6000000c64, 0xc6600000c70, - 0xc8200000c84, + 0xc8000000c84, 0xc8500000c8d, 0xc8e00000c91, 0xc9200000ca9, @@ -987,15 +1162,14 @@ codepoint_classes = { 0xce000000ce4, 0xce600000cf0, 0xcf100000cf3, - 0xd0200000d04, + 0xd0000000d04, 0xd0500000d0d, 0xd0e00000d11, - 0xd1200000d3b, - 0xd3d00000d45, + 0xd1200000d45, 0xd4600000d49, 0xd4a00000d4f, - 0xd5700000d58, - 0xd6000000d64, + 0xd5400000d58, + 0xd5f00000d64, 0xd6600000d70, 0xd7a00000d80, 0xd8200000d84, @@ -1008,6 +1182,7 @@ codepoint_classes = { 0xdcf00000dd5, 0xdd600000dd7, 0xdd800000de0, + 0xde600000df0, 0xdf200000df4, 0xe0100000e33, 0xe3400000e3b, @@ -1082,11 +1257,12 @@ codepoint_classes = { 0x13180000135b, 0x135d00001360, 0x138000001390, - 0x13a0000013f5, + 0x13a0000013f6, 0x14010000166d, 0x166f00001680, 0x16810000169b, 0x16a0000016eb, + 0x16f1000016f9, 0x17000000170d, 0x170e00001715, 0x172000001735, @@ -1103,7 +1279,7 @@ codepoint_classes = { 0x182000001878, 0x1880000018ab, 0x18b0000018f6, - 0x19000000191d, + 0x19000000191f, 0x19200000192c, 0x19300000193c, 0x19460000196e, @@ -1117,6 +1293,7 @@ codepoint_classes = { 0x1a7f00001a8a, 0x1a9000001a9a, 0x1aa700001aa8, + 0x1ab000001abe, 0x1b0000001b4c, 0x1b5000001b5a, 0x1b6b00001b74, @@ -1125,15 +1302,15 @@ codepoint_classes = { 0x1c4000001c4a, 0x1c4d00001c7e, 0x1cd000001cd3, - 0x1cd400001cf7, + 0x1cd400001cfa, 0x1d0000001d2c, 0x1d2f00001d30, 0x1d3b00001d3c, 0x1d4e00001d4f, 0x1d6b00001d78, 0x1d7900001d9b, - 0x1dc000001de7, - 0x1dfc00001e00, + 0x1dc000001dfa, + 0x1dfb00001e00, 0x1e0100001e02, 0x1e0300001e04, 0x1e0500001e06, @@ -1367,11 +1544,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312e, + 0x31050000312f, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009fcd, + 0x4e0000009feb, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1413,7 +1590,9 @@ codepoint_classes = { 0xa6930000a694, 0xa6950000a696, 0xa6970000a698, - 0xa69f0000a6e6, + 0xa6990000a69a, + 0xa69b0000a69c, + 0xa69e0000a6e6, 0xa6f00000a6f2, 0xa7170000a720, 0xa7230000a724, @@ -1463,30 +1642,39 @@ codepoint_classes = { 0xa7850000a786, 0xa7870000a789, 0xa78c0000a78d, - 0xa78e0000a78f, + 0xa78e0000a790, 0xa7910000a792, - 0xa7930000a794, + 0xa7930000a796, + 0xa7970000a798, + 0xa7990000a79a, + 0xa79b0000a79c, + 0xa79d0000a79e, + 0xa79f0000a7a0, 0xa7a10000a7a2, 0xa7a30000a7a4, 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7b50000a7b6, + 0xa7b70000a7b8, + 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, - 0xa8800000a8c5, + 0xa8800000a8c6, 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, + 0xa8fd0000a8fe, 0xa9000000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, + 0xa9e00000a9ff, 0xaa000000aa37, 0xaa400000aa4e, 0xaa500000aa5a, 0xaa600000aa77, - 0xaa7a0000aa7c, - 0xaa800000aac3, + 0xaa7a0000aac3, 0xaadb0000aade, 0xaae00000aaf0, 0xaaf20000aaf7, @@ -1495,6 +1683,8 @@ codepoint_classes = { 0xab110000ab17, 0xab200000ab27, 0xab280000ab2f, + 0xab300000ab5b, + 0xab600000ab66, 0xabc00000abeb, 0xabec0000abee, 0xabf00000abfa, @@ -1507,7 +1697,7 @@ codepoint_classes = { 0xfa230000fa25, 0xfa270000fa2a, 0xfb1e0000fb1f, - 0xfe200000fe27, + 0xfe200000fe30, 0xfe730000fe74, 0x100000001000c, 0x1000d00010027, @@ -1519,20 +1709,32 @@ codepoint_classes = { 0x101fd000101fe, 0x102800001029d, 0x102a0000102d1, - 0x103000001031f, - 0x1033000010341, + 0x102e0000102e1, + 0x1030000010320, + 0x1032d00010341, 0x103420001034a, + 0x103500001037b, 0x103800001039e, 0x103a0000103c4, 0x103c8000103d0, 0x104280001049e, 0x104a0000104aa, + 0x104d8000104fc, + 0x1050000010528, + 0x1053000010564, + 0x1060000010737, + 0x1074000010756, + 0x1076000010768, 0x1080000010806, 0x1080800010809, 0x1080a00010836, 0x1083700010839, 0x1083c0001083d, 0x1083f00010856, + 0x1086000010877, + 0x108800001089f, + 0x108e0000108f3, + 0x108f4000108f6, 0x1090000010916, 0x109200001093a, 0x10980000109b8, @@ -1545,31 +1747,137 @@ codepoint_classes = { 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, + 0x10a8000010a9d, + 0x10ac000010ac8, + 0x10ac900010ae7, 0x10b0000010b36, 0x10b4000010b56, 0x10b6000010b73, + 0x10b8000010b92, 0x10c0000010c49, + 0x10cc000010cf3, 0x1100000011047, 0x1106600011070, - 0x11080000110bb, + 0x1107f000110bb, 0x110d0000110e9, 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1115000011174, + 0x1117600011177, 0x11180000111c5, - 0x111d0000111da, + 0x111ca000111cd, + 0x111d0000111db, + 0x111dc000111dd, + 0x1120000011212, + 0x1121300011238, + 0x1123e0001123f, + 0x1128000011287, + 0x1128800011289, + 0x1128a0001128e, + 0x1128f0001129e, + 0x1129f000112a9, + 0x112b0000112eb, + 0x112f0000112fa, + 0x1130000011304, + 0x113050001130d, + 0x1130f00011311, + 0x1131300011329, + 0x1132a00011331, + 0x1133200011334, + 0x113350001133a, + 0x1133c00011345, + 0x1134700011349, + 0x1134b0001134e, + 0x1135000011351, + 0x1135700011358, + 0x1135d00011364, + 0x113660001136d, + 0x1137000011375, + 0x114000001144b, + 0x114500001145a, + 0x11480000114c6, + 0x114c7000114c8, + 0x114d0000114da, + 0x11580000115b6, + 0x115b8000115c1, + 0x115d8000115de, + 0x1160000011641, + 0x1164400011645, + 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x120000001236f, + 0x117000001171a, + 0x1171d0001172c, + 0x117300001173a, + 0x118c0000118ea, + 0x118ff00011900, + 0x11a0000011a3f, + 0x11a4700011a48, + 0x11a5000011a84, + 0x11a8600011a9a, + 0x11ac000011af9, + 0x11c0000011c09, + 0x11c0a00011c37, + 0x11c3800011c41, + 0x11c5000011c5a, + 0x11c7200011c90, + 0x11c9200011ca8, + 0x11ca900011cb7, + 0x11d0000011d07, + 0x11d0800011d0a, + 0x11d0b00011d37, + 0x11d3a00011d3b, + 0x11d3c00011d3e, + 0x11d3f00011d48, + 0x11d5000011d5a, + 0x120000001239a, + 0x1248000012544, 0x130000001342f, + 0x1440000014647, 0x1680000016a39, + 0x16a4000016a5f, + 0x16a6000016a6a, + 0x16ad000016aee, + 0x16af000016af5, + 0x16b0000016b37, + 0x16b4000016b44, + 0x16b5000016b5a, + 0x16b6300016b78, + 0x16b7d00016b90, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, - 0x1b0000001b002, + 0x16fe000016fe2, + 0x17000000187ed, + 0x1880000018af3, + 0x1b0000001b11f, + 0x1b1700001b2fc, + 0x1bc000001bc6b, + 0x1bc700001bc7d, + 0x1bc800001bc89, + 0x1bc900001bc9a, + 0x1bc9d0001bc9f, + 0x1da000001da37, + 0x1da3b0001da6d, + 0x1da750001da76, + 0x1da840001da85, + 0x1da9b0001daa0, + 0x1daa10001dab0, + 0x1e0000001e007, + 0x1e0080001e019, + 0x1e01b0001e022, + 0x1e0230001e025, + 0x1e0260001e02b, + 0x1e8000001e8c5, + 0x1e8d00001e8d7, + 0x1e9220001e94b, + 0x1e9500001e95a, 0x200000002a6d7, 0x2a7000002b735, 0x2b7400002b81e, + 0x2b8200002cea2, + 0x2ceb00002ebe1, ), 'CONTEXTJ': ( 0x200c0000200e, diff --git a/src/pip/_vendor/idna/intranges.py b/src/pip/_vendor/idna/intranges.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/package_data.py b/src/pip/_vendor/idna/package_data.py old mode 100644 new mode 100755 index fc3313927..39c192bae --- a/src/pip/_vendor/idna/package_data.py +++ b/src/pip/_vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.6' +__version__ = '2.7' diff --git a/src/pip/_vendor/idna/uts46data.py b/src/pip/_vendor/idna/uts46data.py old mode 100644 new mode 100755 index f9b3236f4..79731cb9e --- a/src/pip/_vendor/idna/uts46data.py +++ b/src/pip/_vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "6.3.0" +__version__ = "10.0.0" def _seg_0(): return [ (0x0, '3'), @@ -635,7 +635,8 @@ def _seg_6(): (0x37A, '3', u' ι'), (0x37B, 'V'), (0x37E, '3', u';'), - (0x37F, 'X'), + (0x37F, 'M', u'ϳ'), + (0x380, 'X'), (0x384, '3', u' ́'), (0x385, '3', u' ̈́'), (0x386, 'M', u'ά'), @@ -730,11 +731,11 @@ def _seg_6(): (0x400, 'M', u'ѐ'), (0x401, 'M', u'ё'), (0x402, 'M', u'ђ'), - (0x403, 'M', u'ѓ'), ] def _seg_7(): return [ + (0x403, 'M', u'ѓ'), (0x404, 'M', u'є'), (0x405, 'M', u'ѕ'), (0x406, 'M', u'і'), @@ -834,11 +835,11 @@ def _seg_7(): (0x49B, 'V'), (0x49C, 'M', u'ҝ'), (0x49D, 'V'), - (0x49E, 'M', u'ҟ'), ] def _seg_8(): return [ + (0x49E, 'M', u'ҟ'), (0x49F, 'V'), (0x4A0, 'M', u'ҡ'), (0x4A1, 'V'), @@ -938,11 +939,11 @@ def _seg_8(): (0x500, 'M', u'ԁ'), (0x501, 'V'), (0x502, 'M', u'ԃ'), - (0x503, 'V'), ] def _seg_9(): return [ + (0x503, 'V'), (0x504, 'M', u'ԅ'), (0x505, 'V'), (0x506, 'M', u'ԇ'), @@ -979,7 +980,15 @@ def _seg_9(): (0x525, 'V'), (0x526, 'M', u'ԧ'), (0x527, 'V'), - (0x528, 'X'), + (0x528, 'M', u'ԩ'), + (0x529, 'V'), + (0x52A, 'M', u'ԫ'), + (0x52B, 'V'), + (0x52C, 'M', u'ԭ'), + (0x52D, 'V'), + (0x52E, 'M', u'ԯ'), + (0x52F, 'V'), + (0x530, 'X'), (0x531, 'M', u'ա'), (0x532, 'M', u'բ'), (0x533, 'M', u'գ'), @@ -1026,7 +1035,7 @@ def _seg_9(): (0x588, 'X'), (0x589, 'V'), (0x58B, 'X'), - (0x58F, 'V'), + (0x58D, 'V'), (0x590, 'X'), (0x591, 'V'), (0x5C8, 'X'), @@ -1034,6 +1043,10 @@ def _seg_9(): (0x5EB, 'X'), (0x5F0, 'V'), (0x5F5, 'X'), + ] + +def _seg_10(): + return [ (0x606, 'V'), (0x61C, 'X'), (0x61E, 'V'), @@ -1043,10 +1056,6 @@ def _seg_9(): (0x678, 'M', u'يٴ'), (0x679, 'V'), (0x6DD, 'X'), - ] - -def _seg_10(): - return [ (0x6DE, 'V'), (0x70E, 'X'), (0x710, 'V'), @@ -1063,13 +1072,15 @@ def _seg_10(): (0x85C, 'X'), (0x85E, 'V'), (0x85F, 'X'), + (0x860, 'V'), + (0x86B, 'X'), (0x8A0, 'V'), - (0x8A1, 'X'), - (0x8A2, 'V'), - (0x8AD, 'X'), - (0x8E4, 'V'), - (0x8FF, 'X'), - (0x900, 'V'), + (0x8B5, 'X'), + (0x8B6, 'V'), + (0x8BE, 'X'), + (0x8D4, 'V'), + (0x8E2, 'X'), + (0x8E3, 'V'), (0x958, 'M', u'क़'), (0x959, 'M', u'ख़'), (0x95A, 'M', u'ग़'), @@ -1079,10 +1090,6 @@ def _seg_10(): (0x95E, 'M', u'फ़'), (0x95F, 'M', u'य़'), (0x960, 'V'), - (0x978, 'X'), - (0x979, 'V'), - (0x980, 'X'), - (0x981, 'V'), (0x984, 'X'), (0x985, 'V'), (0x98D, 'X'), @@ -1111,7 +1118,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FC, 'X'), + (0x9FE, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1140,6 +1147,10 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + ] + +def _seg_11(): + return [ (0xA59, 'M', u'ਖ਼'), (0xA5A, 'M', u'ਗ਼'), (0xA5B, 'M', u'ਜ਼'), @@ -1147,10 +1158,6 @@ def _seg_10(): (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), - ] - -def _seg_11(): - return [ (0xA66, 'V'), (0xA76, 'X'), (0xA81, 'V'), @@ -1179,6 +1186,8 @@ def _seg_11(): (0xAE4, 'X'), (0xAE6, 'V'), (0xAF2, 'X'), + (0xAF9, 'V'), + (0xB00, 'X'), (0xB01, 'V'), (0xB04, 'X'), (0xB05, 'V'), @@ -1240,8 +1249,12 @@ def _seg_11(): (0xBD8, 'X'), (0xBE6, 'V'), (0xBFB, 'X'), - (0xC01, 'V'), + (0xC00, 'V'), (0xC04, 'X'), + ] + +def _seg_12(): + return [ (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), @@ -1249,12 +1262,6 @@ def _seg_11(): (0xC12, 'V'), (0xC29, 'X'), (0xC2A, 'V'), - (0xC34, 'X'), - (0xC35, 'V'), - ] - -def _seg_12(): - return [ (0xC3A, 'X'), (0xC3D, 'V'), (0xC45, 'X'), @@ -1265,14 +1272,12 @@ def _seg_12(): (0xC55, 'V'), (0xC57, 'X'), (0xC58, 'V'), - (0xC5A, 'X'), + (0xC5B, 'X'), (0xC60, 'V'), (0xC64, 'X'), (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC80, 'X'), - (0xC82, 'V'), (0xC84, 'X'), (0xC85, 'V'), (0xC8D, 'X'), @@ -1300,27 +1305,21 @@ def _seg_12(): (0xCF0, 'X'), (0xCF1, 'V'), (0xCF3, 'X'), - (0xD02, 'V'), + (0xD00, 'V'), (0xD04, 'X'), (0xD05, 'V'), (0xD0D, 'X'), (0xD0E, 'V'), (0xD11, 'X'), (0xD12, 'V'), - (0xD3B, 'X'), - (0xD3D, 'V'), (0xD45, 'X'), (0xD46, 'V'), (0xD49, 'X'), (0xD4A, 'V'), - (0xD4F, 'X'), - (0xD57, 'V'), - (0xD58, 'X'), - (0xD60, 'V'), + (0xD50, 'X'), + (0xD54, 'V'), (0xD64, 'X'), (0xD66, 'V'), - (0xD76, 'X'), - (0xD79, 'V'), (0xD80, 'X'), (0xD82, 'V'), (0xD84, 'X'), @@ -1342,6 +1341,8 @@ def _seg_12(): (0xDD7, 'X'), (0xDD8, 'V'), (0xDE0, 'X'), + (0xDE6, 'V'), + (0xDF0, 'X'), (0xDF2, 'V'), (0xDF5, 'X'), (0xE01, 'V'), @@ -1354,11 +1355,11 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - (0xE87, 'V'), ] def _seg_13(): return [ + (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), (0xE8B, 'X'), @@ -1458,11 +1459,11 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - (0x1258, 'V'), ] def _seg_14(): return [ + (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), (0x125E, 'X'), @@ -1493,13 +1494,20 @@ def _seg_14(): (0x1380, 'V'), (0x139A, 'X'), (0x13A0, 'V'), - (0x13F5, 'X'), + (0x13F6, 'X'), + (0x13F8, 'M', u'Ᏸ'), + (0x13F9, 'M', u'Ᏹ'), + (0x13FA, 'M', u'Ᏺ'), + (0x13FB, 'M', u'Ᏻ'), + (0x13FC, 'M', u'Ᏼ'), + (0x13FD, 'M', u'Ᏽ'), + (0x13FE, 'X'), (0x1400, 'V'), (0x1680, 'X'), (0x1681, 'V'), (0x169D, 'X'), (0x16A0, 'V'), - (0x16F1, 'X'), + (0x16F9, 'X'), (0x1700, 'V'), (0x170D, 'X'), (0x170E, 'V'), @@ -1536,7 +1544,7 @@ def _seg_14(): (0x18B0, 'V'), (0x18F6, 'X'), (0x1900, 'V'), - (0x191D, 'X'), + (0x191F, 'X'), (0x1920, 'V'), (0x192C, 'X'), (0x1930, 'V'), @@ -1555,6 +1563,10 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), + ] + +def _seg_15(): + return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1563,12 +1575,10 @@ def _seg_14(): (0x1A8A, 'X'), (0x1A90, 'V'), (0x1A9A, 'X'), - ] - -def _seg_15(): - return [ (0x1AA0, 'V'), (0x1AAE, 'X'), + (0x1AB0, 'V'), + (0x1ABF, 'X'), (0x1B00, 'V'), (0x1B4C, 'X'), (0x1B50, 'V'), @@ -1580,11 +1590,19 @@ def _seg_15(): (0x1C3B, 'V'), (0x1C4A, 'X'), (0x1C4D, 'V'), - (0x1C80, 'X'), + (0x1C80, 'M', u'в'), + (0x1C81, 'M', u'д'), + (0x1C82, 'M', u'о'), + (0x1C83, 'M', u'с'), + (0x1C84, 'M', u'т'), + (0x1C86, 'M', u'ъ'), + (0x1C87, 'M', u'ѣ'), + (0x1C88, 'M', u'ꙋ'), + (0x1C89, 'X'), (0x1CC0, 'V'), (0x1CC8, 'X'), (0x1CD0, 'V'), - (0x1CF7, 'X'), + (0x1CFA, 'X'), (0x1D00, 'V'), (0x1D2C, 'M', u'a'), (0x1D2D, 'M', u'æ'), @@ -1649,6 +1667,10 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), + ] + +def _seg_16(): + return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1667,10 +1689,6 @@ def _seg_15(): (0x1DA7, 'M', u'ᵻ'), (0x1DA8, 'M', u'ʝ'), (0x1DA9, 'M', u'ɭ'), - ] - -def _seg_16(): - return [ (0x1DAA, 'M', u'ᶅ'), (0x1DAB, 'M', u'ʟ'), (0x1DAC, 'M', u'ɱ'), @@ -1694,8 +1712,8 @@ def _seg_16(): (0x1DBE, 'M', u'ʒ'), (0x1DBF, 'M', u'θ'), (0x1DC0, 'V'), - (0x1DE7, 'X'), - (0x1DFC, 'V'), + (0x1DFA, 'X'), + (0x1DFB, 'V'), (0x1E00, 'M', u'ḁ'), (0x1E01, 'V'), (0x1E02, 'M', u'ḃ'), @@ -1753,6 +1771,10 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), + ] + +def _seg_17(): + return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1771,10 +1793,6 @@ def _seg_16(): (0x1E48, 'M', u'ṉ'), (0x1E49, 'V'), (0x1E4A, 'M', u'ṋ'), - ] - -def _seg_17(): - return [ (0x1E4B, 'V'), (0x1E4C, 'M', u'ṍ'), (0x1E4D, 'V'), @@ -1857,6 +1875,10 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), + ] + +def _seg_18(): + return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1875,10 +1897,6 @@ def _seg_17(): (0x1EB1, 'V'), (0x1EB2, 'M', u'ẳ'), (0x1EB3, 'V'), - ] - -def _seg_18(): - return [ (0x1EB4, 'M', u'ẵ'), (0x1EB5, 'V'), (0x1EB6, 'M', u'ặ'), @@ -1961,6 +1979,10 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), + ] + +def _seg_19(): + return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1979,10 +2001,6 @@ def _seg_18(): (0x1F2B, 'M', u'ἣ'), (0x1F2C, 'M', u'ἤ'), (0x1F2D, 'M', u'ἥ'), - ] - -def _seg_19(): - return [ (0x1F2E, 'M', u'ἦ'), (0x1F2F, 'M', u'ἧ'), (0x1F30, 'V'), @@ -2065,6 +2083,10 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), + ] + +def _seg_20(): + return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2083,10 +2105,6 @@ def _seg_19(): (0x1FAC, 'M', u'ὤι'), (0x1FAD, 'M', u'ὥι'), (0x1FAE, 'M', u'ὦι'), - ] - -def _seg_20(): - return [ (0x1FAF, 'M', u'ὧι'), (0x1FB0, 'V'), (0x1FB2, 'M', u'ὰι'), @@ -2169,6 +2187,10 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), + ] + +def _seg_21(): + return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2187,10 +2209,6 @@ def _seg_20(): (0x204A, 'V'), (0x2057, 'M', u'′′′′'), (0x2058, 'V'), - ] - -def _seg_21(): - return [ (0x205F, '3', u' '), (0x2060, 'I'), (0x2061, 'X'), @@ -2244,7 +2262,7 @@ def _seg_21(): (0x20A0, 'V'), (0x20A8, 'M', u'rs'), (0x20A9, 'V'), - (0x20BB, 'X'), + (0x20C0, 'X'), (0x20D0, 'V'), (0x20F1, 'X'), (0x2100, '3', u'a/c'), @@ -2273,6 +2291,10 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), + ] + +def _seg_22(): + return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2291,10 +2313,6 @@ def _seg_21(): (0x2133, 'M', u'm'), (0x2134, 'M', u'o'), (0x2135, 'M', u'א'), - ] - -def _seg_22(): - return [ (0x2136, 'M', u'ב'), (0x2137, 'M', u'ג'), (0x2138, 'M', u'ד'), @@ -2363,7 +2381,8 @@ def _seg_22(): (0x2183, 'X'), (0x2184, 'V'), (0x2189, 'M', u'0⁄3'), - (0x218A, 'X'), + (0x218A, 'V'), + (0x218C, 'X'), (0x2190, 'V'), (0x222C, 'M', u'∫∫'), (0x222D, 'M', u'∫∫∫'), @@ -2376,10 +2395,12 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), + ] + +def _seg_23(): + return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), - (0x23F4, 'X'), - (0x2400, 'V'), (0x2427, 'X'), (0x2440, 'V'), (0x244B, 'X'), @@ -2395,10 +2416,6 @@ def _seg_22(): (0x2469, 'M', u'10'), (0x246A, 'M', u'11'), (0x246B, 'M', u'12'), - ] - -def _seg_23(): - return [ (0x246C, 'M', u'13'), (0x246D, 'M', u'14'), (0x246E, 'M', u'15'), @@ -2482,6 +2499,10 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), + ] + +def _seg_24(): + return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2499,10 +2520,6 @@ def _seg_23(): (0x24E0, 'M', u'q'), (0x24E1, 'M', u'r'), (0x24E2, 'M', u's'), - ] - -def _seg_24(): - return [ (0x24E3, 'M', u't'), (0x24E4, 'M', u'u'), (0x24E5, 'M', u'v'), @@ -2512,8 +2529,6 @@ def _seg_24(): (0x24E9, 'M', u'z'), (0x24EA, 'M', u'0'), (0x24EB, 'V'), - (0x2700, 'X'), - (0x2701, 'V'), (0x2A0C, 'M', u'∫∫∫∫'), (0x2A0D, 'V'), (0x2A74, '3', u'::='), @@ -2522,9 +2537,17 @@ def _seg_24(): (0x2A77, 'V'), (0x2ADC, 'M', u'⫝̸'), (0x2ADD, 'V'), - (0x2B4D, 'X'), - (0x2B50, 'V'), - (0x2B5A, 'X'), + (0x2B74, 'X'), + (0x2B76, 'V'), + (0x2B96, 'X'), + (0x2B98, 'V'), + (0x2BBA, 'X'), + (0x2BBD, 'V'), + (0x2BC9, 'X'), + (0x2BCA, 'V'), + (0x2BD3, 'X'), + (0x2BEC, 'V'), + (0x2BF0, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2580,6 +2603,10 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), + ] + +def _seg_25(): + return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2603,10 +2630,6 @@ def _seg_24(): (0x2C80, 'M', u'ⲁ'), (0x2C81, 'V'), (0x2C82, 'M', u'ⲃ'), - ] - -def _seg_25(): - return [ (0x2C83, 'V'), (0x2C84, 'M', u'ⲅ'), (0x2C85, 'V'), @@ -2684,6 +2707,10 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), + ] + +def _seg_26(): + return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2707,10 +2734,6 @@ def _seg_25(): (0x2CEB, 'M', u'ⳬ'), (0x2CEC, 'V'), (0x2CED, 'M', u'ⳮ'), - ] - -def _seg_26(): - return [ (0x2CEE, 'V'), (0x2CF2, 'M', u'ⳳ'), (0x2CF3, 'V'), @@ -2745,7 +2768,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E3C, 'X'), + (0x2E4A, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2788,6 +2811,10 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), + ] + +def _seg_27(): + return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2811,10 +2838,6 @@ def _seg_26(): (0x2F37, 'M', u'弋'), (0x2F38, 'M', u'弓'), (0x2F39, 'M', u'彐'), - ] - -def _seg_27(): - return [ (0x2F3A, 'M', u'彡'), (0x2F3B, 'M', u'彳'), (0x2F3C, 'M', u'心'), @@ -2892,6 +2915,10 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), + ] + +def _seg_28(): + return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2915,10 +2942,6 @@ def _seg_27(): (0x2F9B, 'M', u'走'), (0x2F9C, 'M', u'足'), (0x2F9D, 'M', u'身'), - ] - -def _seg_28(): - return [ (0x2F9E, 'M', u'車'), (0x2F9F, 'M', u'辛'), (0x2FA0, 'M', u'辰'), @@ -2996,9 +3019,13 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), + ] + +def _seg_29(): + return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312E, 'X'), + (0x312F, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3019,10 +3046,6 @@ def _seg_28(): (0x3142, 'M', u'ᄇ'), (0x3143, 'M', u'ᄈ'), (0x3144, 'M', u'ᄡ'), - ] - -def _seg_29(): - return [ (0x3145, 'M', u'ᄉ'), (0x3146, 'M', u'ᄊ'), (0x3147, 'M', u'ᄋ'), @@ -3100,6 +3123,10 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), + ] + +def _seg_30(): + return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3123,10 +3150,6 @@ def _seg_29(): (0x3202, '3', u'(ᄃ)'), (0x3203, '3', u'(ᄅ)'), (0x3204, '3', u'(ᄆ)'), - ] - -def _seg_30(): - return [ (0x3205, '3', u'(ᄇ)'), (0x3206, '3', u'(ᄉ)'), (0x3207, '3', u'(ᄋ)'), @@ -3204,6 +3227,10 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), + ] + +def _seg_31(): + return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3227,10 +3254,6 @@ def _seg_30(): (0x326D, 'M', u'ᄒ'), (0x326E, 'M', u'가'), (0x326F, 'M', u'나'), - ] - -def _seg_31(): - return [ (0x3270, 'M', u'다'), (0x3271, 'M', u'라'), (0x3272, 'M', u'마'), @@ -3308,6 +3331,10 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), + ] + +def _seg_32(): + return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3331,10 +3358,6 @@ def _seg_31(): (0x32D1, 'M', u'イ'), (0x32D2, 'M', u'ウ'), (0x32D3, 'M', u'エ'), - ] - -def _seg_32(): - return [ (0x32D4, 'M', u'オ'), (0x32D5, 'M', u'カ'), (0x32D6, 'M', u'キ'), @@ -3412,6 +3435,10 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), + ] + +def _seg_33(): + return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3435,10 +3462,6 @@ def _seg_32(): (0x3335, 'M', u'フラン'), (0x3336, 'M', u'ヘクタール'), (0x3337, 'M', u'ペソ'), - ] - -def _seg_33(): - return [ (0x3338, 'M', u'ペニヒ'), (0x3339, 'M', u'ヘルツ'), (0x333A, 'M', u'ペンス'), @@ -3516,6 +3539,10 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), + ] + +def _seg_34(): + return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3539,10 +3566,6 @@ def _seg_33(): (0x3399, 'M', u'fm'), (0x339A, 'M', u'nm'), (0x339B, 'M', u'μm'), - ] - -def _seg_34(): - return [ (0x339C, 'M', u'mm'), (0x339D, 'M', u'cm'), (0x339E, 'M', u'km'), @@ -3620,6 +3643,10 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), + ] + +def _seg_35(): + return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3643,14 +3670,10 @@ def _seg_34(): (0x33FD, 'M', u'30日'), (0x33FE, 'M', u'31日'), (0x33FF, 'M', u'gal'), - ] - -def _seg_35(): - return [ (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FCD, 'X'), + (0x9FEB, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3724,11 +3747,20 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), + ] + +def _seg_36(): + return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), - (0xA698, 'X'), - (0xA69F, 'V'), + (0xA698, 'M', u'ꚙ'), + (0xA699, 'V'), + (0xA69A, 'M', u'ꚛ'), + (0xA69B, 'V'), + (0xA69C, 'M', u'ъ'), + (0xA69D, 'M', u'ь'), + (0xA69E, 'V'), (0xA6F8, 'X'), (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), @@ -3747,10 +3779,6 @@ def _seg_35(): (0xA72F, 'V'), (0xA732, 'M', u'ꜳ'), (0xA733, 'V'), - ] - -def _seg_36(): - return [ (0xA734, 'M', u'ꜵ'), (0xA735, 'V'), (0xA736, 'M', u'ꜷ'), @@ -3823,6 +3851,10 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), + ] + +def _seg_37(): + return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3832,12 +3864,20 @@ def _seg_36(): (0xA78C, 'V'), (0xA78D, 'M', u'ɥ'), (0xA78E, 'V'), - (0xA78F, 'X'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), - (0xA794, 'X'), + (0xA796, 'M', u'ꞗ'), + (0xA797, 'V'), + (0xA798, 'M', u'ꞙ'), + (0xA799, 'V'), + (0xA79A, 'M', u'ꞛ'), + (0xA79B, 'V'), + (0xA79C, 'M', u'ꞝ'), + (0xA79D, 'V'), + (0xA79E, 'M', u'ꞟ'), + (0xA79F, 'V'), (0xA7A0, 'M', u'ꞡ'), (0xA7A1, 'V'), (0xA7A2, 'M', u'ꞣ'), @@ -3849,12 +3889,22 @@ def _seg_36(): (0xA7A8, 'M', u'ꞩ'), (0xA7A9, 'V'), (0xA7AA, 'M', u'ɦ'), - (0xA7AB, 'X'), + (0xA7AB, 'M', u'ɜ'), + (0xA7AC, 'M', u'ɡ'), + (0xA7AD, 'M', u'ɬ'), + (0xA7AE, 'M', u'ɪ'), + (0xA7AF, 'X'), + (0xA7B0, 'M', u'ʞ'), + (0xA7B1, 'M', u'ʇ'), + (0xA7B2, 'M', u'ʝ'), + (0xA7B3, 'M', u'ꭓ'), + (0xA7B4, 'M', u'ꞵ'), + (0xA7B5, 'V'), + (0xA7B6, 'M', u'ꞷ'), + (0xA7B7, 'V'), + (0xA7B8, 'X'), + (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), - ] - -def _seg_37(): - return [ (0xA7F9, 'M', u'œ'), (0xA7FA, 'V'), (0xA82C, 'X'), @@ -3863,11 +3913,11 @@ def _seg_37(): (0xA840, 'V'), (0xA878, 'X'), (0xA880, 'V'), - (0xA8C5, 'X'), + (0xA8C6, 'X'), (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FC, 'X'), + (0xA8FE, 'X'), (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), @@ -3877,7 +3927,7 @@ def _seg_37(): (0xA9CF, 'V'), (0xA9DA, 'X'), (0xA9DE, 'V'), - (0xA9E0, 'X'), + (0xA9FF, 'X'), (0xAA00, 'V'), (0xAA37, 'X'), (0xAA40, 'V'), @@ -3885,8 +3935,6 @@ def _seg_37(): (0xAA50, 'V'), (0xAA5A, 'X'), (0xAA5C, 'V'), - (0xAA7C, 'X'), - (0xAA80, 'V'), (0xAAC3, 'X'), (0xAADB, 'V'), (0xAAF7, 'X'), @@ -3900,6 +3948,97 @@ def _seg_37(): (0xAB27, 'X'), (0xAB28, 'V'), (0xAB2F, 'X'), + (0xAB30, 'V'), + (0xAB5C, 'M', u'ꜧ'), + (0xAB5D, 'M', u'ꬷ'), + (0xAB5E, 'M', u'ɫ'), + (0xAB5F, 'M', u'ꭒ'), + (0xAB60, 'V'), + (0xAB66, 'X'), + ] + +def _seg_38(): + return [ + (0xAB70, 'M', u'Ꭰ'), + (0xAB71, 'M', u'Ꭱ'), + (0xAB72, 'M', u'Ꭲ'), + (0xAB73, 'M', u'Ꭳ'), + (0xAB74, 'M', u'Ꭴ'), + (0xAB75, 'M', u'Ꭵ'), + (0xAB76, 'M', u'Ꭶ'), + (0xAB77, 'M', u'Ꭷ'), + (0xAB78, 'M', u'Ꭸ'), + (0xAB79, 'M', u'Ꭹ'), + (0xAB7A, 'M', u'Ꭺ'), + (0xAB7B, 'M', u'Ꭻ'), + (0xAB7C, 'M', u'Ꭼ'), + (0xAB7D, 'M', u'Ꭽ'), + (0xAB7E, 'M', u'Ꭾ'), + (0xAB7F, 'M', u'Ꭿ'), + (0xAB80, 'M', u'Ꮀ'), + (0xAB81, 'M', u'Ꮁ'), + (0xAB82, 'M', u'Ꮂ'), + (0xAB83, 'M', u'Ꮃ'), + (0xAB84, 'M', u'Ꮄ'), + (0xAB85, 'M', u'Ꮅ'), + (0xAB86, 'M', u'Ꮆ'), + (0xAB87, 'M', u'Ꮇ'), + (0xAB88, 'M', u'Ꮈ'), + (0xAB89, 'M', u'Ꮉ'), + (0xAB8A, 'M', u'Ꮊ'), + (0xAB8B, 'M', u'Ꮋ'), + (0xAB8C, 'M', u'Ꮌ'), + (0xAB8D, 'M', u'Ꮍ'), + (0xAB8E, 'M', u'Ꮎ'), + (0xAB8F, 'M', u'Ꮏ'), + (0xAB90, 'M', u'Ꮐ'), + (0xAB91, 'M', u'Ꮑ'), + (0xAB92, 'M', u'Ꮒ'), + (0xAB93, 'M', u'Ꮓ'), + (0xAB94, 'M', u'Ꮔ'), + (0xAB95, 'M', u'Ꮕ'), + (0xAB96, 'M', u'Ꮖ'), + (0xAB97, 'M', u'Ꮗ'), + (0xAB98, 'M', u'Ꮘ'), + (0xAB99, 'M', u'Ꮙ'), + (0xAB9A, 'M', u'Ꮚ'), + (0xAB9B, 'M', u'Ꮛ'), + (0xAB9C, 'M', u'Ꮜ'), + (0xAB9D, 'M', u'Ꮝ'), + (0xAB9E, 'M', u'Ꮞ'), + (0xAB9F, 'M', u'Ꮟ'), + (0xABA0, 'M', u'Ꮠ'), + (0xABA1, 'M', u'Ꮡ'), + (0xABA2, 'M', u'Ꮢ'), + (0xABA3, 'M', u'Ꮣ'), + (0xABA4, 'M', u'Ꮤ'), + (0xABA5, 'M', u'Ꮥ'), + (0xABA6, 'M', u'Ꮦ'), + (0xABA7, 'M', u'Ꮧ'), + (0xABA8, 'M', u'Ꮨ'), + (0xABA9, 'M', u'Ꮩ'), + (0xABAA, 'M', u'Ꮪ'), + (0xABAB, 'M', u'Ꮫ'), + (0xABAC, 'M', u'Ꮬ'), + (0xABAD, 'M', u'Ꮭ'), + (0xABAE, 'M', u'Ꮮ'), + (0xABAF, 'M', u'Ꮯ'), + (0xABB0, 'M', u'Ꮰ'), + (0xABB1, 'M', u'Ꮱ'), + (0xABB2, 'M', u'Ꮲ'), + (0xABB3, 'M', u'Ꮳ'), + (0xABB4, 'M', u'Ꮴ'), + (0xABB5, 'M', u'Ꮵ'), + (0xABB6, 'M', u'Ꮶ'), + (0xABB7, 'M', u'Ꮷ'), + (0xABB8, 'M', u'Ꮸ'), + (0xABB9, 'M', u'Ꮹ'), + (0xABBA, 'M', u'Ꮺ'), + (0xABBB, 'M', u'Ꮻ'), + (0xABBC, 'M', u'Ꮼ'), + (0xABBD, 'M', u'Ꮽ'), + (0xABBE, 'M', u'Ꮾ'), + (0xABBF, 'M', u'Ꮿ'), (0xABC0, 'V'), (0xABEE, 'X'), (0xABF0, 'V'), @@ -3920,6 +4059,10 @@ def _seg_37(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), + ] + +def _seg_39(): + return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -3955,10 +4098,6 @@ def _seg_37(): (0xF92B, 'M', u'狼'), (0xF92C, 'M', u'郎'), (0xF92D, 'M', u'來'), - ] - -def _seg_38(): - return [ (0xF92E, 'M', u'冷'), (0xF92F, 'M', u'勞'), (0xF930, 'M', u'擄'), @@ -4024,6 +4163,10 @@ def _seg_38(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), + ] + +def _seg_40(): + return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4059,10 +4202,6 @@ def _seg_38(): (0xF98F, 'M', u'憐'), (0xF990, 'M', u'戀'), (0xF991, 'M', u'撚'), - ] - -def _seg_39(): - return [ (0xF992, 'M', u'漣'), (0xF993, 'M', u'煉'), (0xF994, 'M', u'璉'), @@ -4128,6 +4267,10 @@ def _seg_39(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), + ] + +def _seg_41(): + return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4163,10 +4306,6 @@ def _seg_39(): (0xF9F3, 'M', u'麟'), (0xF9F4, 'M', u'林'), (0xF9F5, 'M', u'淋'), - ] - -def _seg_40(): - return [ (0xF9F6, 'M', u'臨'), (0xF9F7, 'M', u'立'), (0xF9F8, 'M', u'笠'), @@ -4232,6 +4371,10 @@ def _seg_40(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), + ] + +def _seg_42(): + return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4267,10 +4410,6 @@ def _seg_40(): (0xFA5C, 'M', u'臭'), (0xFA5D, 'M', u'艹'), (0xFA5F, 'M', u'著'), - ] - -def _seg_41(): - return [ (0xFA60, 'M', u'褐'), (0xFA61, 'M', u'視'), (0xFA62, 'M', u'謁'), @@ -4336,6 +4475,10 @@ def _seg_41(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), + ] + +def _seg_43(): + return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4371,10 +4514,6 @@ def _seg_41(): (0xFAC2, 'M', u'輸'), (0xFAC3, 'M', u'遲'), (0xFAC4, 'M', u'醙'), - ] - -def _seg_42(): - return [ (0xFAC5, 'M', u'鉶'), (0xFAC6, 'M', u'陼'), (0xFAC7, 'M', u'難'), @@ -4440,6 +4579,10 @@ def _seg_42(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), + ] + +def _seg_44(): + return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4475,10 +4618,6 @@ def _seg_42(): (0xFB7A, 'M', u'چ'), (0xFB7E, 'M', u'ڇ'), (0xFB82, 'M', u'ڍ'), - ] - -def _seg_43(): - return [ (0xFB84, 'M', u'ڌ'), (0xFB86, 'M', u'ڎ'), (0xFB88, 'M', u'ڈ'), @@ -4544,6 +4683,10 @@ def _seg_43(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), + ] + +def _seg_45(): + return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4579,10 +4722,6 @@ def _seg_43(): (0xFC3C, 'M', u'كم'), (0xFC3D, 'M', u'كى'), (0xFC3E, 'M', u'كي'), - ] - -def _seg_44(): - return [ (0xFC3F, 'M', u'لج'), (0xFC40, 'M', u'لح'), (0xFC41, 'M', u'لخ'), @@ -4648,6 +4787,10 @@ def _seg_44(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), + ] + +def _seg_46(): + return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4683,10 +4826,6 @@ def _seg_44(): (0xFCA0, 'M', u'به'), (0xFCA1, 'M', u'تج'), (0xFCA2, 'M', u'تح'), - ] - -def _seg_45(): - return [ (0xFCA3, 'M', u'تخ'), (0xFCA4, 'M', u'تم'), (0xFCA5, 'M', u'ته'), @@ -4752,6 +4891,10 @@ def _seg_45(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), + ] + +def _seg_47(): + return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4787,10 +4930,6 @@ def _seg_45(): (0xFD04, 'M', u'خي'), (0xFD05, 'M', u'صى'), (0xFD06, 'M', u'صي'), - ] - -def _seg_46(): - return [ (0xFD07, 'M', u'ضى'), (0xFD08, 'M', u'ضي'), (0xFD09, 'M', u'شج'), @@ -4856,6 +4995,10 @@ def _seg_46(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), + ] + +def _seg_48(): + return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -4891,10 +5034,6 @@ def _seg_46(): (0xFD87, 'M', u'لمح'), (0xFD89, 'M', u'محج'), (0xFD8A, 'M', u'محم'), - ] - -def _seg_47(): - return [ (0xFD8B, 'M', u'محي'), (0xFD8C, 'M', u'مجح'), (0xFD8D, 'M', u'مجم'), @@ -4960,6 +5099,10 @@ def _seg_47(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), + ] + +def _seg_49(): + return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -4981,7 +5124,7 @@ def _seg_47(): (0xFE18, 'M', u'〗'), (0xFE19, 'X'), (0xFE20, 'V'), - (0xFE27, 'X'), + (0xFE30, 'X'), (0xFE31, 'M', u'—'), (0xFE32, 'M', u'–'), (0xFE33, '3', u'_'), @@ -4995,10 +5138,6 @@ def _seg_47(): (0xFE3C, 'M', u'】'), (0xFE3D, 'M', u'《'), (0xFE3E, 'M', u'》'), - ] - -def _seg_48(): - return [ (0xFE3F, 'M', u'〈'), (0xFE40, 'M', u'〉'), (0xFE41, 'M', u'「'), @@ -5064,6 +5203,10 @@ def _seg_48(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), + ] + +def _seg_50(): + return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5099,10 +5242,6 @@ def _seg_48(): (0xFF00, 'X'), (0xFF01, '3', u'!'), (0xFF02, '3', u'"'), - ] - -def _seg_49(): - return [ (0xFF03, '3', u'#'), (0xFF04, '3', u'$'), (0xFF05, '3', u'%'), @@ -5168,6 +5307,10 @@ def _seg_49(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), + ] + +def _seg_51(): + return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5203,10 +5346,6 @@ def _seg_49(): (0xFF64, 'M', u'、'), (0xFF65, 'M', u'・'), (0xFF66, 'M', u'ヲ'), - ] - -def _seg_50(): - return [ (0xFF67, 'M', u'ァ'), (0xFF68, 'M', u'ィ'), (0xFF69, 'M', u'ゥ'), @@ -5272,6 +5411,10 @@ def _seg_50(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), + ] + +def _seg_52(): + return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5307,10 +5450,6 @@ def _seg_50(): (0xFFCB, 'M', u'ᅨ'), (0xFFCC, 'M', u'ᅩ'), (0xFFCD, 'M', u'ᅪ'), - ] - -def _seg_51(): - return [ (0xFFCE, 'M', u'ᅫ'), (0xFFCF, 'M', u'ᅬ'), (0xFFD0, 'X'), @@ -5360,21 +5499,29 @@ def _seg_51(): (0x10107, 'V'), (0x10134, 'X'), (0x10137, 'V'), - (0x1018B, 'X'), + (0x1018F, 'X'), (0x10190, 'V'), (0x1019C, 'X'), + (0x101A0, 'V'), + (0x101A1, 'X'), (0x101D0, 'V'), (0x101FE, 'X'), (0x10280, 'V'), (0x1029D, 'X'), (0x102A0, 'V'), (0x102D1, 'X'), + (0x102E0, 'V'), + (0x102FC, 'X'), (0x10300, 'V'), - (0x1031F, 'X'), - (0x10320, 'V'), (0x10324, 'X'), - (0x10330, 'V'), + (0x1032D, 'V'), + ] + +def _seg_53(): + return [ (0x1034B, 'X'), + (0x10350, 'V'), + (0x1037B, 'X'), (0x10380, 'V'), (0x1039E, 'X'), (0x1039F, 'V'), @@ -5411,10 +5558,6 @@ def _seg_51(): (0x1041B, 'M', u'𐑃'), (0x1041C, 'M', u'𐑄'), (0x1041D, 'M', u'𐑅'), - ] - -def _seg_52(): - return [ (0x1041E, 'M', u'𐑆'), (0x1041F, 'M', u'𐑇'), (0x10420, 'M', u'𐑈'), @@ -5429,6 +5572,61 @@ def _seg_52(): (0x1049E, 'X'), (0x104A0, 'V'), (0x104AA, 'X'), + (0x104B0, 'M', u'𐓘'), + (0x104B1, 'M', u'𐓙'), + (0x104B2, 'M', u'𐓚'), + (0x104B3, 'M', u'𐓛'), + (0x104B4, 'M', u'𐓜'), + (0x104B5, 'M', u'𐓝'), + (0x104B6, 'M', u'𐓞'), + (0x104B7, 'M', u'𐓟'), + (0x104B8, 'M', u'𐓠'), + (0x104B9, 'M', u'𐓡'), + (0x104BA, 'M', u'𐓢'), + (0x104BB, 'M', u'𐓣'), + (0x104BC, 'M', u'𐓤'), + (0x104BD, 'M', u'𐓥'), + (0x104BE, 'M', u'𐓦'), + (0x104BF, 'M', u'𐓧'), + (0x104C0, 'M', u'𐓨'), + (0x104C1, 'M', u'𐓩'), + (0x104C2, 'M', u'𐓪'), + (0x104C3, 'M', u'𐓫'), + (0x104C4, 'M', u'𐓬'), + (0x104C5, 'M', u'𐓭'), + (0x104C6, 'M', u'𐓮'), + (0x104C7, 'M', u'𐓯'), + (0x104C8, 'M', u'𐓰'), + (0x104C9, 'M', u'𐓱'), + (0x104CA, 'M', u'𐓲'), + (0x104CB, 'M', u'𐓳'), + (0x104CC, 'M', u'𐓴'), + (0x104CD, 'M', u'𐓵'), + (0x104CE, 'M', u'𐓶'), + (0x104CF, 'M', u'𐓷'), + (0x104D0, 'M', u'𐓸'), + (0x104D1, 'M', u'𐓹'), + (0x104D2, 'M', u'𐓺'), + (0x104D3, 'M', u'𐓻'), + (0x104D4, 'X'), + (0x104D8, 'V'), + (0x104FC, 'X'), + (0x10500, 'V'), + (0x10528, 'X'), + (0x10530, 'V'), + (0x10564, 'X'), + (0x1056F, 'V'), + (0x10570, 'X'), + (0x10600, 'V'), + (0x10737, 'X'), + ] + +def _seg_54(): + return [ + (0x10740, 'V'), + (0x10756, 'X'), + (0x10760, 'V'), + (0x10768, 'X'), (0x10800, 'V'), (0x10806, 'X'), (0x10808, 'V'), @@ -5442,8 +5640,14 @@ def _seg_52(): (0x1083F, 'V'), (0x10856, 'X'), (0x10857, 'V'), - (0x10860, 'X'), - (0x10900, 'V'), + (0x1089F, 'X'), + (0x108A7, 'V'), + (0x108B0, 'X'), + (0x108E0, 'V'), + (0x108F3, 'X'), + (0x108F4, 'V'), + (0x108F6, 'X'), + (0x108FB, 'V'), (0x1091C, 'X'), (0x1091F, 'V'), (0x1093A, 'X'), @@ -5451,9 +5655,9 @@ def _seg_52(): (0x10940, 'X'), (0x10980, 'V'), (0x109B8, 'X'), - (0x109BE, 'V'), - (0x109C0, 'X'), - (0x10A00, 'V'), + (0x109BC, 'V'), + (0x109D0, 'X'), + (0x109D2, 'V'), (0x10A04, 'X'), (0x10A05, 'V'), (0x10A07, 'X'), @@ -5470,7 +5674,11 @@ def _seg_52(): (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), - (0x10A80, 'X'), + (0x10AA0, 'X'), + (0x10AC0, 'V'), + (0x10AE7, 'X'), + (0x10AEB, 'V'), + (0x10AF7, 'X'), (0x10B00, 'V'), (0x10B36, 'X'), (0x10B39, 'V'), @@ -5478,16 +5686,80 @@ def _seg_52(): (0x10B58, 'V'), (0x10B73, 'X'), (0x10B78, 'V'), - (0x10B80, 'X'), + (0x10B92, 'X'), + (0x10B99, 'V'), + (0x10B9D, 'X'), + (0x10BA9, 'V'), + (0x10BB0, 'X'), (0x10C00, 'V'), (0x10C49, 'X'), + (0x10C80, 'M', u'𐳀'), + (0x10C81, 'M', u'𐳁'), + (0x10C82, 'M', u'𐳂'), + (0x10C83, 'M', u'𐳃'), + (0x10C84, 'M', u'𐳄'), + (0x10C85, 'M', u'𐳅'), + (0x10C86, 'M', u'𐳆'), + (0x10C87, 'M', u'𐳇'), + (0x10C88, 'M', u'𐳈'), + (0x10C89, 'M', u'𐳉'), + (0x10C8A, 'M', u'𐳊'), + (0x10C8B, 'M', u'𐳋'), + (0x10C8C, 'M', u'𐳌'), + (0x10C8D, 'M', u'𐳍'), + (0x10C8E, 'M', u'𐳎'), + (0x10C8F, 'M', u'𐳏'), + (0x10C90, 'M', u'𐳐'), + (0x10C91, 'M', u'𐳑'), + (0x10C92, 'M', u'𐳒'), + (0x10C93, 'M', u'𐳓'), + (0x10C94, 'M', u'𐳔'), + (0x10C95, 'M', u'𐳕'), + (0x10C96, 'M', u'𐳖'), + (0x10C97, 'M', u'𐳗'), + (0x10C98, 'M', u'𐳘'), + (0x10C99, 'M', u'𐳙'), + (0x10C9A, 'M', u'𐳚'), + (0x10C9B, 'M', u'𐳛'), + (0x10C9C, 'M', u'𐳜'), + (0x10C9D, 'M', u'𐳝'), + ] + +def _seg_55(): + return [ + (0x10C9E, 'M', u'𐳞'), + (0x10C9F, 'M', u'𐳟'), + (0x10CA0, 'M', u'𐳠'), + (0x10CA1, 'M', u'𐳡'), + (0x10CA2, 'M', u'𐳢'), + (0x10CA3, 'M', u'𐳣'), + (0x10CA4, 'M', u'𐳤'), + (0x10CA5, 'M', u'𐳥'), + (0x10CA6, 'M', u'𐳦'), + (0x10CA7, 'M', u'𐳧'), + (0x10CA8, 'M', u'𐳨'), + (0x10CA9, 'M', u'𐳩'), + (0x10CAA, 'M', u'𐳪'), + (0x10CAB, 'M', u'𐳫'), + (0x10CAC, 'M', u'𐳬'), + (0x10CAD, 'M', u'𐳭'), + (0x10CAE, 'M', u'𐳮'), + (0x10CAF, 'M', u'𐳯'), + (0x10CB0, 'M', u'𐳰'), + (0x10CB1, 'M', u'𐳱'), + (0x10CB2, 'M', u'𐳲'), + (0x10CB3, 'X'), + (0x10CC0, 'V'), + (0x10CF3, 'X'), + (0x10CFA, 'V'), + (0x10D00, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), (0x11070, 'X'), - (0x11080, 'V'), + (0x1107F, 'V'), (0x110BD, 'X'), (0x110BE, 'V'), (0x110C2, 'X'), @@ -5499,36 +5771,235 @@ def _seg_52(): (0x11135, 'X'), (0x11136, 'V'), (0x11144, 'X'), + (0x11150, 'V'), + (0x11177, 'X'), (0x11180, 'V'), - (0x111C9, 'X'), + (0x111CE, 'X'), (0x111D0, 'V'), - (0x111DA, 'X'), + (0x111E0, 'X'), + (0x111E1, 'V'), + (0x111F5, 'X'), + (0x11200, 'V'), + (0x11212, 'X'), + (0x11213, 'V'), + (0x1123F, 'X'), + (0x11280, 'V'), + (0x11287, 'X'), + (0x11288, 'V'), + (0x11289, 'X'), + (0x1128A, 'V'), + (0x1128E, 'X'), + (0x1128F, 'V'), + (0x1129E, 'X'), + (0x1129F, 'V'), + (0x112AA, 'X'), + (0x112B0, 'V'), + (0x112EB, 'X'), + (0x112F0, 'V'), + (0x112FA, 'X'), + (0x11300, 'V'), + (0x11304, 'X'), + (0x11305, 'V'), + (0x1130D, 'X'), + (0x1130F, 'V'), + (0x11311, 'X'), + (0x11313, 'V'), + (0x11329, 'X'), + (0x1132A, 'V'), + (0x11331, 'X'), + (0x11332, 'V'), + (0x11334, 'X'), + (0x11335, 'V'), + (0x1133A, 'X'), + (0x1133C, 'V'), + (0x11345, 'X'), + (0x11347, 'V'), + (0x11349, 'X'), + (0x1134B, 'V'), + (0x1134E, 'X'), + (0x11350, 'V'), + (0x11351, 'X'), + (0x11357, 'V'), + (0x11358, 'X'), + (0x1135D, 'V'), + (0x11364, 'X'), + (0x11366, 'V'), + (0x1136D, 'X'), + (0x11370, 'V'), + (0x11375, 'X'), + ] + +def _seg_56(): + return [ + (0x11400, 'V'), + (0x1145A, 'X'), + (0x1145B, 'V'), + (0x1145C, 'X'), + (0x1145D, 'V'), + (0x1145E, 'X'), + (0x11480, 'V'), + (0x114C8, 'X'), + (0x114D0, 'V'), + (0x114DA, 'X'), + (0x11580, 'V'), + (0x115B6, 'X'), + (0x115B8, 'V'), + (0x115DE, 'X'), + (0x11600, 'V'), + (0x11645, 'X'), + (0x11650, 'V'), + (0x1165A, 'X'), + (0x11660, 'V'), + (0x1166D, 'X'), (0x11680, 'V'), (0x116B8, 'X'), (0x116C0, 'V'), (0x116CA, 'X'), - (0x12000, 'V'), - (0x1236F, 'X'), - (0x12400, 'V'), - (0x12463, 'X'), - (0x12470, 'V'), - (0x12474, 'X'), - (0x13000, 'V'), - (0x1342F, 'X'), + (0x11700, 'V'), + (0x1171A, 'X'), + (0x1171D, 'V'), + (0x1172C, 'X'), + (0x11730, 'V'), + (0x11740, 'X'), + (0x118A0, 'M', u'𑣀'), + (0x118A1, 'M', u'𑣁'), + (0x118A2, 'M', u'𑣂'), + (0x118A3, 'M', u'𑣃'), + (0x118A4, 'M', u'𑣄'), + (0x118A5, 'M', u'𑣅'), + (0x118A6, 'M', u'𑣆'), + (0x118A7, 'M', u'𑣇'), + (0x118A8, 'M', u'𑣈'), + (0x118A9, 'M', u'𑣉'), + (0x118AA, 'M', u'𑣊'), + (0x118AB, 'M', u'𑣋'), + (0x118AC, 'M', u'𑣌'), + (0x118AD, 'M', u'𑣍'), + (0x118AE, 'M', u'𑣎'), + (0x118AF, 'M', u'𑣏'), + (0x118B0, 'M', u'𑣐'), + (0x118B1, 'M', u'𑣑'), + (0x118B2, 'M', u'𑣒'), + (0x118B3, 'M', u'𑣓'), + (0x118B4, 'M', u'𑣔'), + (0x118B5, 'M', u'𑣕'), + (0x118B6, 'M', u'𑣖'), + (0x118B7, 'M', u'𑣗'), + (0x118B8, 'M', u'𑣘'), + (0x118B9, 'M', u'𑣙'), + (0x118BA, 'M', u'𑣚'), + (0x118BB, 'M', u'𑣛'), + (0x118BC, 'M', u'𑣜'), + (0x118BD, 'M', u'𑣝'), + (0x118BE, 'M', u'𑣞'), + (0x118BF, 'M', u'𑣟'), + (0x118C0, 'V'), + (0x118F3, 'X'), + (0x118FF, 'V'), + (0x11900, 'X'), + (0x11A00, 'V'), + (0x11A48, 'X'), + (0x11A50, 'V'), + (0x11A84, 'X'), + (0x11A86, 'V'), + (0x11A9D, 'X'), + (0x11A9E, 'V'), + (0x11AA3, 'X'), + (0x11AC0, 'V'), + (0x11AF9, 'X'), + (0x11C00, 'V'), + (0x11C09, 'X'), + (0x11C0A, 'V'), + (0x11C37, 'X'), + (0x11C38, 'V'), + (0x11C46, 'X'), + (0x11C50, 'V'), + (0x11C6D, 'X'), + (0x11C70, 'V'), + (0x11C90, 'X'), + (0x11C92, 'V'), + (0x11CA8, 'X'), + (0x11CA9, 'V'), + (0x11CB7, 'X'), + (0x11D00, 'V'), + (0x11D07, 'X'), + (0x11D08, 'V'), + (0x11D0A, 'X'), + (0x11D0B, 'V'), + (0x11D37, 'X'), + (0x11D3A, 'V'), + (0x11D3B, 'X'), + (0x11D3C, 'V'), + (0x11D3E, 'X'), ] -def _seg_53(): +def _seg_57(): return [ + (0x11D3F, 'V'), + (0x11D48, 'X'), + (0x11D50, 'V'), + (0x11D5A, 'X'), + (0x12000, 'V'), + (0x1239A, 'X'), + (0x12400, 'V'), + (0x1246F, 'X'), + (0x12470, 'V'), + (0x12475, 'X'), + (0x12480, 'V'), + (0x12544, 'X'), + (0x13000, 'V'), + (0x1342F, 'X'), + (0x14400, 'V'), + (0x14647, 'X'), (0x16800, 'V'), (0x16A39, 'X'), + (0x16A40, 'V'), + (0x16A5F, 'X'), + (0x16A60, 'V'), + (0x16A6A, 'X'), + (0x16A6E, 'V'), + (0x16A70, 'X'), + (0x16AD0, 'V'), + (0x16AEE, 'X'), + (0x16AF0, 'V'), + (0x16AF6, 'X'), + (0x16B00, 'V'), + (0x16B46, 'X'), + (0x16B50, 'V'), + (0x16B5A, 'X'), + (0x16B5B, 'V'), + (0x16B62, 'X'), + (0x16B63, 'V'), + (0x16B78, 'X'), + (0x16B7D, 'V'), + (0x16B90, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), (0x16F7F, 'X'), (0x16F8F, 'V'), (0x16FA0, 'X'), + (0x16FE0, 'V'), + (0x16FE2, 'X'), + (0x17000, 'V'), + (0x187ED, 'X'), + (0x18800, 'V'), + (0x18AF3, 'X'), (0x1B000, 'V'), - (0x1B002, 'X'), + (0x1B11F, 'X'), + (0x1B170, 'V'), + (0x1B2FC, 'X'), + (0x1BC00, 'V'), + (0x1BC6B, 'X'), + (0x1BC70, 'V'), + (0x1BC7D, 'X'), + (0x1BC80, 'V'), + (0x1BC89, 'X'), + (0x1BC90, 'V'), + (0x1BC9A, 'X'), + (0x1BC9C, 'V'), + (0x1BCA0, 'I'), + (0x1BCA4, 'X'), (0x1D000, 'V'), (0x1D0F6, 'X'), (0x1D100, 'V'), @@ -5551,7 +6022,7 @@ def _seg_53(): (0x1D1BF, 'M', u'𝆹𝅥𝅯'), (0x1D1C0, 'M', u'𝆺𝅥𝅯'), (0x1D1C1, 'V'), - (0x1D1DE, 'X'), + (0x1D1E9, 'X'), (0x1D200, 'V'), (0x1D246, 'X'), (0x1D300, 'V'), @@ -5564,6 +6035,10 @@ def _seg_53(): (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), + ] + +def _seg_58(): + return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -5619,10 +6094,6 @@ def _seg_53(): (0x1D43A, 'M', u'g'), (0x1D43B, 'M', u'h'), (0x1D43C, 'M', u'i'), - ] - -def _seg_54(): - return [ (0x1D43D, 'M', u'j'), (0x1D43E, 'M', u'k'), (0x1D43F, 'M', u'l'), @@ -5668,6 +6139,10 @@ def _seg_54(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), + ] + +def _seg_59(): + return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -5723,10 +6198,6 @@ def _seg_54(): (0x1D49E, 'M', u'c'), (0x1D49F, 'M', u'd'), (0x1D4A0, 'X'), - ] - -def _seg_55(): - return [ (0x1D4A2, 'M', u'g'), (0x1D4A3, 'X'), (0x1D4A5, 'M', u'j'), @@ -5772,6 +6243,10 @@ def _seg_55(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), + ] + +def _seg_60(): + return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -5827,10 +6302,6 @@ def _seg_55(): (0x1D505, 'M', u'b'), (0x1D506, 'X'), (0x1D507, 'M', u'd'), - ] - -def _seg_56(): - return [ (0x1D508, 'M', u'e'), (0x1D509, 'M', u'f'), (0x1D50A, 'M', u'g'), @@ -5876,6 +6347,10 @@ def _seg_56(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), + ] + +def _seg_61(): + return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -5931,10 +6406,6 @@ def _seg_56(): (0x1D56C, 'M', u'a'), (0x1D56D, 'M', u'b'), (0x1D56E, 'M', u'c'), - ] - -def _seg_57(): - return [ (0x1D56F, 'M', u'd'), (0x1D570, 'M', u'e'), (0x1D571, 'M', u'f'), @@ -5980,6 +6451,10 @@ def _seg_57(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), + ] + +def _seg_62(): + return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6035,10 +6510,6 @@ def _seg_57(): (0x1D5D0, 'M', u'w'), (0x1D5D1, 'M', u'x'), (0x1D5D2, 'M', u'y'), - ] - -def _seg_58(): - return [ (0x1D5D3, 'M', u'z'), (0x1D5D4, 'M', u'a'), (0x1D5D5, 'M', u'b'), @@ -6084,6 +6555,10 @@ def _seg_58(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), + ] + +def _seg_63(): + return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6139,10 +6614,6 @@ def _seg_58(): (0x1D634, 'M', u's'), (0x1D635, 'M', u't'), (0x1D636, 'M', u'u'), - ] - -def _seg_59(): - return [ (0x1D637, 'M', u'v'), (0x1D638, 'M', u'w'), (0x1D639, 'M', u'x'), @@ -6188,6 +6659,10 @@ def _seg_59(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), + ] + +def _seg_64(): + return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6243,10 +6718,6 @@ def _seg_59(): (0x1D698, 'M', u'o'), (0x1D699, 'M', u'p'), (0x1D69A, 'M', u'q'), - ] - -def _seg_60(): - return [ (0x1D69B, 'M', u'r'), (0x1D69C, 'M', u's'), (0x1D69D, 'M', u't'), @@ -6292,6 +6763,10 @@ def _seg_60(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), + ] + +def _seg_65(): + return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6347,10 +6822,6 @@ def _seg_60(): (0x1D6FE, 'M', u'γ'), (0x1D6FF, 'M', u'δ'), (0x1D700, 'M', u'ε'), - ] - -def _seg_61(): - return [ (0x1D701, 'M', u'ζ'), (0x1D702, 'M', u'η'), (0x1D703, 'M', u'θ'), @@ -6396,6 +6867,10 @@ def _seg_61(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), + ] + +def _seg_66(): + return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6451,10 +6926,6 @@ def _seg_61(): (0x1D764, 'M', u'ο'), (0x1D765, 'M', u'π'), (0x1D766, 'M', u'ρ'), - ] - -def _seg_62(): - return [ (0x1D767, 'M', u'θ'), (0x1D768, 'M', u'σ'), (0x1D769, 'M', u'τ'), @@ -6500,6 +6971,10 @@ def _seg_62(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), + ] + +def _seg_67(): + return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -6555,10 +7030,6 @@ def _seg_62(): (0x1D7CA, 'M', u'ϝ'), (0x1D7CC, 'X'), (0x1D7CE, 'M', u'0'), - ] - -def _seg_63(): - return [ (0x1D7CF, 'M', u'1'), (0x1D7D0, 'M', u'2'), (0x1D7D1, 'M', u'3'), @@ -6604,11 +7075,74 @@ def _seg_63(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), + ] + +def _seg_68(): + return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), (0x1D7FF, 'M', u'9'), - (0x1D800, 'X'), + (0x1D800, 'V'), + (0x1DA8C, 'X'), + (0x1DA9B, 'V'), + (0x1DAA0, 'X'), + (0x1DAA1, 'V'), + (0x1DAB0, 'X'), + (0x1E000, 'V'), + (0x1E007, 'X'), + (0x1E008, 'V'), + (0x1E019, 'X'), + (0x1E01B, 'V'), + (0x1E022, 'X'), + (0x1E023, 'V'), + (0x1E025, 'X'), + (0x1E026, 'V'), + (0x1E02B, 'X'), + (0x1E800, 'V'), + (0x1E8C5, 'X'), + (0x1E8C7, 'V'), + (0x1E8D7, 'X'), + (0x1E900, 'M', u'𞤢'), + (0x1E901, 'M', u'𞤣'), + (0x1E902, 'M', u'𞤤'), + (0x1E903, 'M', u'𞤥'), + (0x1E904, 'M', u'𞤦'), + (0x1E905, 'M', u'𞤧'), + (0x1E906, 'M', u'𞤨'), + (0x1E907, 'M', u'𞤩'), + (0x1E908, 'M', u'𞤪'), + (0x1E909, 'M', u'𞤫'), + (0x1E90A, 'M', u'𞤬'), + (0x1E90B, 'M', u'𞤭'), + (0x1E90C, 'M', u'𞤮'), + (0x1E90D, 'M', u'𞤯'), + (0x1E90E, 'M', u'𞤰'), + (0x1E90F, 'M', u'𞤱'), + (0x1E910, 'M', u'𞤲'), + (0x1E911, 'M', u'𞤳'), + (0x1E912, 'M', u'𞤴'), + (0x1E913, 'M', u'𞤵'), + (0x1E914, 'M', u'𞤶'), + (0x1E915, 'M', u'𞤷'), + (0x1E916, 'M', u'𞤸'), + (0x1E917, 'M', u'𞤹'), + (0x1E918, 'M', u'𞤺'), + (0x1E919, 'M', u'𞤻'), + (0x1E91A, 'M', u'𞤼'), + (0x1E91B, 'M', u'𞤽'), + (0x1E91C, 'M', u'𞤾'), + (0x1E91D, 'M', u'𞤿'), + (0x1E91E, 'M', u'𞥀'), + (0x1E91F, 'M', u'𞥁'), + (0x1E920, 'M', u'𞥂'), + (0x1E921, 'M', u'𞥃'), + (0x1E922, 'V'), + (0x1E94B, 'X'), + (0x1E950, 'V'), + (0x1E95A, 'X'), + (0x1E95E, 'V'), + (0x1E960, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -6645,6 +7179,10 @@ def _seg_63(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), + ] + +def _seg_69(): + return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -6659,10 +7197,6 @@ def _seg_63(): (0x1EE30, 'M', u'ف'), (0x1EE31, 'M', u'ص'), (0x1EE32, 'M', u'ق'), - ] - -def _seg_64(): - return [ (0x1EE33, 'X'), (0x1EE34, 'M', u'ش'), (0x1EE35, 'M', u'ت'), @@ -6749,6 +7283,10 @@ def _seg_64(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), + ] + +def _seg_70(): + return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -6763,10 +7301,6 @@ def _seg_64(): (0x1EEA2, 'M', u'ج'), (0x1EEA3, 'M', u'د'), (0x1EEA4, 'X'), - ] - -def _seg_65(): - return [ (0x1EEA5, 'M', u'و'), (0x1EEA6, 'M', u'ز'), (0x1EEA7, 'M', u'ح'), @@ -6800,11 +7334,11 @@ def _seg_65(): (0x1F0A0, 'V'), (0x1F0AF, 'X'), (0x1F0B1, 'V'), - (0x1F0BF, 'X'), + (0x1F0C0, 'X'), (0x1F0C1, 'V'), (0x1F0D0, 'X'), (0x1F0D1, 'V'), - (0x1F0E0, 'X'), + (0x1F0F6, 'X'), (0x1F101, '3', u'0,'), (0x1F102, '3', u'1,'), (0x1F103, '3', u'2,'), @@ -6815,7 +7349,8 @@ def _seg_65(): (0x1F108, '3', u'7,'), (0x1F109, '3', u'8,'), (0x1F10A, '3', u'9,'), - (0x1F10B, 'X'), + (0x1F10B, 'V'), + (0x1F10D, 'X'), (0x1F110, '3', u'(a)'), (0x1F111, '3', u'(b)'), (0x1F112, '3', u'(c)'), @@ -6852,6 +7387,10 @@ def _seg_65(): (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), + ] + +def _seg_71(): + return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -6867,10 +7406,6 @@ def _seg_65(): (0x1F140, 'M', u'q'), (0x1F141, 'M', u'r'), (0x1F142, 'M', u's'), - ] - -def _seg_66(): - return [ (0x1F143, 'M', u't'), (0x1F144, 'M', u'u'), (0x1F145, 'M', u'v'), @@ -6891,7 +7426,7 @@ def _seg_66(): (0x1F170, 'V'), (0x1F190, 'M', u'dj'), (0x1F191, 'V'), - (0x1F19B, 'X'), + (0x1F1AD, 'X'), (0x1F1E6, 'V'), (0x1F200, 'M', u'ほか'), (0x1F201, 'M', u'ココ'), @@ -6940,7 +7475,8 @@ def _seg_66(): (0x1F238, 'M', u'申'), (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), - (0x1F23B, 'X'), + (0x1F23B, 'M', u'配'), + (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), (0x1F242, 'M', u'〔二〕'), @@ -6954,52 +7490,56 @@ def _seg_66(): (0x1F250, 'M', u'得'), (0x1F251, 'M', u'可'), (0x1F252, 'X'), - (0x1F300, 'V'), - (0x1F321, 'X'), - (0x1F330, 'V'), - (0x1F336, 'X'), - (0x1F337, 'V'), - (0x1F37D, 'X'), - (0x1F380, 'V'), - (0x1F394, 'X'), - (0x1F3A0, 'V'), - (0x1F3C5, 'X'), - (0x1F3C6, 'V'), - (0x1F3CB, 'X'), - (0x1F3E0, 'V'), - (0x1F3F1, 'X'), - (0x1F400, 'V'), - (0x1F43F, 'X'), - (0x1F440, 'V'), + (0x1F260, 'V'), ] -def _seg_67(): +def _seg_72(): return [ - (0x1F441, 'X'), - (0x1F442, 'V'), - (0x1F4F8, 'X'), - (0x1F4F9, 'V'), - (0x1F4FD, 'X'), - (0x1F500, 'V'), - (0x1F53E, 'X'), - (0x1F540, 'V'), - (0x1F544, 'X'), - (0x1F550, 'V'), - (0x1F568, 'X'), - (0x1F5FB, 'V'), - (0x1F641, 'X'), - (0x1F645, 'V'), - (0x1F650, 'X'), - (0x1F680, 'V'), - (0x1F6C6, 'X'), + (0x1F266, 'X'), + (0x1F300, 'V'), + (0x1F6D5, 'X'), + (0x1F6E0, 'V'), + (0x1F6ED, 'X'), + (0x1F6F0, 'V'), + (0x1F6F9, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), + (0x1F780, 'V'), + (0x1F7D5, 'X'), + (0x1F800, 'V'), + (0x1F80C, 'X'), + (0x1F810, 'V'), + (0x1F848, 'X'), + (0x1F850, 'V'), + (0x1F85A, 'X'), + (0x1F860, 'V'), + (0x1F888, 'X'), + (0x1F890, 'V'), + (0x1F8AE, 'X'), + (0x1F900, 'V'), + (0x1F90C, 'X'), + (0x1F910, 'V'), + (0x1F93F, 'X'), + (0x1F940, 'V'), + (0x1F94D, 'X'), + (0x1F950, 'V'), + (0x1F96C, 'X'), + (0x1F980, 'V'), + (0x1F998, 'X'), + (0x1F9C0, 'V'), + (0x1F9C1, 'X'), + (0x1F9D0, 'V'), + (0x1F9E7, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), (0x2B735, 'X'), (0x2B740, 'V'), (0x2B81E, 'X'), + (0x2B820, 'V'), + (0x2CEA2, 'X'), + (0x2CEB0, 'V'), + (0x2EBE1, 'X'), (0x2F800, 'M', u'丽'), (0x2F801, 'M', u'丸'), (0x2F802, 'M', u'乁'), @@ -7055,6 +7595,10 @@ def _seg_67(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), + ] + +def _seg_73(): + return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7075,10 +7619,6 @@ def _seg_67(): (0x2F84B, 'M', u'圖'), (0x2F84C, 'M', u'嘆'), (0x2F84D, 'M', u'圗'), - ] - -def _seg_68(): - return [ (0x2F84E, 'M', u'噑'), (0x2F84F, 'M', u'噴'), (0x2F850, 'M', u'切'), @@ -7159,6 +7699,10 @@ def _seg_68(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), + ] + +def _seg_74(): + return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7179,10 +7723,6 @@ def _seg_68(): (0x2F8B2, 'M', u'成'), (0x2F8B3, 'M', u'戛'), (0x2F8B4, 'M', u'扝'), - ] - -def _seg_69(): - return [ (0x2F8B5, 'M', u'抱'), (0x2F8B6, 'M', u'拔'), (0x2F8B7, 'M', u'捐'), @@ -7263,6 +7803,10 @@ def _seg_69(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), + ] + +def _seg_75(): + return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7283,10 +7827,6 @@ def _seg_69(): (0x2F916, 'M', u'㶖'), (0x2F917, 'M', u'灊'), (0x2F918, 'M', u'災'), - ] - -def _seg_70(): - return [ (0x2F919, 'M', u'灷'), (0x2F91A, 'M', u'炭'), (0x2F91B, 'M', u'𠔥'), @@ -7367,6 +7907,10 @@ def _seg_70(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), + ] + +def _seg_76(): + return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7387,10 +7931,6 @@ def _seg_70(): (0x2F97D, 'M', u'聠'), (0x2F97E, 'M', u'𦖨'), (0x2F97F, 'M', u'聰'), - ] - -def _seg_71(): - return [ (0x2F980, 'M', u'𣍟'), (0x2F981, 'M', u'䏕'), (0x2F982, 'M', u'育'), @@ -7471,6 +8011,10 @@ def _seg_71(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), + ] + +def _seg_77(): + return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -7491,10 +8035,6 @@ def _seg_71(): (0x2F9E1, 'M', u'𨗭'), (0x2F9E2, 'M', u'邔'), (0x2F9E3, 'M', u'郱'), - ] - -def _seg_72(): - return [ (0x2F9E4, 'M', u'鄑'), (0x2F9E5, 'M', u'𨜮'), (0x2F9E6, 'M', u'鄛'), @@ -7631,4 +8171,9 @@ uts46data = tuple( + _seg_70() + _seg_71() + _seg_72() + + _seg_73() + + _seg_74() + + _seg_75() + + _seg_76() + + _seg_77() ) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index f3adf70f7..3ae627b1b 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -15,7 +15,7 @@ pytoml==0.1.16 retrying==1.3.3 requests==2.18.4 chardet==3.0.4 - idna==2.6 + idna==2.7 urllib3==1.23 certifi==2018.4.16 setuptools==39.2.0 From 7ccc1a8f34b657b1a5014417b6b76f29064e2031 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 18:40:52 +0530 Subject: [PATCH 010/129] Update requests to 2.19.1 --- news/requests.vendor | 1 + src/pip/_vendor/requests/__init__.py | 19 +++- src/pip/_vendor/requests/__version__.py | 6 +- src/pip/_vendor/requests/adapters.py | 17 ++-- src/pip/_vendor/requests/api.py | 2 +- src/pip/_vendor/requests/auth.py | 12 +++ src/pip/_vendor/requests/compat.py | 2 + src/pip/_vendor/requests/cookies.py | 10 +- src/pip/_vendor/requests/exceptions.py | 4 + src/pip/_vendor/requests/help.py | 2 +- src/pip/_vendor/requests/models.py | 20 ++-- src/pip/_vendor/requests/sessions.py | 18 ++-- src/pip/_vendor/requests/status_codes.py | 39 +++++++- src/pip/_vendor/requests/structures.py | 10 +- src/pip/_vendor/requests/utils.py | 116 ++++++++++++++++++----- src/pip/_vendor/vendor.txt | 2 +- tasks/vendoring/patches/requests.patch | 10 +- 17 files changed, 224 insertions(+), 66 deletions(-) create mode 100644 news/requests.vendor diff --git a/news/requests.vendor b/news/requests.vendor new file mode 100644 index 000000000..1144da4c6 --- /dev/null +++ b/news/requests.vendor @@ -0,0 +1 @@ +Update requests to 2.19.1 diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py index ccd361adf..9fb6633ed 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.22 + # urllib3 >= 1.21.1, <= 1.23 assert major == 1 assert minor >= 21 - assert minor <= 22 + assert minor <= 23 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -71,6 +71,17 @@ def check_compatibility(urllib3_version, chardet_version): assert patch >= 2 +def _check_cryptography(cryptography_version): + # cryptography < 1.3.4 + try: + cryptography_version = list(map(int, cryptography_version.split('.'))) + except ValueError: + return + + if cryptography_version < [1, 3, 4]: + warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warnings.warn(warning, RequestsDependencyWarning) + # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) @@ -85,6 +96,10 @@ if not WINDOWS: try: from pip._vendor.urllib3.contrib import pyopenssl pyopenssl.inject_into_urllib3() + + # Check cryptography version + from cryptography import __version__ as cryptography_version + _check_cryptography(cryptography_version) except ImportError: pass diff --git a/src/pip/_vendor/requests/__version__.py b/src/pip/_vendor/requests/__version__.py index dc33eef65..ef61ec0f5 100644 --- a/src/pip/_vendor/requests/__version__.py +++ b/src/pip/_vendor/requests/__version__.py @@ -5,10 +5,10 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.18.4' -__build__ = 0x021804 +__version__ = '2.19.1' +__build__ = 0x021901 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2017 Kenneth Reitz' +__copyright__ = 'Copyright 2018 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/src/pip/_vendor/requests/adapters.py b/src/pip/_vendor/requests/adapters.py index 57876383c..f6f3f9965 100644 --- a/src/pip/_vendor/requests/adapters.py +++ b/src/pip/_vendor/requests/adapters.py @@ -13,6 +13,7 @@ import socket from pip._vendor.urllib3.poolmanager import PoolManager, proxy_from_url from pip._vendor.urllib3.response import HTTPResponse +from pip._vendor.urllib3.util import parse_url from pip._vendor.urllib3.util import Timeout as TimeoutSauce from pip._vendor.urllib3.util.retry import Retry from pip._vendor.urllib3.exceptions import ClosedPoolError @@ -28,13 +29,13 @@ from pip._vendor.urllib3.exceptions import ResponseError from .models import Response from .compat import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) +from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, + get_encoding_from_headers, prepend_scheme_if_needed, + get_auth_from_url, urldefragauth, select_proxy) from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL) from .auth import _basic_auth_str try: @@ -219,7 +220,7 @@ class HTTPAdapter(BaseAdapter): cert_loc = verify if not cert_loc: - cert_loc = DEFAULT_CA_BUNDLE_PATH + cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " @@ -300,6 +301,10 @@ class HTTPAdapter(BaseAdapter): if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') + proxy_url = parse_url(proxy) + if not proxy_url.host: + raise InvalidProxyURL("Please check proxy URL. It is malformed" + " and could be missing the host.") proxy_manager = self.proxy_manager_for(proxy) conn = proxy_manager.connection_from_url(url) else: @@ -406,7 +411,7 @@ class HTTPAdapter(BaseAdapter): self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) - self.add_headers(request) + self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) chunked = not (request.body is None or 'Content-Length' in request.headers) diff --git a/src/pip/_vendor/requests/api.py b/src/pip/_vendor/requests/api.py index bc2115c15..a2cc84d76 100644 --- a/src/pip/_vendor/requests/api.py +++ b/src/pip/_vendor/requests/api.py @@ -20,7 +20,7 @@ def request(method, url, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. diff --git a/src/pip/_vendor/requests/auth.py b/src/pip/_vendor/requests/auth.py index 1a182dffd..4ae459474 100644 --- a/src/pip/_vendor/requests/auth.py +++ b/src/pip/_vendor/requests/auth.py @@ -153,6 +153,18 @@ class HTTPDigestAuth(AuthBase): x = x.encode('utf-8') return hashlib.sha1(x).hexdigest() hash_utf8 = sha_utf8 + elif _algorithm == 'SHA-256': + def sha256_utf8(x): + if isinstance(x, str): + x = x.encode('utf-8') + return hashlib.sha256(x).hexdigest() + hash_utf8 = sha256_utf8 + elif _algorithm == 'SHA-512': + def sha512_utf8(x): + if isinstance(x, str): + x = x.encode('utf-8') + return hashlib.sha512(x).hexdigest() + hash_utf8 = sha512_utf8 KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py index 20da3e0e0..ec5d30585 100644 --- a/src/pip/_vendor/requests/compat.py +++ b/src/pip/_vendor/requests/compat.py @@ -47,6 +47,7 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO + from collections import Callable, Mapping, MutableMapping from pip._vendor.urllib3.packages.ordered_dict import OrderedDict @@ -64,6 +65,7 @@ elif is_py3: from http.cookies import Morsel from io import StringIO from collections import OrderedDict + from collections.abc import Callable, Mapping, MutableMapping builtin_str = str str = str diff --git a/src/pip/_vendor/requests/cookies.py b/src/pip/_vendor/requests/cookies.py index ab3c88b9b..50883a84f 100644 --- a/src/pip/_vendor/requests/cookies.py +++ b/src/pip/_vendor/requests/cookies.py @@ -12,10 +12,9 @@ requests.utils imports from here, so be careful with imports. import copy import time import calendar -import collections from ._internal_utils import to_native_string -from .compat import cookielib, urlparse, urlunparse, Morsel +from .compat import cookielib, urlparse, urlunparse, Morsel, MutableMapping try: import threading @@ -169,7 +168,7 @@ class CookieConflictError(RuntimeError): """ -class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): +class RequestsCookieJar(cookielib.CookieJar, MutableMapping): """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface. @@ -415,9 +414,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def copy(self): """Return a copy of this RequestsCookieJar.""" new_cj = RequestsCookieJar() + new_cj.set_policy(self.get_policy()) new_cj.update(self) return new_cj + def get_policy(self): + """Return the CookiePolicy instance used.""" + return self._policy + def _copy_cookie_jar(jar): if jar is None: diff --git a/src/pip/_vendor/requests/exceptions.py b/src/pip/_vendor/requests/exceptions.py index 3e5d0b2db..a91e1fd11 100644 --- a/src/pip/_vendor/requests/exceptions.py +++ b/src/pip/_vendor/requests/exceptions.py @@ -85,6 +85,10 @@ class InvalidHeader(RequestException, ValueError): """The header value provided was somehow invalid.""" +class InvalidProxyURL(InvalidURL): + """The proxy URL provided is invalid.""" + + class ChunkedEncodingError(RequestException): """The server declared chunked encoding but sent an invalid chunk.""" diff --git a/src/pip/_vendor/requests/help.py b/src/pip/_vendor/requests/help.py index 7c4b193c9..df1b4ebcf 100644 --- a/src/pip/_vendor/requests/help.py +++ b/src/pip/_vendor/requests/help.py @@ -13,7 +13,7 @@ from pip._vendor import chardet from . import __version__ as requests_version try: - from .packages.urllib3.contrib import pyopenssl + from pip._vendor.urllib3.contrib import pyopenssl except ImportError: pyopenssl = None OpenSSL = None diff --git a/src/pip/_vendor/requests/models.py b/src/pip/_vendor/requests/models.py index 4ab4feca9..4230535d2 100644 --- a/src/pip/_vendor/requests/models.py +++ b/src/pip/_vendor/requests/models.py @@ -7,7 +7,6 @@ requests.models This module contains the primary objects that power Requests. """ -import collections import datetime import sys @@ -37,6 +36,7 @@ from .utils import ( stream_decode_response_unicode, to_key_val_list, parse_header_links, iter_slices, guess_json_utf, super_len, check_header_validity) from .compat import ( + Callable, Mapping, cookielib, urlunparse, urlsplit, urlencode, str, bytes, is_py2, chardet, builtin_str, basestring) from .compat import json as complexjson @@ -155,8 +155,12 @@ class RequestEncodingMixin(object): if isinstance(fp, (str, bytes, bytearray)): fdata = fp - else: + elif hasattr(fp, 'read'): fdata = fp.read() + elif fp is None: + continue + else: + fdata = fp rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) @@ -174,10 +178,10 @@ class RequestHooksMixin(object): if event not in self.hooks: raise ValueError('Unsupported event specified, with event name "%s"' % (event)) - if isinstance(hook, collections.Callable): + if isinstance(hook, Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): - self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) + self.hooks[event].extend(h for h in hook if isinstance(h, Callable)) def deregister_hook(self, event, hook): """Deregister a previously registered hook. @@ -461,7 +465,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): is_stream = all([ hasattr(data, '__iter__'), - not isinstance(data, (basestring, list, tuple, collections.Mapping)) + not isinstance(data, (basestring, list, tuple, Mapping)) ]) try: @@ -686,11 +690,11 @@ class Response(object): @property def ok(self): - """Returns True if :attr:`status_code` is less than 400. + """Returns True if :attr:`status_code` is less than 400, False if not. This attribute checks if the status code of the response is between 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This + the status code is between 200 and 400, this will return True. This is **not** a check to see if the response code is ``200 OK``. """ try: @@ -820,7 +824,7 @@ class Response(object): if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() + self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b'' self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 diff --git a/src/pip/_vendor/requests/sessions.py b/src/pip/_vendor/requests/sessions.py index 6570e7334..ba135268a 100644 --- a/src/pip/_vendor/requests/sessions.py +++ b/src/pip/_vendor/requests/sessions.py @@ -8,13 +8,12 @@ This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). """ import os -import platform +import sys import time -from collections import Mapping from datetime import timedelta from .auth import _basic_auth_str -from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse +from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse, Mapping from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT @@ -38,8 +37,8 @@ from .status_codes import codes from .models import REDIRECT_STATI # Preferred clock, based on which one is more accurate on a given system. -if platform.system() == 'Windows': - try: # Python 3.3+ +if sys.platform == 'win32': + try: # Python 3.4+ preferred_clock = time.perf_counter except AttributeError: # Earlier than Python 3. preferred_clock = time.clock @@ -123,6 +122,7 @@ class SessionRedirectMixin(object): hist = [] # keep track of history url = self.get_redirect_target(resp) + previous_fragment = urlparse(req.url).fragment while url: prepared_request = req.copy() @@ -147,8 +147,12 @@ class SessionRedirectMixin(object): parsed_rurl = urlparse(resp.url) url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) - # The scheme should be lower case... + # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) parsed = urlparse(url) + if parsed.fragment == '' and previous_fragment: + parsed = parsed._replace(fragment=previous_fragment) + elif parsed.fragment: + previous_fragment = parsed.fragment url = parsed.geturl() # Facilitate relative 'location' headers, as allowed by RFC 7231. @@ -696,7 +700,7 @@ class Session(SessionRedirectMixin): """ for (prefix, adapter) in self.adapters.items(): - if url.lower().startswith(prefix): + if url.lower().startswith(prefix.lower()): return adapter # Nothing matches :-/ diff --git a/src/pip/_vendor/requests/status_codes.py b/src/pip/_vendor/requests/status_codes.py index dee89190c..ff462c6c6 100644 --- a/src/pip/_vendor/requests/status_codes.py +++ b/src/pip/_vendor/requests/status_codes.py @@ -1,5 +1,22 @@ # -*- coding: utf-8 -*- +""" +The ``codes`` object defines a mapping from common names for HTTP statuses +to their numerical codes, accessible either as attributes or as dictionary +items. + +>>> requests.codes['temporary_redirect'] +307 +>>> requests.codes.teapot +418 +>>> requests.codes['\o/'] +200 + +Some codes have multiple names, and both upper- and lower-case versions of +the names are allowed. For example, ``codes.ok``, ``codes.OK``, and +``codes.okay`` all correspond to the HTTP status code 200. +""" + from .structures import LookupDict _codes = { @@ -84,8 +101,20 @@ _codes = { codes = LookupDict(name='status_codes') -for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(('\\', '/')): - setattr(codes, title.upper(), code) +def _init(): + for code, titles in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith(('\\', '/')): + setattr(codes, title.upper(), code) + + def doc(code): + names = ', '.join('``%s``' % n for n in _codes[code]) + return '* %d: %s' % (code, names) + + global __doc__ + __doc__ = (__doc__ + '\n' + + '\n'.join(doc(code) for code in sorted(_codes)) + if __doc__ is not None else None) + +_init() diff --git a/src/pip/_vendor/requests/structures.py b/src/pip/_vendor/requests/structures.py index 05d2b3f57..da930e285 100644 --- a/src/pip/_vendor/requests/structures.py +++ b/src/pip/_vendor/requests/structures.py @@ -7,16 +7,14 @@ requests.structures Data structures that power Requests. """ -import collections - -from .compat import OrderedDict +from .compat import OrderedDict, Mapping, MutableMapping -class CaseInsensitiveDict(collections.MutableMapping): +class CaseInsensitiveDict(MutableMapping): """A case-insensitive ``dict``-like object. Implements all methods and operations of - ``collections.MutableMapping`` as well as dict's ``copy``. Also + ``MutableMapping`` as well as dict's ``copy``. Also provides ``lower_items``. All keys are expected to be strings. The structure remembers the @@ -71,7 +69,7 @@ class CaseInsensitiveDict(collections.MutableMapping): ) def __eq__(self, other): - if isinstance(other, collections.Mapping): + if isinstance(other, Mapping): other = CaseInsensitiveDict(other) else: return NotImplemented diff --git a/src/pip/_vendor/requests/utils.py b/src/pip/_vendor/requests/utils.py index 5c47de989..431f6be07 100644 --- a/src/pip/_vendor/requests/utils.py +++ b/src/pip/_vendor/requests/utils.py @@ -8,17 +8,17 @@ This module provides utility functions that are used within Requests that are also useful for external consumption. """ -import cgi import codecs -import collections import contextlib import io import os -import platform import re import socket import struct +import sys +import tempfile import warnings +import zipfile from .__version__ import __version__ from . import certs @@ -28,7 +28,7 @@ from .compat import parse_http_list as _parse_list_header from .compat import ( quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, proxy_bypass, urlunparse, basestring, integer_types, is_py3, - proxy_bypass_environment, getproxies_environment) + proxy_bypass_environment, getproxies_environment, Mapping) from .cookies import cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import ( @@ -39,19 +39,25 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() -if platform.system() == 'Windows': +if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): - if is_py3: - import winreg - else: - import _winreg as winreg + try: + if is_py3: + import winreg + else: + import _winreg as winreg + except ImportError: + return False + try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') - proxyEnable = winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0] + # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it + proxyEnable = int(winreg.QueryValueEx(internetSettings, + 'ProxyEnable')[0]) + # ProxyOverride is almost always a string proxyOverride = winreg.QueryValueEx(internetSettings, 'ProxyOverride')[0] except OSError: @@ -216,6 +222,38 @@ def guess_filename(obj): return os.path.basename(name) +def extract_zipped_paths(path): + """Replace nonexistent paths that look like they refer to a member of a zip + archive with the location of an extracted copy of the target, or else + just return the provided path unchanged. + """ + if os.path.exists(path): + # this is already a valid path, no need to do anything further + return path + + # find the first valid part of the provided path and treat that as a zip archive + # assume the rest of the path is the name of a member in the archive + archive, member = os.path.split(path) + while archive and not os.path.exists(archive): + archive, prefix = os.path.split(archive) + member = '/'.join([prefix, member]) + + if not zipfile.is_zipfile(archive): + return path + + zip_file = zipfile.ZipFile(archive) + if member not in zip_file.namelist(): + return path + + # we have a valid zip archive and a valid member of that archive + tmp = tempfile.gettempdir() + extracted_path = os.path.join(tmp, *member.split('/')) + if not os.path.exists(extracted_path): + extracted_path = zip_file.extract(member, path=tmp) + + return extracted_path + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an @@ -262,7 +300,7 @@ def to_key_val_list(value): if isinstance(value, (str, bytes, bool, int)): raise ValueError('cannot encode objects that are not 2-tuples') - if isinstance(value, collections.Mapping): + if isinstance(value, Mapping): value = value.items() return list(value) @@ -407,6 +445,31 @@ def get_encodings_from_content(content): xml_re.findall(content)) +def _parse_content_type_header(header): + """Returns content type and parameters from given header + + :param header: string + :return: tuple containing content type and dictionary of + parameters + """ + + tokens = header.split(';') + content_type, params = tokens[0].strip(), tokens[1:] + params_dict = {} + items_to_strip = "\"' " + + for param in params: + param = param.strip() + if param: + key, value = param, True + index_of_equals = param.find("=") + if index_of_equals != -1: + key = param[:index_of_equals].strip(items_to_strip) + value = param[index_of_equals + 1:].strip(items_to_strip) + params_dict[key] = value + return content_type, params_dict + + def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. @@ -419,7 +482,7 @@ def get_encoding_from_headers(headers): if not content_type: return None - content_type, params = cgi.parse_header(content_type) + content_type, params = _parse_content_type_header(content_type) if 'charset' in params: return params['charset'].strip("'\"") @@ -632,6 +695,8 @@ def should_bypass_proxies(url, no_proxy): :rtype: bool """ + # Prioritize lowercase environment variables over uppercase + # to keep a consistent behaviour with other http projects (curl, wget). get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) # First check whether no_proxy is defined. If it is, check that the URL @@ -639,28 +704,31 @@ def should_bypass_proxies(url, no_proxy): no_proxy_arg = no_proxy if no_proxy is None: no_proxy = get_proxy('no_proxy') - netloc = urlparse(url).netloc + parsed = urlparse(url) if no_proxy: # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. + # the end of the hostname, both with and without the port. no_proxy = ( host for host in no_proxy.replace(' ', '').split(',') if host ) - ip = netloc.split(':')[0] - if is_ipv4_address(ip): + if is_ipv4_address(parsed.hostname): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): - if address_in_network(ip, proxy_ip): + if address_in_network(parsed.hostname, proxy_ip): return True - elif ip == proxy_ip: + elif parsed.hostname == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True else: + host_with_port = parsed.hostname + if parsed.port: + host_with_port += ':{0}'.format(parsed.port) + for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + if parsed.hostname.endswith(host) or host_with_port.endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return True @@ -673,7 +741,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = proxy_bypass(netloc) + bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): bypass = False @@ -743,7 +811,7 @@ def default_headers(): def parse_header_links(value): - """Return a dict of parsed link headers proxies. + """Return a list of parsed link headers proxies. i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" @@ -754,6 +822,10 @@ def parse_header_links(value): replace_chars = ' \'"' + value = value.strip(replace_chars) + if not value: + return links + for val in re.split(', *<', value): try: url, params = val.split(';', 1) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 3ae627b1b..a9911cc24 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -13,7 +13,7 @@ packaging==17.1 pyparsing==2.2.0 pytoml==0.1.16 retrying==1.3.3 -requests==2.18.4 +requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 diff --git a/tasks/vendoring/patches/requests.patch b/tasks/vendoring/patches/requests.patch index d7cc8c8d6..ceee9e833 100644 --- a/tasks/vendoring/patches/requests.patch +++ b/tasks/vendoring/patches/requests.patch @@ -24,13 +24,17 @@ diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__i index 9c3b769..36a4ef40 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py -@@ -80,10 +80,12 @@ except (AssertionError, ValueError): +@@ -80,13 +80,15 @@ except (AssertionError, ValueError): RequestsDependencyWarning) # Attempt to enable urllib3's SNI support, if possible -try: - from pip._vendor.urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() +- +- # Check cryptography version +- from cryptography import __version__ as cryptography_version +- _check_cryptography(cryptography_version) -except ImportError: - pass +from pip._internal.compat import WINDOWS @@ -38,6 +42,10 @@ index 9c3b769..36a4ef40 100644 + try: + from pip._vendor.urllib3.contrib import pyopenssl + pyopenssl.inject_into_urllib3() ++ ++ # Check cryptography version ++ from cryptography import __version__ as cryptography_version ++ _check_cryptography(cryptography_version) + except ImportError: + pass From 52782f05472ce16318f662dddb380d398419648a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 19:12:15 +0530 Subject: [PATCH 011/129] Remove the "legacy" format for 'pip list' --- news/3651.removal | 1 + news/3654.removal | 1 + src/pip/_internal/commands/list.py | 43 +------- tests/functional/test_list.py | 172 +---------------------------- 4 files changed, 10 insertions(+), 207 deletions(-) create mode 100644 news/3651.removal create mode 100644 news/3654.removal diff --git a/news/3651.removal b/news/3651.removal new file mode 100644 index 000000000..d7cbb0362 --- /dev/null +++ b/news/3651.removal @@ -0,0 +1 @@ +Remove the legacy format from pip list diff --git a/news/3654.removal b/news/3654.removal new file mode 100644 index 000000000..d7cbb0362 --- /dev/null +++ b/news/3654.removal @@ -0,0 +1 @@ +Remove the legacy format from pip list diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 09f633f39..53fb015f0 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import json import logging -import warnings from pip._vendor import six from pip._vendor.six.moves import zip_longest @@ -11,7 +10,6 @@ from pip._internal.basecommand import Command from pip._internal.cmdoptions import index_group, make_option_group from pip._internal.exceptions import CommandError from pip._internal.index import PackageFinder -from pip._internal.utils.deprecation import RemovedInPip11Warning from pip._internal.utils.misc import ( dist_is_editable, get_installed_distributions, ) @@ -78,9 +76,9 @@ class ListCommand(Command): action='store', dest='list_format', default="columns", - choices=('legacy', 'columns', 'freeze', 'json'), + choices=('columns', 'freeze', 'json'), help="Select the output format among: columns (default), freeze, " - "json, or legacy.", + "or json", ) cmd_opts.add_option( @@ -123,13 +121,6 @@ class ListCommand(Command): ) def run(self, options, args): - if options.list_format == "legacy": - warnings.warn( - "The legacy format has been deprecated and will be removed " - "in the future.", - RemovedInPip11Warning, - ) - if options.outdated and options.uptodate: raise CommandError( "Options --outdated and --uptodate cannot be combined.") @@ -208,30 +199,6 @@ class ListCommand(Command): dist.latest_filetype = typ yield dist - def output_legacy(self, dist, options): - if options.verbose >= 1: - return '%s (%s, %s, %s)' % ( - dist.project_name, - dist.version, - dist.location, - get_installer(dist), - ) - elif dist_is_editable(dist): - return '%s (%s, %s)' % ( - dist.project_name, - dist.version, - dist.location, - ) - else: - return '%s (%s)' % (dist.project_name, dist.version) - - def output_legacy_latest(self, dist, options): - return '%s - Latest: %s [%s]' % ( - self.output_legacy(dist, options), - dist.latest_version, - dist.latest_filetype, - ) - def output_package_listing(self, packages, options): packages = sorted( packages, @@ -249,12 +216,6 @@ class ListCommand(Command): logger.info("%s==%s", dist.project_name, dist.version) elif options.list_format == 'json': logger.info(format_for_json(packages, options)) - elif options.list_format == "legacy": - for dist in packages: - if options.outdated: - logger.info(self.output_legacy_latest(dist, options)) - else: - logger.info(self.output_legacy(dist, options)) def output_package_listing_columns(self, data, header): # insert the header first: we need to know the size of column names diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index 234119b95..1f12d91e0 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -51,19 +51,6 @@ def test_columns_flag(script, data): assert 'simple2 3.0' in result.stdout, str(result) -def test_legacy_format(script, data): - """ - Test that legacy format - """ - script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', - ) - result = script.pip('list', '--format=legacy', expect_stderr=True) - assert 'simple (1.0)' in result.stdout, str(result) - assert 'simple2 (3.0)' in result.stdout, str(result) - - def test_format_priority(script, data): """ Test that latest format has priority over previous ones. @@ -72,18 +59,18 @@ def test_format_priority(script, data): 'install', '-f', data.find_links, '--no-index', 'simple==1.0', 'simple2==3.0', ) - result = script.pip('list', '--format=columns', '--format=legacy', + result = script.pip('list', '--format=columns', '--format=freeze', expect_stderr=True) - assert 'simple (1.0)' in result.stdout, str(result) - assert 'simple2 (3.0)' in result.stdout, str(result) + assert 'simple==1.0' in result.stdout, str(result) + assert 'simple2==3.0' in result.stdout, str(result) assert 'simple 1.0' not in result.stdout, str(result) assert 'simple2 3.0' not in result.stdout, str(result) - result = script.pip('list', '--format=legacy', '--format=columns') + result = script.pip('list', '--format=freeze', '--format=columns') assert 'Package' in result.stdout, str(result) assert 'Version' in result.stdout, str(result) - assert 'simple (1.0)' not in result.stdout, str(result) - assert 'simple2 (3.0)' not in result.stdout, str(result) + assert 'simple==1.0' not in result.stdout, str(result) + assert 'simple2==3.0' not in result.stdout, str(result) assert 'simple 1.0' in result.stdout, str(result) assert 'simple2 3.0' in result.stdout, str(result) @@ -111,17 +98,6 @@ def test_local_columns_flag(script, data): assert 'simple 1.0' in result.stdout, str(result) -def test_local_legacy_flag(script, data): - """ - Test the behavior of --local --format=legacy flags in the list - command. - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - result = script.pip('list', '--local', '--format=legacy', - expect_stderr=True) - assert 'simple (1.0)' in result.stdout - - @pytest.mark.network def test_user_flag(script, data, virtualenv): """ @@ -157,23 +133,6 @@ def test_user_columns_flag(script, data, virtualenv): assert 'simple2 2.0' in result.stdout, str(result) -@pytest.mark.network -def test_user_legacy(script, data, virtualenv): - """ - Test the behavior of --user flag in the list command - - """ - virtualenv.system_site_packages = True - script.pip('download', 'setuptools', 'wheel', '-d', data.packages) - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip('install', '-f', data.find_links, '--no-index', - '--user', 'simple2==2.0') - result = script.pip('list', '--user', '--format=legacy', - expect_stderr=True) - assert 'simple (1.0)' not in result.stdout - assert 'simple2 (2.0)' in result.stdout, str(result) - - @pytest.mark.network def test_uptodate_flag(script, data): """ @@ -225,30 +184,6 @@ def test_uptodate_columns_flag(script, data): assert 'simple2 3.0' in result.stdout, str(result) -@pytest.mark.network -def test_uptodate_legacy_flag(script, data): - """ - Test the behavior of --uptodate --format=legacy flag in the list command - - """ - script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', - ) - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--uptodate', - '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout # 3.0 is latest - assert 'pip-test-package (0.1.1,' in result.stdout # editables included - assert 'simple2 (3.0)' in result.stdout, str(result) - - @pytest.mark.network def test_outdated_flag(script, data): """ @@ -314,33 +249,6 @@ def test_outdated_columns_flag(script, data): assert 'simple2' not in result.stdout, str(result) # 3.0 is latest -@pytest.mark.network -def test_outdated_legacy(script, data): - """ - Test the behavior of --outdated --format=legacy flag in the list command - - """ - script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', 'simplewheel==1.0', - ) - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--outdated', - '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0) - Latest: 3.0 [sdist]' in result.stdout - assert 'simplewheel (1.0) - Latest: 2.0 [wheel]' in result.stdout - assert 'pip-test-package (0.1, ' in result.stdout - assert ' Latest: 0.1.1 [sdist]' in result.stdout - assert 'simple2' not in result.stdout, str(result) # 3.0 is latest - - @pytest.mark.network def test_editables_flag(script, data): """ @@ -393,25 +301,6 @@ def test_editables_columns_flag(script, data): ) -@pytest.mark.network -def test_editables_legacy(script, data): - """ - Test the behavior of --editables flag in the list command - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' - ) - result = script.pip( - 'list', '--editable', '--format=legacy', expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout, str(result) - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) - - @pytest.mark.network def test_uptodate_editables_flag(script, data): """ @@ -455,28 +344,6 @@ def test_uptodate_editables_columns_flag(script, data): ) -@pytest.mark.network -def test_uptodate_editables_legacy(script, data): - """ - test the behavior of --editable --uptodate --format=columns --format=legacy - flag in the list command - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--editable', - '--uptodate', '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout, str(result) - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) - - @pytest.mark.network def test_outdated_editables_flag(script, data): """ @@ -519,28 +386,6 @@ def test_outdated_editables_columns_flag(script, data): ) -@pytest.mark.network -def test_outdated_editables_legacy(script, data): - """ - test the behavior of --editable --outdated flag in the list command - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', - '--editable', '--outdated', '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout, str(result) - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) - - def test_outdated_pre(script, data): script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') @@ -583,11 +428,6 @@ def test_outdated_formats(script, data): ) assert 'simple==1.0' in result.stdout - # Check legacy - result = script.pip('list', '--no-index', '--find-links', wheelhouse_path, - '--outdated', '--format=legacy', expect_stderr=True) - assert 'simple (1.0) - Latest: 1.1 [wheel]' in result.stdout - # Check columns result = script.pip( 'list', '--no-index', '--find-links', wheelhouse_path, From f015aa9608c573ee2081096602e05acf19cb0402 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 19:13:21 +0530 Subject: [PATCH 012/129] Remove support for cleaning up #egg fragment postfixes --- news/4174.removal | 1 + src/pip/_internal/req/req_install.py | 22 ++-------------------- 2 files changed, 3 insertions(+), 20 deletions(-) create mode 100644 news/4174.removal diff --git a/news/4174.removal b/news/4174.removal new file mode 100644 index 000000000..a1362cf71 --- /dev/null +++ b/news/4174.removal @@ -0,0 +1 @@ +Remove support for cleaning up #egg fragment postfixes diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index bd6d4e9fe..782517f28 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -33,9 +33,7 @@ from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) from pip._internal.req.req_uninstall import UninstallPathSet -from pip._internal.utils.deprecation import ( - RemovedInPip11Warning, RemovedInPip12Warning, -) +from pip._internal.utils.deprecation import RemovedInPip12Warning from pip._internal.utils.hashes import Hashes from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( @@ -1051,22 +1049,6 @@ class InstallRequirement(object): return install_args -def _strip_postfix(req): - """ - Strip req postfix ( -dev, 0.2, etc ) - """ - # FIXME: use package_to_requirement? - match = re.search(r'^(.*?)(?:-dev|-\d.*)$', req) - if match: - # Strip off -dev, -0.2, etc. - warnings.warn( - "#egg cleanup for editable urls will be dropped in the future", - RemovedInPip11Warning, - ) - req = match.group(1) - return req - - def parse_editable(editable_req): """Parses an editable requirement into: - a requirement name @@ -1131,7 +1113,7 @@ def parse_editable(editable_req): "Could not detect requirement name for '%s', please specify one " "with #egg=your_package_name" % editable_req ) - return _strip_postfix(package_name), url, None + return package_name, url, None def deduce_helpful_msg(req): From aa146d5dc2b783553161411f06e60de9d0ba1c95 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 19:12:35 +0530 Subject: [PATCH 013/129] Kick the can down the road for dependency-links --- src/pip/_internal/index.py | 4 ++-- src/pip/_internal/operations/freeze.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index fe318a24f..857486cc1 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -29,7 +29,7 @@ from pip._internal.exceptions import ( ) from pip._internal.models.index import PyPI from pip._internal.pep425tags import get_supported -from pip._internal.utils.deprecation import RemovedInPip11Warning +from pip._internal.utils.deprecation import RemovedInPip12Warning from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, @@ -215,7 +215,7 @@ class PackageFinder(object): warnings.warn( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - RemovedInPip11Warning, + RemovedInPip12Warning, ) self.dependency_links.extend(links) diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index b6821c0e7..82d0a8c4b 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -13,7 +13,7 @@ from pip._vendor.pkg_resources import RequirementParseError from pip._internal.exceptions import InstallationError from pip._internal.req import InstallRequirement from pip._internal.req.req_file import COMMENT_RE -from pip._internal.utils.deprecation import RemovedInPip11Warning +from pip._internal.utils.deprecation import RemovedInPip12Warning from pip._internal.utils.misc import ( dist_is_editable, get_installed_distributions, ) @@ -219,7 +219,7 @@ class FrozenRequirement(object): warnings.warn( "SVN editable detection based on dependency links " "will be dropped in the future.", - RemovedInPip11Warning, + RemovedInPip12Warning, ) comments.append( '# Installing as editable to satisfy requirement %s:' % From 803286000db7968b05357415ec22a233f7497a2b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 21 Jun 2018 23:52:40 +0530 Subject: [PATCH 014/129] Remove no longer valid deprecation warning --- src/pip/_internal/utils/deprecation.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 7ec40feeb..89b66e9ad 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -20,10 +20,6 @@ class Pending(object): pass -class RemovedInPip11Warning(PipDeprecationWarning): - pass - - class RemovedInPip12Warning(PipDeprecationWarning, Pending): pass From c02af381f2e2265ff3cd181037cd0320d04046f9 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 24 Jun 2018 02:37:39 +0530 Subject: [PATCH 015/129] Caplog! --- tests/unit/test_wheel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6d3c1c7a7..ba469e34d 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -1,5 +1,6 @@ """Tests for wheel binary packages and .dist-info.""" import os +import logging import pytest from mock import Mock, patch @@ -367,7 +368,8 @@ class TestWheelBuilder(object): wb = wheel.WheelBuilder( finder=Mock(), preparer=Mock(), wheel_cache=None, ) - wb.build([wheel_req], session=Mock()) + with caplog.at_level(logging.INFO): + wb.build([wheel_req], session=Mock()) assert "due to already being wheel" in caplog.text assert mock_build_one.mock_calls == [] From 44d79d37002416ae15b7675d52d7607cc6339cf4 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 17 Jun 2018 15:01:18 +0530 Subject: [PATCH 016/129] Rename Index -> PackageIndex and add tests --- src/pip/_internal/models/index.py | 15 ++++++++++----- tests/unit/test_models.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_models.py diff --git a/src/pip/_internal/models/index.py b/src/pip/_internal/models/index.py index a7f10c8db..9539c1ba5 100644 --- a/src/pip/_internal/models/index.py +++ b/src/pip/_internal/models/index.py @@ -1,15 +1,20 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse -class Index(object): +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + def __init__(self, url): + super(PackageIndex, self).__init__() + self.url = url self.netloc = urllib_parse.urlsplit(url).netloc - self.simple_url = self.url_to_path('simple') - self.pypi_url = self.url_to_path('pypi') + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') - def url_to_path(self, path): + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = PackageIndex('https://pypi.org/') diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py new file mode 100644 index 000000000..b5960cb21 --- /dev/null +++ b/tests/unit/test_models.py @@ -0,0 +1,28 @@ +"""Tests for various classes in pip._internal.models +""" + +from pip._vendor.packaging.version import parse as parse_version + +from pip._internal.models import index + + +class TestPackageIndex(object): + """Tests for pip._internal.models.index.PackageIndex + """ + + def test_gives_right_urls(self): + url = "https://mypypi.internal/path/" + pack_index = index.PackageIndex(url) + + assert pack_index.netloc == "mypypi.internal" + assert pack_index.url == url + assert pack_index.simple_url == url + "simple" + assert pack_index.pypi_url == url + "pypi" + + def test_PyPI_urls_are_correct(self): + pack_index = index.PyPI + + assert pack_index.netloc == "pypi.org" + assert pack_index.url == "https://pypi.org/" + assert pack_index.simple_url == "https://pypi.org/simple" + assert pack_index.pypi_url == "https://pypi.org/pypi" From f1cf9a06e9559647a59e18c30052d00c224aed92 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 17 Jun 2018 15:01:55 +0530 Subject: [PATCH 017/129] Move InstallationCandidate to models + add tests --- src/pip/_internal/index.py | 42 +----------------------- src/pip/_internal/models/candidate.py | 47 +++++++++++++++++++++++++++ tests/unit/test_models.py | 21 +++++++++++- 3 files changed, 68 insertions(+), 42 deletions(-) create mode 100644 src/pip/_internal/models/candidate.py diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index fe318a24f..a1fb90289 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -28,6 +28,7 @@ from pip._internal.exceptions import ( UnsupportedWheel, ) from pip._internal.models.index import PyPI +from pip._internal.models.candidate import InstallationCandidate from pip._internal.pep425tags import get_supported from pip._internal.utils.deprecation import RemovedInPip11Warning from pip._internal.utils.logging import indent_log @@ -57,47 +58,6 @@ SECURE_ORIGINS = [ logger = logging.getLogger(__name__) -class InstallationCandidate(object): - - def __init__(self, project, version, location): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) - - def __repr__(self): - return "".format( - self.project, self.version, self.location, - ) - - def __hash__(self): - return hash(self._key) - - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) - - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) - - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) - - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) - - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) - - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) - - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented - - return method(self._key, other._key) - - class PackageFinder(object): """This finds packages. diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py new file mode 100644 index 000000000..f8e921467 --- /dev/null +++ b/src/pip/_internal/models/candidate.py @@ -0,0 +1,47 @@ +from pip._vendor.packaging.version import parse as parse_version + + +class InstallationCandidate(object): + """Represents a potential "candidate" for installation. + """ + + + def __init__(self, project, version, location): + self.project = project + self.version = parse_version(version) + self.location = location + self._key = (self.project, self.version, self.location) + + def __repr__(self): + return "".format( + self.project, self.version, self.location, + ) + + # NOTE: pip._internal.index.Link does something similar. + # We could have a "key-based-compare" mixin that these both use. + def __hash__(self): + return hash(self._key) + + def __lt__(self, other): + return self._compare(other, lambda s, o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s, o: s <= o) + + def __eq__(self, other): + return self._compare(other, lambda s, o: s == o) + + def __ge__(self, other): + return self._compare(other, lambda s, o: s >= o) + + def __gt__(self, other): + return self._compare(other, lambda s, o: s > o) + + def __ne__(self, other): + return self._compare(other, lambda s, o: s != o) + + def _compare(self, other, method): + if not isinstance(other, InstallationCandidate): + return NotImplemented + + return method(self._key, other._key) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index b5960cb21..eac475847 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -3,7 +3,7 @@ from pip._vendor.packaging.version import parse as parse_version -from pip._internal.models import index +from pip._internal.models import index, candidate class TestPackageIndex(object): @@ -26,3 +26,22 @@ class TestPackageIndex(object): assert pack_index.url == "https://pypi.org/" assert pack_index.simple_url == "https://pypi.org/simple" assert pack_index.pypi_url == "https://pypi.org/pypi" + + +class TestInstallationCandidate(object): + + def test_sets_correct_variables(self): + obj = candidate.InstallationCandidate( + "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" + ) + assert obj.project == "A" + assert obj.version == parse_version("1.0.0") + assert obj.location == "https://somewhere.com/path/A-1.0.0.tar.gz" + + # NOTE: This isn't checking the ordering logic; only the data provided to + # it is correct. + def test_sets_the_right_key(self): + obj = candidate.InstallationCandidate( + "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" + ) + assert obj._key == (obj.project, obj.version, obj.location) From 39995b32db57599ab50bccb674151b6dde24ab64 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 18 Jun 2018 16:29:56 +0530 Subject: [PATCH 018/129] Move Link to models --- src/pip/_internal/cache.py | 3 +- src/pip/_internal/index.py | 163 +------------------------- src/pip/_internal/models/link.py | 168 +++++++++++++++++++++++++++ src/pip/_internal/req/req_install.py | 2 +- src/pip/_internal/vcs/subversion.py | 2 +- src/pip/_internal/wheel.py | 3 +- 6 files changed, 176 insertions(+), 165 deletions(-) create mode 100644 src/pip/_internal/models/link.py diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 1aa17aa96..91d54b1b0 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -11,6 +11,7 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._internal import index from pip._internal.compat import expanduser from pip._internal.download import path_to_url +from pip._internal.models.link import Link from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import InvalidWheelFilename, Wheel @@ -101,7 +102,7 @@ class Cache(object): root = self.get_path_for_link(link) path = os.path.join(root, candidate) - return index.Link(path_to_url(path)) + return Link(path_to_url(path)) def cleanup(self): pass diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index a1fb90289..42f45cc9b 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -29,12 +29,13 @@ from pip._internal.exceptions import ( ) from pip._internal.models.index import PyPI from pip._internal.models.candidate import InstallationCandidate +from pip._internal.models.link import Link from pip._internal.pep425tags import get_supported from pip._internal.utils.deprecation import RemovedInPip11Warning from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, - remove_auth_from_url, splitext, + remove_auth_from_url, ) from pip._internal.utils.packaging import check_requires_python from pip._internal.wheel import Wheel, wheel_ext @@ -868,166 +869,6 @@ class HTMLPage(object): lambda match: '%%%2x' % ord(match.group(0)), url) -class Link(object): - - def __init__(self, url, comes_from=None, requires_python=None): - """ - Object representing a parsed link from https://pypi.org/simple/* - - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. - """ - - # url can be a UNC windows share - if url.startswith('\\\\'): - url = path_to_url(url) - - self.url = url - self.comes_from = comes_from - self.requires_python = requires_python if requires_python else None - - def __str__(self): - if self.requires_python: - rp = ' (requires-python:%s)' % self.requires_python - else: - rp = '' - if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) - else: - return str(self.url) - - def __repr__(self): - return '' % self - - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc - name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) - return name - - @property - def scheme(self): - return urllib_parse.urlsplit(self.url)[0] - - @property - def netloc(self): - return urllib_parse.urlsplit(self.url)[1] - - @property - def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - @property - def ext(self): - return self.splitext()[1] - - @property - def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) - return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) - - _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') - - @property - def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _hash_re = re.compile( - r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' - ) - - @property - def hash(self): - match = self._hash_re.search(self.url) - if match: - return match.group(2) - return None - - @property - def hash_name(self): - match = self._hash_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - @property - def is_wheel(self): - return self.ext == wheel_ext - - @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ - from pip._internal.vcs import vcs - - if self.scheme in vcs.all_schemes: - return False - - return True - - FormatControl = namedtuple('FormatControl', 'no_binary only_binary') """This object has two fields, no_binary and only_binary. diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py new file mode 100644 index 000000000..f2d8d99cb --- /dev/null +++ b/src/pip/_internal/models/link.py @@ -0,0 +1,168 @@ +import posixpath +import re + +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.download import path_to_url +from pip._internal.utils.misc import splitext +from pip._internal.wheel import wheel_ext + + +class Link(object): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__(self, url, comes_from=None, requires_python=None): + """ + url: + url of the resource pointed to (href of the link) + comes_from: + instance of HTMLPage where the link was found, or string. + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self.url = url + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + + def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (self.url, self.comes_from, rp) + else: + return str(self.url) + + def __repr__(self): + return '' % self + + def __eq__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.url == other.url + + def __ne__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.url != other.url + + def __lt__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.url < other.url + + def __le__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.url <= other.url + + def __gt__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.url > other.url + + def __ge__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.url >= other.url + + def __hash__(self): + return hash(self.url) + + @property + def filename(self): + _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) + name = posixpath.basename(path.rstrip('/')) or netloc + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self.url) + return name + + @property + def scheme(self): + return urllib_parse.urlsplit(self.url)[0] + + @property + def netloc(self): + return urllib_parse.urlsplit(self.url)[1] + + @property + def path(self): + return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + return self.splitext()[1] + + @property + def url_without_fragment(self): + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + match = self._subdirectory_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + match = self._hash_re.search(self.url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + match = self._hash_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_wheel(self): + return self.ext == wheel_ext + + @property + def is_artifact(self): + """ + Determines if this points to an actual artifact (e.g. a tarball) or if + it points to an "abstract" thing like a path or a VCS location. + """ + from pip._internal.vcs import vcs + + if self.scheme in vcs.all_schemes: + return False + + return True diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 043353048..0a3463567 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -137,7 +137,7 @@ class InstallRequirement(object): @classmethod def from_editable(cls, editable_req, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): - from pip._internal.index import Link + from pip._internal.models.link import Link # TODO: move this to top name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index de448c7c6..8f1949060 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -6,7 +6,7 @@ import re from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._internal.index import Link +from pip._internal.models.link import Link from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import display_path, remove_auth_from_url, rmtree from pip._internal.vcs import VersionControl, vcs diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 667385976..fc129ca74 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -710,6 +710,7 @@ class WheelBuilder(object): :return: True if all the wheels built correctly. """ from pip._internal import index + from pip._internal.models.link import Link building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir @@ -802,7 +803,7 @@ class WheelBuilder(object): self.preparer.build_dir ) # Update the link for this. - req.link = index.Link(path_to_url(wheel_file)) + req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( From 06314455e9bd67748042f03cd7d235317180fd8f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 18 Jun 2018 16:40:05 +0530 Subject: [PATCH 019/129] Add a utility for comparision logic and switch to it It uses the operator module, instead of inline-lambdas --- src/pip/_internal/models/candidate.py | 40 ++++++-------------------- src/pip/_internal/models/link.py | 41 +++++---------------------- src/pip/_internal/utils/models.py | 40 ++++++++++++++++++++++++++ tests/unit/test_models.py | 2 +- 4 files changed, 56 insertions(+), 67 deletions(-) create mode 100644 src/pip/_internal/utils/models.py diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index f8e921467..c736de6c9 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -1,47 +1,23 @@ from pip._vendor.packaging.version import parse as parse_version +from pip._internal.utils.models import KeyBasedCompareMixin -class InstallationCandidate(object): + +class InstallationCandidate(KeyBasedCompareMixin): """Represents a potential "candidate" for installation. """ - def __init__(self, project, version, location): self.project = project self.version = parse_version(version) self.location = location - self._key = (self.project, self.version, self.location) + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), + defining_class=InstallationCandidate + ) def __repr__(self): return "".format( self.project, self.version, self.location, ) - - # NOTE: pip._internal.index.Link does something similar. - # We could have a "key-based-compare" mixin that these both use. - def __hash__(self): - return hash(self._key) - - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) - - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) - - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) - - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) - - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) - - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) - - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented - - return method(self._key, other._key) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index f2d8d99cb..5decb7cc4 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -5,10 +5,11 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._internal.download import path_to_url from pip._internal.utils.misc import splitext +from pip._internal.utils.models import KeyBasedCompareMixin from pip._internal.wheel import wheel_ext -class Link(object): +class Link(KeyBasedCompareMixin): """Represents a parsed link from a Package Index's simple URL """ @@ -32,6 +33,11 @@ class Link(object): self.comes_from = comes_from self.requires_python = requires_python if requires_python else None + super(Link, self).__init__( + key=(self.url), + defining_class=Link + ) + def __str__(self): if self.requires_python: rp = ' (requires-python:%s)' % self.requires_python @@ -45,39 +51,6 @@ class Link(object): def __repr__(self): return '' % self - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - @property def filename(self): _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) diff --git a/src/pip/_internal/utils/models.py b/src/pip/_internal/utils/models.py new file mode 100644 index 000000000..d5cb80a7c --- /dev/null +++ b/src/pip/_internal/utils/models.py @@ -0,0 +1,40 @@ +"""Utilities for defining models +""" + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparision capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index eac475847..8d11dc1a6 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -44,4 +44,4 @@ class TestInstallationCandidate(object): obj = candidate.InstallationCandidate( "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" ) - assert obj._key == (obj.project, obj.version, obj.location) + assert obj._compare_key == (obj.project, obj.version, obj.location) From 89223dd47c5cb865c4ef383d7ad293e3a7cad2d5 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 24 Jun 2018 02:40:40 +0530 Subject: [PATCH 020/129] Sort imports --- src/pip/_internal/index.py | 2 +- tests/unit/test_models.py | 2 +- tests/unit/test_wheel.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 42f45cc9b..a0d37202a 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -27,8 +27,8 @@ from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) -from pip._internal.models.index import PyPI from pip._internal.models.candidate import InstallationCandidate +from pip._internal.models.index import PyPI from pip._internal.models.link import Link from pip._internal.pep425tags import get_supported from pip._internal.utils.deprecation import RemovedInPip11Warning diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 8d11dc1a6..1c8aed12b 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -3,7 +3,7 @@ from pip._vendor.packaging.version import parse as parse_version -from pip._internal.models import index, candidate +from pip._internal.models import candidate, index class TestPackageIndex(object): diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index ba469e34d..e142ee326 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -1,6 +1,6 @@ """Tests for wheel binary packages and .dist-info.""" -import os import logging +import os import pytest from mock import Mock, patch From f5c6e1adec9adda3063b0b2018f02866d6d5c39c Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 24 Jun 2018 14:03:14 +0530 Subject: [PATCH 021/129] Move import to top --- src/pip/_internal/req/req_install.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 0a3463567..9c2fc7530 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -32,6 +32,7 @@ from pip._internal.exceptions import InstallationError from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pip._internal.models.link import Link from pip._internal.req.req_uninstall import UninstallPathSet from pip._internal.utils.deprecation import ( RemovedInPip11Warning, RemovedInPip12Warning, @@ -137,8 +138,6 @@ class InstallRequirement(object): @classmethod def from_editable(cls, editable_req, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): - from pip._internal.models.link import Link # TODO: move this to top - name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) From b458b25f979b26a23ae7eb8977df09cae3a4a047 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 16:08:14 +0530 Subject: [PATCH 022/129] Update progress to 1.4 --- news/progress.vendor | 1 + src/pip/_vendor/progress/__init__.py | 2 +- src/pip/_vendor/progress/bar.py | 8 +++++++- src/pip/_vendor/progress/helpers.py | 14 +++++++------- src/pip/_vendor/vendor.txt | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 news/progress.vendor diff --git a/news/progress.vendor b/news/progress.vendor new file mode 100644 index 000000000..583728427 --- /dev/null +++ b/news/progress.vendor @@ -0,0 +1 @@ +Update progress to 1.4 diff --git a/src/pip/_vendor/progress/__init__.py b/src/pip/_vendor/progress/__init__.py index 09dfc1ebe..a41f65dc5 100644 --- a/src/pip/_vendor/progress/__init__.py +++ b/src/pip/_vendor/progress/__init__.py @@ -21,7 +21,7 @@ from sys import stderr from time import time -__version__ = '1.3' +__version__ = '1.4' class Infinite(object): diff --git a/src/pip/_vendor/progress/bar.py b/src/pip/_vendor/progress/bar.py index 5ee968f0c..025e61c45 100644 --- a/src/pip/_vendor/progress/bar.py +++ b/src/pip/_vendor/progress/bar.py @@ -15,6 +15,9 @@ # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals + +import sys + from . import Progress from .helpers import WritelnMixin @@ -61,7 +64,10 @@ class FillingCirclesBar(ChargingBar): class IncrementalBar(Bar): - phases = (' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█') + if sys.platform.startswith('win'): + phases = (u' ', u'▌', u'█') + else: + phases = (' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█') def update(self): nphases = len(self.phases) diff --git a/src/pip/_vendor/progress/helpers.py b/src/pip/_vendor/progress/helpers.py index 9ed90b2bc..0cde44ec2 100644 --- a/src/pip/_vendor/progress/helpers.py +++ b/src/pip/_vendor/progress/helpers.py @@ -28,14 +28,14 @@ class WriteMixin(object): if message: self.message = message - if self.file.isatty(): + if self.file and self.file.isatty(): if self.hide_cursor: print(HIDE_CURSOR, end='', file=self.file) print(self.message, end='', file=self.file) self.file.flush() def write(self, s): - if self.file.isatty(): + if self.file and self.file.isatty(): b = '\b' * self._width c = s.ljust(self._width) print(b + c, end='', file=self.file) @@ -43,7 +43,7 @@ class WriteMixin(object): self.file.flush() def finish(self): - if self.file.isatty() and self.hide_cursor: + if self.file and self.file.isatty() and self.hide_cursor: print(SHOW_CURSOR, end='', file=self.file) @@ -55,21 +55,21 @@ class WritelnMixin(object): if message: self.message = message - if self.file.isatty() and self.hide_cursor: + if self.file and self.file.isatty() and self.hide_cursor: print(HIDE_CURSOR, end='', file=self.file) def clearln(self): - if self.file.isatty(): + if self.file and self.file.isatty(): print('\r\x1b[K', end='', file=self.file) def writeln(self, line): - if self.file.isatty(): + if self.file and self.file.isatty(): self.clearln() print(line, end='', file=self.file) self.file.flush() def finish(self): - if self.file.isatty(): + if self.file and self.file.isatty(): print(file=self.file) if self.hide_cursor: print(SHOW_CURSOR, end='', file=self.file) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index a9911cc24..b9854e9ad 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -7,7 +7,7 @@ colorama==0.3.9 CacheControl==0.12.5 msgpack-python==0.5.6 lockfile==0.12.2 -progress==1.3 +progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 packaging==17.1 pyparsing==2.2.0 From 4ddc705d209e0253c4dd45d0341a0450b2d2e446 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 17:00:40 +0530 Subject: [PATCH 023/129] Simplify linter configuration to be consistent --- setup.cfg | 12 +++++++++--- tox.ini | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 66c036df6..d811e7989 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,9 @@ [isort] skip = - _vendor - __main__.py + .tox, + .scratch, + _vendor, + data multi_line_output = 5 known_third_party = pip._vendor @@ -12,7 +14,11 @@ default_section = THIRDPARTY include_trailing_comma = true [flake8] -exclude = .tox,.idea,.scratch,*.egg,build,_vendor,data +exclude = + .tox, + .scratch, + _vendor, + data select = E,W,F [mypy] diff --git a/tox.ini b/tox.ini index 866db977f..46f0fb3e0 100644 --- a/tox.ini +++ b/tox.ini @@ -36,8 +36,8 @@ commands = [lint] deps = -r{toxinidir}/tools/lint-requirements.txt commands = - flake8 . - isort --recursive --check-only --diff src/pip tests + flake8 + isort --check-only --diff [testenv:lint-py2] basepython = python2 From c2b1ce9fdc3e46acf0fe1cb1950f529d8c8c0d95 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 17:00:54 +0530 Subject: [PATCH 024/129] Fix lint errors on newly covered files --- docs/conf.py | 2 +- setup.py | 2 +- src/pip/__main__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e615e6be8..0d7ce04a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,10 +11,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import glob import os import re import sys -import glob on_rtd = os.environ.get('READTHEDOCS', None) == 'True' diff --git a/setup.py b/setup.py index f52160a8f..d15c09f80 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import os import re import sys -from setuptools import setup, find_packages +from setuptools import find_packages, setup here = os.path.abspath(os.path.dirname(__file__)) diff --git a/src/pip/__main__.py b/src/pip/__main__.py index 4609582c3..0c223f8c1 100644 --- a/src/pip/__main__.py +++ b/src/pip/__main__.py @@ -13,7 +13,7 @@ if __package__ == '': path = os.path.dirname(os.path.dirname(__file__)) sys.path.insert(0, path) -from pip._internal import main as _main # noqa +from pip._internal import main as _main # isort:skip # noqa if __name__ == '__main__': sys.exit(_main()) From 3ae521974f2d5913585ecb6ffc90990f9fb9aa43 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Thu, 19 Apr 2018 20:01:26 +0200 Subject: [PATCH 025/129] enable source installs for build dependencies --- docs/reference/pip.rst | 5 +++-- news/5229.feature | 1 + src/pip/_internal/build_env.py | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 news/5229.feature diff --git a/docs/reference/pip.rst b/docs/reference/pip.rst index dd68f65ad..905be1ea6 100644 --- a/docs/reference/pip.rst +++ b/docs/reference/pip.rst @@ -154,8 +154,9 @@ appropriately. installation of build dependencies from source has been disabled until a safe resolution of this issue is found. -* ``pip<18.0`` does not support the use of environment markers and extras, only - version specifiers are respected. +* ``pip<18.0``: only support installing build requirements from wheels, and + does not support the use of environment markers and extras (only version + specifiers are respected). Future Developments diff --git a/news/5229.feature b/news/5229.feature new file mode 100644 index 000000000..8df75ad16 --- /dev/null +++ b/news/5229.feature @@ -0,0 +1 @@ +Add support for installing PEP 518 build dependencies from source. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 612b46352..875ad3f84 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -80,10 +80,13 @@ class BuildEnvironment(object): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--no-user', '--prefix', self.path, '--no-warn-script-location', - '--only-binary', ':all:', ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) if finder.index_urls: args.extend(['-i', finder.index_urls[0]]) for extra_index in finder.index_urls[1:]: From 43b8ed4945f259deed630f7eddfc4d22c12872e7 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 23 Apr 2018 11:41:34 +0200 Subject: [PATCH 026/129] detect fork-bombs during build dependencies installs --- src/pip/_internal/commands/download.py | 4 +- src/pip/_internal/commands/install.py | 4 +- src/pip/_internal/commands/wheel.py | 5 +- src/pip/_internal/operations/prepare.py | 9 +- src/pip/_internal/req/req_tracker.py | 77 ++++++++++++++++++ tests/conftest.py | 3 + .../data/packages/pep518_forkbomb-235.tar.gz | Bin 0 -> 752 bytes .../pep518_twin_forkbombs_first-234.tar.gz | Bin 0 -> 806 bytes .../pep518_twin_forkbombs_second-238.tar.gz | Bin 0 -> 808 bytes .../data/src/pep518_forkbomb-235/MANIFEST.in | 1 + .../pep518_forkbomb-235/pep518_forkbomb.py | 0 .../src/pep518_forkbomb-235/pyproject.toml | 2 + tests/data/src/pep518_forkbomb-235/setup.cfg | 0 tests/data/src/pep518_forkbomb-235/setup.py | 5 ++ .../MANIFEST.in | 1 + .../pep518_twin_forkbombs_first.py | 0 .../pyproject.toml | 2 + .../pep518_twin_forkbombs_first-234/setup.cfg | 0 .../pep518_twin_forkbombs_first-234/setup.py | 5 ++ .../MANIFEST.in | 1 + .../pep518_twin_forkbombs_second.py | 0 .../pyproject.toml | 2 + .../setup.cfg | 0 .../pep518_twin_forkbombs_second-238/setup.py | 5 ++ tests/functional/test_install.py | 19 +++++ tests/unit/test_req.py | 2 + 26 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 src/pip/_internal/req/req_tracker.py create mode 100644 tests/data/packages/pep518_forkbomb-235.tar.gz create mode 100644 tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz create mode 100644 tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz create mode 100644 tests/data/src/pep518_forkbomb-235/MANIFEST.in create mode 100644 tests/data/src/pep518_forkbomb-235/pep518_forkbomb.py create mode 100644 tests/data/src/pep518_forkbomb-235/pyproject.toml create mode 100644 tests/data/src/pep518_forkbomb-235/setup.cfg create mode 100644 tests/data/src/pep518_forkbomb-235/setup.py create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/pep518_twin_forkbombs_first.py create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/setup.cfg create mode 100644 tests/data/src/pep518_twin_forkbombs_first-234/setup.py create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/pep518_twin_forkbombs_second.py create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/setup.cfg create mode 100644 tests/data/src/pep518_twin_forkbombs_second-238/setup.py diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 66bcbd5c8..cf4827c58 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError from pip._internal.index import FormatControl from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.misc import ensure_dir, normalize_path @@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: @@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index da1146b9c..f42a1d137 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global from pip._internal.operations.check import check_install_conflicts from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet, install_given_reqs +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.status_codes import ERROR from pip._internal.utils.filesystem import check_path_owner @@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="install" ) as directory: requirement_set = RequirementSet( @@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 0fb72c1a9..41893876d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache from pip._internal.exceptions import CommandError, PreviousBuildDirError from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import WheelBuilder @@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand): build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="wheel" ) as directory: + requirement_set = RequirementSet( require_hashes=options.require_hashes, ) @@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand): wheel_download_dir=options.wheel_dir, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3ad721f01..7740c2843 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -141,11 +141,12 @@ class RequirementPreparer(object): """ def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation): + progress_bar, build_isolation, req_tracker): super(RequirementPreparer, self).__init__() self.src_dir = src_dir self.build_dir = build_dir + self.req_tracker = req_tracker # Where still packed archives should be written to. If None, they are # not saved, and are deleted immediately after unpacking. @@ -293,7 +294,8 @@ class RequirementPreparer(object): (req, exc, req.link) ) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: @@ -319,7 +321,8 @@ class RequirementPreparer(object): req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: req.archive(self.download_dir) diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py new file mode 100644 index 000000000..4869bb3a9 --- /dev/null +++ b/src/pip/_internal/req/req_tracker.py @@ -0,0 +1,77 @@ +from __future__ import absolute_import + +import contextlib +import errno +import hashlib +import logging +import os + +from pip._internal.utils.temp_dir import TempDirectory + + +logger = logging.getLogger(__name__) + + +class RequirementTracker(object): + + def __init__(self): + self._root = os.environ.get('PIP_REQ_TRACKER') + if self._root is None: + self._temp_dir = TempDirectory(delete=False, kind='req-tracker') + self._temp_dir.create() + self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path + logger.debug('Created requirements tracker %r', self._root) + else: + self._temp_dir = None + logger.debug('Re-using requirements tracker %r', self._root) + self._entries = set() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + def _entry_path(self, link): + hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() + return os.path.join(self._root, hashed) + + def add(self, req): + link = req.link + info = str(req) + entry_path = self._entry_path(link) + try: + with open(entry_path) as fp: + # Error, these's already a build in progress. + raise LookupError('%s is already being built: %s' + % (link, fp.read())) + except IOError as e: + if e.errno != errno.ENOENT: + raise + assert req not in self._entries + with open(entry_path, 'w') as fp: + fp.write(info) + self._entries.add(req) + logger.debug('Added %s to build tracker %r', req, self._root) + + def remove(self, req): + link = req.link + self._entries.remove(req) + os.unlink(self._entry_path(link)) + logger.debug('Removed %s from build tracker %r', req, self._root) + + def cleanup(self): + for req in set(self._entries): + self.remove(req) + remove = self._temp_dir is not None + if remove: + self._temp_dir.cleanup() + logger.debug('%s build tracker %r', + 'Removed' if remove else 'Cleaned', + self._root) + + @contextlib.contextmanager + def track(self, req): + self.add(req) + yield + self.remove(req) diff --git a/tests/conftest.py b/tests/conftest.py index 1e927fc13..f8e3a41b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,6 +129,9 @@ def isolate(tmpdir): # We want to disable the version check from running in the tests os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: diff --git a/tests/data/packages/pep518_forkbomb-235.tar.gz b/tests/data/packages/pep518_forkbomb-235.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1edce52d8a4dd3d54cc58893710e99abb6fb9654 GIT binary patch literal 752 zcmVV(ErKlfB62bvESW0 zxQG7UHTqk=UDMy@uB$N){U4o=q{30CLgNc%c_h+pX12NB4HMdC^L=0}_JO`!Qf(5w zS0qW8mD}v7`@VbdwX44q2^lh+&?>Z3s1uRZuMJAVC|*TBL_$=NQm!-_aHyWz85=pG*qzc&{jwip*rYml0 z7WLa~r>hr@NK!hYbXW#)l%D3-ufAu6g8y&j|I4&kNB!q!&HrB20>J-GfPxSI*ZqIb zF@JsYzv($1`u`^~{|A`=pPB#9^@NT1ze)ge|J$Zz`F`bp->lLA{NDgkI*f}Ev09@K zV?1O3FQ>eQ>OXh9TK)Gej`_b4j;rA=m=1S+U4@fiP(<;Fb=d5p>I1KIo^iwFqtg(h19|72-%bT4paa5XVFUvzhAZeM0^a%*C5ZDMm@W@&PBbS*M7 zG%j>uascg`U2mH(6o$FxSD@T?V^ehAt#g-Fay($e?O8e zNy}Q2Zb_D|?{kAOcD{}u+s8^miKV@9#BC5eL!MsudDPFGVUT7*)eUn`S^=!YW^8oT zZ0p{2iliBarddq4TV=goHeixsNgE)~gqtdg)K3CRQ+l-BqUX~uGQB<%k_GXvdEO-7 zSA%%S_f{T%o0*9J^ND}=^y2g3QMW06h{t}KR!#g(qY{6E*}5Sq4Dnw(pAI@F$4A}s zW)RDDu}%s(DUy_bBTqC1k3u;kkta=W=>IB>S}ZeI&OW?TPddk^@>7S_avcVgP&}a6 zE1WQhue0TQ3Z6J2-B9@0IQ}15iT~4()~HeEzhP-r|23lp|2N`433uQMSN%e17Vvmq z(V9%|xDo9ulh~`H*sHVHtK-k-K>mxoNSlTy#EkPQP~K_dFssaebOs9pvJQ zr`3CvA~y)%Wfme3(mt#38I9jVp1bdPWE5L52~|in_yAA<0Kr9>RR910 literal 0 HcmV?d00001 diff --git a/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz b/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..75998c3d4df1b86c8f4b770692f7ff3d94a5ca75 GIT binary patch literal 808 zcmV+@1K0c?iwFqZg(h19|72-%bT4paa5XVFUvzhAZeM0^a%*C5ZDMm@b7f<1Ze%So zGdM1EVR8WNnNL%jKpciS^C_5IY-c3y0s+UF92~2yV;h}D+k=x~2-zSN*exttzI{p3 zBpM^enxxUb&jqqs^6%Ye+5JT@(V)AoybTk#&(rHJkGq+hF`p+r-7t4|l$S$F96|<1 z>NM^h$0*b^DQytLF)f7{gj!V58p;~TGwG$8rgTNf(v&^g@2Tgk9{)73lE_ocO8I!rF`~Be0*`UWlgXQexqkh~vI#r)Nwidji#{^4y zEb$9B3X|*X8J9{f+=$(v_`Czp*q6 zd9tU`I#F9*%=WZV^!gxreGHnP#;!Em(mHr#f z$bZW*2{jdBP}_n3YavYhDDN?CW^WHM;Qs>me^D&O|8f0y;=fI81O9JB{*{%T<;?#N z>OTfGEQ0(uqyF>A-VpVl|6~3i*Y;QFzhO-GfA*lgT>n{E|E-07%42PqvE)3;v@jMt zm7^AQ^vBM-q?`?!TU8@vYg^TZ+>B<0=2kg$CXJ$SW8TXnmNhS{U#9aOnr_ zCH_<6ZvDq7{WnqnT@#nxJdApJR%DXJ9W`Y?^Dt$Z*3>R*(`li$t<`R?7>foc^M_17 zOfy;Qz~47{|M&aB>hAx Date: Fri, 27 Apr 2018 11:43:25 +0200 Subject: [PATCH 027/129] tests: drop `package4` test data No need for a wheel of `simple` now that build dependencies support source installs. --- .../packages4/simple-1.0-py2.py3-none-any.whl | Bin 1745 -> 0 bytes tests/functional/test_install.py | 3 --- tests/lib/__init__.py | 8 -------- 3 files changed, 11 deletions(-) delete mode 100644 tests/data/packages4/simple-1.0-py2.py3-none-any.whl diff --git a/tests/data/packages4/simple-1.0-py2.py3-none-any.whl b/tests/data/packages4/simple-1.0-py2.py3-none-any.whl deleted file mode 100644 index 3b91f5e0102903daa87cd40f85828d90ca61433b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1745 zcmWIWW@Zs#U|`^2(4S@C*wDlp#tG!H1F;Yg7iZ=c%u{r100!{f1#7ekK%S|mwOi3(B)XOT)&x`Fp&2`8?z_os9 z%2I`euguOzk_F^WX^)h2|ibYKD}3q2#7gXJLfcyi_?m|MJv`k+-l0fX8Dcp z#3}{xZRuC1PKS)Pi3{YI8*0!s$w=IRg*i^omgE=2=cJaU=IE7FlxUwjqkF>hDSD`V zyL)M0JTSEKfLIDwsD*pDy83XR_wYS?k=I*S>)e_1n}ZCl7(XcTKkKdQrE^k$Q;>(( zNgcgTefE`s7Yr^Kn_V!z@PzBsdHsuDb*}Mh=z5(!>9Z+FLsQH5DVL|u_8^TFCYR2h zKJR_i_sr^+P6=g~9cW2oaaH&K2B33J0kJGDXZgB@IJ!87IL7wwYHYwTd3<~3RgQ(t_Zwx{9B z-Mp38cDP*FE|Pb|=g=QCr-xl>+nvhDz@W&)z#xsw=|Qf}{y{FK7rpZr8;Gzzs9pL` z*oXJ&)uTFRlv~@Jm^d%w2(WON`6o_xs;@OwPAyxr{)_C~V$FBQw$*qB-$|af?9Yy( zE$3V!e@@;L8lNY0fLXcv<($iU!2wT>o|IZRJNe%94Jwa!=X9@`Xz}%D=fCEL1J3&n zOkuneez-fc)$M_dU*G$W!m0eX%>6#U(R^n*@AfUec<#zo9*+GwVn^e)>gJ1I;WC`^ z$*h3s&IbFn!bTT4Z7<}1Y2IoZV;57?sde0cj*-p%>nm8N&yN)rPH>-9<$CJ~n_tVL zHmQg<`57|X3;S|U7s+?}I@&7ymVfam;WqP)=}o(?9{DSE=uVGz=C%FypO2kh<@vbf ztX%BLO-jj<{x2@rXIOlVU%;B~IRD_pl4G|^moLlcQ9hdaHe7RMKYMM;z6HWZIyY@_ z+??oh`%~ODo9KW2^Iu=>dLwF5FsnG}N%i%!>}PlBxb9jti8Vo@=WlG(uJ5<+@BhaX z;LXS+!i>AP0!9%SG&F)J%)$#%mbGg=;okjYJ@oUZWMYZLm0)+ ZjKe5MmJ9G^WdkW@2SPret!r68JOF_(W}N^4 diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index a2bcd3d38..0ef340af0 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -80,9 +80,6 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels): 'wheel', '--no-index', '-f', common_wheels, '-f', data.find_links, - # Add tests/data/packages4, which contains a wheel for - # simple==1.0 (needed by requires_simple_extra[extra]). - '-f', data.find_links4, data.src.join("pep518_with_extra_and_markers-1.0"), use_module=True, ) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b445e89e8..e0e08fd76 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -100,10 +100,6 @@ class TestData(object): def packages3(self): return self.root.join("packages3") - @property - def packages4(self): - return self.root.join("packages4") - @property def src(self): return self.root.join("src") @@ -132,10 +128,6 @@ class TestData(object): def find_links3(self): return path_to_url(self.packages3) - @property - def find_links4(self): - return path_to_url(self.packages4) - def index_url(self, index="simple"): return path_to_url(self.root.join("indexes", index)) From e7b7239be35b717fadf77e12fca59efcf1f79b4f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 26 Jun 2018 15:15:40 +0530 Subject: [PATCH 028/129] Move all logging setup into a separate function --- src/pip/_internal/basecommand.py | 93 +++--------------------------- src/pip/_internal/utils/logging.py | 90 +++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 84 deletions(-) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index 64411697b..4a3033bf2 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -24,7 +24,7 @@ from pip._internal.status_codes import ( ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND, ) -from pip._internal.utils.logging import IndentingFormatter +from pip._internal.utils.logging import setup_logging from pip._internal.utils.misc import get_prog, normalize_path from pip._internal.utils.outdated import pip_version_check from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -42,7 +42,6 @@ class Command(object): usage = None # type: Optional[str] hidden = False # type: bool ignore_require_venv = False # type: bool - log_streams = ("ext://sys.stdout", "ext://sys.stderr") def __init__(self, isolated=False): parser_kw = { @@ -114,89 +113,15 @@ class Command(object): # Set verbosity so that it can be used elsewhere. self.verbosity = options.verbose - options.quiet - if self.verbosity >= 1: - level = "DEBUG" - elif self.verbosity == -1: - level = "WARNING" - elif self.verbosity == -2: - level = "ERROR" - elif self.verbosity <= -3: - level = "CRITICAL" - else: - level = "INFO" + setup_logging( + verbosity=self.verbosity, + additional_log_file=options.log, + no_color=options.no_color, + ) - # The root logger should match the "console" level *unless* we - # specified "--log" to send debug logs to a file. - root_level = level - if options.log: - root_level = "DEBUG" - - logger_class = "pip._internal.utils.logging.ColorizedStreamHandler" - handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler" - - logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": False, - "filters": { - "exclude_warnings": { - "()": "pip._internal.utils.logging.MaxLevelFilter", - "level": logging.WARNING, - }, - }, - "formatters": { - "indent": { - "()": IndentingFormatter, - "format": "%(message)s", - }, - }, - "handlers": { - "console": { - "level": level, - "class": logger_class, - "no_color": options.no_color, - "stream": self.log_streams[0], - "filters": ["exclude_warnings"], - "formatter": "indent", - }, - "console_errors": { - "level": "WARNING", - "class": logger_class, - "no_color": options.no_color, - "stream": self.log_streams[1], - "formatter": "indent", - }, - "user_log": { - "level": "DEBUG", - "class": handler_class, - "filename": options.log or "/dev/null", - "delay": True, - "formatter": "indent", - }, - }, - "root": { - "level": root_level, - "handlers": list(filter(None, [ - "console", - "console_errors", - "user_log" if options.log else None, - ])), - }, - # Disable any logging besides WARNING unless we have DEBUG level - # logging enabled. These use both pip._vendor and the bare names - # for the case where someone unbundles our libraries. - "loggers": { - name: { - "level": ( - "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" - ) - } for name in [ - "pip._vendor", "distlib", "requests", "urllib3" - ] - }, - }) - - # TODO: try to get these passing down from the command? - # without resorting to os.environ to hold these. + # TODO: Try to get these passing down from the command? + # without resorting to os.environ to hold these. + # This also affects isolated builds and it should. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 0c1495dac..0dc029466 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -130,3 +130,93 @@ class MaxLevelFilter(logging.Filter): def filter(self, record): return record.levelno < self.level + + +def setup_logging(verbosity, additional_log_file, no_color): + """Configures and sets up all of the logging + """ + + # Determine the level to be logging at. + if verbosity >= 1: + level = "DEBUG" + elif verbosity == -1: + level = "WARNING" + elif verbosity == -2: + level = "ERROR" + elif verbosity <= -3: + level = "CRITICAL" + else: + level = "INFO" + + # The "root" logger should match the "console" level *unless* we specified + # a "--log" file to send debug logs. + root_level = level + if additional_log_file is not None: + root_level = "DEBUG" + + # Shorthand for clarity + log_streams = ("ext://sys.stdout", "ext://sys.stderr") + + logger_class = "pip._internal.utils.logging.ColorizedStreamHandler" + handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler" + + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": False, + "filters": { + "exclude_warnings": { + "()": "pip._internal.utils.logging.MaxLevelFilter", + "level": logging.WARNING, + }, + }, + "formatters": { + "indent": { + "()": IndentingFormatter, + "format": "%(message)s", + }, + }, + "handlers": { + "console": { + "level": level, + "class": logger_class, + "no_color": no_color, + "stream": log_streams[0], + "filters": ["exclude_warnings"], + "formatter": "indent", + }, + "console_errors": { + "level": "WARNING", + "class": logger_class, + "no_color": no_color, + "stream": log_streams[1], + "formatter": "indent", + }, + "user_log": { + "level": "DEBUG", + "class": handler_class, + "filename": additional_log_file or "/dev/null", + "delay": True, + "formatter": "indent", + }, + }, + "root": { + "level": root_level, + "handlers": list(filter(None, [ + "console", + "console_errors", + "user_log" if additional_log_file else None, + ])), + }, + # Disable any logging besides WARNING unless we have DEBUG level + # logging enabled. These use both pip._vendor and the bare names + # for the case where someone unbundles our libraries. + "loggers": { + name: { + "level": ( + "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" + ) + } for name in [ + "pip._vendor", "distlib", "requests", "urllib3" + ] + }, + }) From 036e5c8eec6322b1110fec43f88742dca63c70c1 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 26 Jun 2018 15:23:19 +0530 Subject: [PATCH 029/129] Simply shutdown logging instead of cleaning up --- src/pip/_internal/basecommand.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index 4a3033bf2..8abf4e1f6 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -137,8 +137,6 @@ class Command(object): ) sys.exit(VIRTUALENV_NOT_FOUND) - original_root_handlers = set(logging.root.handlers) - try: status = self.run(options, args) # FIXME: all commands should return an exit status @@ -178,10 +176,9 @@ class Command(object): retries=0, timeout=min(5, options.timeout)) as session: pip_version_check(session, options) - # Avoid leaking loggers - for handler in set(logging.root.handlers) - original_root_handlers: - # this method benefit from the Logger class internal lock - logging.root.removeHandler(handler) + + # Shutdown the logging module + logging.shutdown() return SUCCESS From 621ff77c94f4d76f199fef857fdd5b3e99bfba3a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 26 Jun 2018 15:29:11 +0530 Subject: [PATCH 030/129] Improve readability of logging setup --- src/pip/_internal/utils/logging.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 0dc029466..2432191d4 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -154,11 +154,15 @@ def setup_logging(verbosity, additional_log_file, no_color): if additional_log_file is not None: root_level = "DEBUG" - # Shorthand for clarity - log_streams = ("ext://sys.stdout", "ext://sys.stderr") - - logger_class = "pip._internal.utils.logging.ColorizedStreamHandler" - handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler" + # Shorthands for clarity + log_streams = { + "stdout": "ext://sys.stdout", + "stderr": "ext://sys.stderr", + } + handler_classes = { + "stream": "pip._internal.utils.logging.ColorizedStreamHandler", + "file": "pip._internal.utils.logging.BetterRotatingFileHandler", + } logging.config.dictConfig({ "version": 1, @@ -178,22 +182,22 @@ def setup_logging(verbosity, additional_log_file, no_color): "handlers": { "console": { "level": level, - "class": logger_class, + "class": handler_classes["stream"], "no_color": no_color, - "stream": log_streams[0], + "stream": log_streams["stdout"], "filters": ["exclude_warnings"], "formatter": "indent", }, "console_errors": { "level": "WARNING", - "class": logger_class, + "class": handler_classes["stream"], "no_color": no_color, - "stream": log_streams[1], + "stream": log_streams["stderr"], "formatter": "indent", }, "user_log": { "level": "DEBUG", - "class": handler_class, + "class": handler_classes["file"], "filename": additional_log_file or "/dev/null", "delay": True, "formatter": "indent", From ba5315956e159f882e811f079a13834ae609ea69 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 26 Jun 2018 15:31:20 +0530 Subject: [PATCH 031/129] Make it clearer when user_log is used --- src/pip/_internal/utils/logging.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 2432191d4..ec3804e2e 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -205,11 +205,9 @@ def setup_logging(verbosity, additional_log_file, no_color): }, "root": { "level": root_level, - "handlers": list(filter(None, [ - "console", - "console_errors", - "user_log" if additional_log_file else None, - ])), + "handlers": ["console", "console_errors"] + ( + ["user_log"] if additional_log_file else [] + ), }, # Disable any logging besides WARNING unless we have DEBUG level # logging enabled. These use both pip._vendor and the bare names From f19d1fc3451d71cb9d531a351d77f2e76b9a71f7 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 26 Jun 2018 16:07:58 +0530 Subject: [PATCH 032/129] Cleanup logic surrounding user_log handler --- src/pip/_internal/basecommand.py | 2 +- src/pip/_internal/utils/logging.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index 8abf4e1f6..edb3ed9a0 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -115,8 +115,8 @@ class Command(object): setup_logging( verbosity=self.verbosity, - additional_log_file=options.log, no_color=options.no_color, + user_log_file=options.log, ) # TODO: Try to get these passing down from the command? diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index ec3804e2e..839d8814d 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -132,7 +132,7 @@ class MaxLevelFilter(logging.Filter): return record.levelno < self.level -def setup_logging(verbosity, additional_log_file, no_color): +def setup_logging(verbosity, no_color, user_log_file): """Configures and sets up all of the logging """ @@ -148,11 +148,16 @@ def setup_logging(verbosity, additional_log_file, no_color): else: level = "INFO" - # The "root" logger should match the "console" level *unless* we specified - # a "--log" file to send debug logs. - root_level = level - if additional_log_file is not None: + # The "root" logger should match the "console" level *unless* we also need + # to log to a user log file. + include_user_log = user_log_file is not None + if include_user_log: + additional_log_file = user_log_file root_level = "DEBUG" + else: + additional_log_file = "/dev/null" + root_level = level + # Shorthands for clarity log_streams = { @@ -198,7 +203,7 @@ def setup_logging(verbosity, additional_log_file, no_color): "user_log": { "level": "DEBUG", "class": handler_classes["file"], - "filename": additional_log_file or "/dev/null", + "filename": additional_log_file, "delay": True, "formatter": "indent", }, @@ -206,7 +211,7 @@ def setup_logging(verbosity, additional_log_file, no_color): "root": { "level": root_level, "handlers": ["console", "console_errors"] + ( - ["user_log"] if additional_log_file else [] + ["user_log"] if include_user_log else [] ), }, # Disable any logging besides WARNING unless we have DEBUG level From e8caa3173670edea36ee498179bbaf94381b9290 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 27 Jun 2018 12:07:29 +0530 Subject: [PATCH 033/129] Disable logging only for vendored dependencies The current list is already not up-to-date. This should be handled by downstream packagers if they unvendor pip's dependencies. --- src/pip/_internal/utils/logging.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 839d8814d..c2a2f9d81 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -158,6 +158,9 @@ def setup_logging(verbosity, no_color, user_log_file): additional_log_file = "/dev/null" root_level = level + # Disable any logging besides WARNING unless we have DEBUG level logging + # enabled for vendored libraries. + vendored_log_level = "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" # Shorthands for clarity log_streams = { @@ -214,16 +217,9 @@ def setup_logging(verbosity, no_color, user_log_file): ["user_log"] if include_user_log else [] ), }, - # Disable any logging besides WARNING unless we have DEBUG level - # logging enabled. These use both pip._vendor and the bare names - # for the case where someone unbundles our libraries. "loggers": { - name: { - "level": ( - "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" - ) - } for name in [ - "pip._vendor", "distlib", "requests", "urllib3" - ] + "pip._vendor": { + "level": vendored_log_level + } }, }) From cc720b76ef89d7dfc63f7b5c1422dc22754698a9 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 17:21:21 +0530 Subject: [PATCH 034/129] Upgrade to a newer flake8 --- tools/lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lint-requirements.txt b/tools/lint-requirements.txt index a19517e78..4c166111a 100644 --- a/tools/lint-requirements.txt +++ b/tools/lint-requirements.txt @@ -1,2 +1,2 @@ -flake8 == 3.3.0 +flake8 == 3.5.0 isort == 4.2.5 From c92d13e9d38818786c0847333f9e659b45628872 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 17:21:41 +0530 Subject: [PATCH 035/129] Fix new lint errors from newer flake8 --- src/pip/_internal/basecommand.py | 2 +- src/pip/_internal/compat.py | 4 ++-- src/pip/_internal/req/__init__.py | 2 +- src/pip/_internal/wheel.py | 12 ++++++------ tests/unit/test_req_file.py | 2 +- tests/unit/test_req_uninstall.py | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index 64411697b..c325f1efa 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -240,7 +240,7 @@ class Command(object): logger.debug('Exception information:', exc_info=True) return ERROR - except: + except Exception: logger.critical('Exception:', exc_info=True) return UNKNOWN_ERROR diff --git a/src/pip/_internal/compat.py b/src/pip/_internal/compat.py index 749115075..e6c008d3b 100644 --- a/src/pip/_internal/compat.py +++ b/src/pip/_internal/compat.py @@ -217,7 +217,7 @@ else: 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') ) - except: + except Exception: return None if cr == (0, 0): return None @@ -228,7 +228,7 @@ else: fd = os.open(os.ctermid(), os.O_RDONLY) cr = ioctl_GWINSZ(fd) os.close(fd) - except: + except Exception: pass if not cr: cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py index c9b4c3ce3..b270498e2 100644 --- a/src/pip/_internal/req/__init__.py +++ b/src/pip/_internal/req/__init__.py @@ -48,7 +48,7 @@ def install_given_reqs(to_install, install_options, global_options=(), *args, **kwargs ) - except: + except Exception: should_rollback = ( requirement.conflicts_with and not requirement.install_succeeded diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 667385976..7ad7a1b8a 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -506,8 +506,8 @@ if __name__ == '__main__': row[1], row[2] = rehash(row[0]) writer.writerow(row) for f in generated: - h, l = rehash(f) - writer.writerow((normpath(f, lib_dir), h, l)) + digest, length = rehash(f) + writer.writerow((normpath(f, lib_dir), digest, length)) for f in installed: writer.writerow((installed[f], '', '')) shutil.move(temp_record, record) @@ -528,7 +528,7 @@ def wheel_version(source_dir): version = wheel_data['Wheel-Version'].strip() version = tuple(map(int, version.split('.'))) return version - except: + except Exception: return False @@ -653,7 +653,7 @@ class WheelBuilder(object): ) logger.info('Stored in directory: %s', output_dir) return wheel_path - except: + except Exception: pass # Ignore return, we can't do anything else useful. self._clean_one(req) @@ -685,7 +685,7 @@ class WheelBuilder(object): call_subprocess(wheel_args, cwd=req.setup_py_dir, show_stdout=False, spinner=spinner) return True - except: + except Exception: spinner.finish("error") logger.error('Failed building wheel for %s', req.name) return False @@ -698,7 +698,7 @@ class WheelBuilder(object): try: call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False) return True - except: + except Exception: logger.error('Failed cleaning build dir for %s', req.name) return False diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 93fec8e73..3f7c293b5 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -639,7 +639,7 @@ class TestParseRequirements(object): popen.return_value.stdout.readline.return_value = b"" try: req.install([]) - except: + except Exception: pass last_call = popen.call_args_list[-1] diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index 5e165d8c1..dc8517961 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -121,12 +121,12 @@ class TestUninstallPathSet(object): f = os.path.join(tmpdir, 'foo') with open(f, 'w'): pass - l = os.path.join(tmpdir, 'foo_link') - os.symlink(f, l) + foo_link = os.path.join(tmpdir, 'foo_link') + os.symlink(f, foo_link) ups = UninstallPathSet(dist=Mock()) - ups.add(l) - assert ups.paths == {l} + ups.add(foo_link) + assert ups.paths == {foo_link} def test_compact_shorter_path(self, monkeypatch): monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', From 85e000ede602f1be36c4be9cf539106836735cd3 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 17:21:54 +0530 Subject: [PATCH 036/129] Upgrade to newer isort --- tools/lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lint-requirements.txt b/tools/lint-requirements.txt index 4c166111a..92ed56ed6 100644 --- a/tools/lint-requirements.txt +++ b/tools/lint-requirements.txt @@ -1,2 +1,2 @@ flake8 == 3.5.0 -isort == 4.2.5 +isort == 4.3.4 From 8a565aac756f7c172a51658452721cc729a85115 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 25 Jun 2018 17:23:15 +0530 Subject: [PATCH 037/129] Sort imports as per newer isort --- src/pip/_internal/build_env.py | 1 - src/pip/_internal/operations/check.py | 1 - src/pip/_internal/req/req_install.py | 2 +- src/pip/_internal/resolve.py | 1 - tests/data/src/withpyproject/setup.py | 1 + tests/unit/test_check.py | 1 + tests/unit/test_unit_outdated.py | 1 - 7 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 612b46352..568570885 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -11,7 +11,6 @@ from pip._internal.utils.misc import call_subprocess from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.ui import open_spinner - logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index cba18ad63..799257aea 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -6,7 +6,6 @@ from collections import namedtuple from pip._vendor.packaging.utils import canonicalize_name from pip._internal.operations.prepare import make_abstract_dist - from pip._internal.utils.misc import get_installed_distributions from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 043353048..d9e3df745 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -18,8 +18,8 @@ from pip._vendor.packaging import specifiers from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import InvalidRequirement, Requirement from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.packaging.version import parse as parse_version from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import RequirementParseError, parse_requirements from pip._internal import wheel diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index 6711f9a5b..8480e48c6 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -18,7 +18,6 @@ from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) - from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_in_usersite, ensure_dir diff --git a/tests/data/src/withpyproject/setup.py b/tests/data/src/withpyproject/setup.py index 326f7fd1e..1ea9e3e41 100644 --- a/tests/data/src/withpyproject/setup.py +++ b/tests/data/src/withpyproject/setup.py @@ -1,2 +1,3 @@ from setuptools import setup + setup(name='withpyproject', version='0.0.1') diff --git a/tests/unit/test_check.py b/tests/unit/test_check.py index 4dc893ada..1d1921484 100644 --- a/tests/unit/test_check.py +++ b/tests/unit/test_check.py @@ -2,6 +2,7 @@ """ import mock + from pip._internal.operations import check diff --git a/tests/unit/test_unit_outdated.py b/tests/unit/test_unit_outdated.py index 853531db5..1f20664e7 100644 --- a/tests/unit/test_unit_outdated.py +++ b/tests/unit/test_unit_outdated.py @@ -3,7 +3,6 @@ import os import sys from contextlib import contextmanager - import freezegun import pretend import pytest From 8d65024c49f6f222e8df6e51c570768368d93063 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 27 Jun 2018 13:37:18 +0530 Subject: [PATCH 038/129] Catch *everything* in basecommand --- src/pip/_internal/basecommand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index c325f1efa..58071639c 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -240,7 +240,7 @@ class Command(object): logger.debug('Exception information:', exc_info=True) return ERROR - except Exception: + except BaseException: logger.critical('Exception:', exc_info=True) return UNKNOWN_ERROR From fdf753ff63f1c9b1653958b82b528a8d37846401 Mon Sep 17 00:00:00 2001 From: Alexander Shtyrov Date: Fri, 29 Jun 2018 17:47:58 +0100 Subject: [PATCH 039/129] Modify the wording of the installation instructions. Fixes #5532 --- docs/installing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installing.rst b/docs/installing.rst index 916c3a3ba..c4a69227f 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -23,7 +23,8 @@ To install pip, securely download `get-pip.py curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -Inspect ``get-pip.py`` for any malevolence. Then run the following:: +As when running any script downloaded from the web, ensure that you have reviewed the code and are happy that it works as you expect. +Then run the following:: python get-pip.py From 00addd16c535e60798d09408a9cb281508ce02ae Mon Sep 17 00:00:00 2001 From: Alexander Shtyrov Date: Fri, 29 Jun 2018 17:57:48 +0100 Subject: [PATCH 040/129] Mark pull request as trivial. --- news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial diff --git a/news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial b/news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial new file mode 100644 index 000000000..e69de29bb From 8f70a5b57a2d42c7c4f157fd3c0939cc548a0fe5 Mon Sep 17 00:00:00 2001 From: Alexander Shtyrov Date: Sat, 30 Jun 2018 13:26:05 +0100 Subject: [PATCH 041/129] Wrapped text to 80 characters. --- docs/installing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installing.rst b/docs/installing.rst index c4a69227f..ac9cd4fb2 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -23,7 +23,8 @@ To install pip, securely download `get-pip.py curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -As when running any script downloaded from the web, ensure that you have reviewed the code and are happy that it works as you expect. +As when running any script downloaded from the web, ensure that you have +reviewed the code and are happy that it works as you expect. Then run the following:: python get-pip.py From 30dcc7dfadae425a9db55686943d7a1d31cb8542 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 12:17:01 +0530 Subject: [PATCH 042/129] Start breaking development documentation --- docs/development.rst | 15 --------------- docs/development/index.rst | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 docs/development/index.rst diff --git a/docs/development.rst b/docs/development.rst index caac58048..2d0dc3257 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -90,21 +90,6 @@ If you are missing one of the VCS tools, you can tell ``py.test`` to skip it:: $ py.test -k 'not (svn or git)' -Getting Involved -================ - -The pip project welcomes help in the following ways: - -- Making Pull Requests for code, tests, or docs. -- Commenting on open issues and pull requests. -- Helping to answer questions on the `mailing list`_. - -If you want to become an official maintainer, start by helping out. - -Later, when you think you're ready, get in touch with one of the maintainers, -and they will initiate a vote. - - Adding a NEWS Entry =================== diff --git a/docs/development/index.rst b/docs/development/index.rst new file mode 100644 index 000000000..f7dc0fb5d --- /dev/null +++ b/docs/development/index.rst @@ -0,0 +1,26 @@ +Development +=========== + +pip, as an open-source project, welcomes contributions of all forms. The +sections below will help you get started with development, testing, and +documentation. + +Please contribute issues, submit bug reports, file feature requests, and help +triage the issues on our issue tracker on `GitHub`_. If submitting a bug report +for the first time, check out `"What to put in your bug report"`_ for guidance. + +You can also join ``#pypa`` (general packaging discussion and user support) and +``#pypa-dev`` (discussion about development of packaging tools) `on Freenode`_, +or the `pypa-dev mailing list`_, to ask questions or get involved. + +.. toctree:: + :maxdepth: 2 + + getting-started + contributing + release-process + +.. _`GitHub`: https://github.com/pypa/pip +.. _`"What to put in your bug report"`: http://www.contribution-guide.org/#what-to-put-in-your-bug-report +.. _`on Freenode`: https://webchat.freenode.net/?channels=%23pypa-dev,pypa +.. _`pypa-dev mailing list`: https://groups.google.com/forum/#!forum/pypa-dev From bf0d736fc040f48c053312214c66a9bd95e256fc Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 12:18:43 +0530 Subject: [PATCH 043/129] Move release-process docs to separate page --- docs/development.rst | 91 -------------------------- docs/development/release-process.rst | 96 ++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 91 deletions(-) create mode 100644 docs/development/release-process.rst diff --git a/docs/development.rst b/docs/development.rst index 2d0dc3257..df2d0ec9b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -149,97 +149,6 @@ typically used, but can be used for things like changing version schemes, updating deprecation policy, etc. -Release Cadence -=============== - -The pip project has a release cadence of releasing whatever is on ``master`` -every 3 months. This gives users a predictable pattern for when releases -are going to happen and prevents locking up improvements for fixes for long -periods of time, while still preventing massively fracturing the user base -with version numbers. - -Our release months are January, April, July, October. The release date within -that month will be up to the release manager for that release. If there are -no changes, then that release month is skipped and the next release will be -3 month later. - -The release manager may, at their discretion, choose whether or not there -will be a pre-release period for a release, and if there is may extend that -period into the next month if needed. - -Because releases are made direct from the ``master`` branch, it is essential -that ``master`` is always in a releasable state. It is acceptable to merge -PRs that partially implement a new feature, but only if the partially -implemented version is usable in that state (for example, with reduced -functionality or disabled by default). In the case where a merged PR is found -to need extra work before being released, the release manager always has the -option to back out the partial change prior to a release. The PR can then be -reworked and resubmitted for the next release. - - -Deprecation Policy -================== - -Any change to pip that removes or significantly alters user-visible behaviour -that is described in the pip documentation will be deprecated for a minimum of -one released version before the change occurs. Deprecation will take the form of -a warning being issued by pip when the feature is used. Longer deprecation -periods, or deprecation warnings for behaviour changes that would not normally -be covered by this policy, are also possible depending on circumstances, but -this is at the discretion of the pip developers. - -Note that the documentation is the sole reference for what counts as agreed -behaviour. If something isn't explicitly mentioned in the documentation, it can -be changed without warning, or any deprecation period, in a pip release. -However, we are aware that the documentation isn't always complete - PRs that -document existing behaviour with the intention of covering that behaviour with -the above deprecation process are always acceptable, and will be considered on -their merits. - - -Release Process -=============== - -#. On the current pip ``master`` branch, generate a new ``AUTHORS.txt`` by - running ``invoke generate.authors`` and commit the results. -#. On the current pip ``master`` branch, make a new commit which bumps the - version in ``pip/__init__.py`` to the release version and adjust the - ``CHANGES.txt`` file to reflect the current date. The release version should - follow a YY.N scheme, where YY is the two digit year, and N is the Nth - release within that year. -#. On the current pip ``master`` branch, generate a new ``NEWS.rst`` by running - ``invoke generate.news`` and commit the results. -#. Create a signed tag of the ``master`` branch of the form ``X.Y.Z`` using the - command ``git tag -s X.Y.Z``. -#. Checkout the tag using ``git checkout X.Y.Z`` and create the distribution - files using ``python setup.py sdist bdist_wheel``. -#. Upload the distribution files to PyPI using twine - (``twine upload -s dist/*``). The upload should include GPG signatures of - the distribution files. -#. Push all of the changes. -#. Regenerate the ``get-pip.py`` script by running - ``invoke generate.installer`` in the get-pip repository, and committing the - results. - - -Creating a Bugfix Release -========================= - -Sometimes we need to release a bugfix release of the form ``X.Y.Z+1``. In order -to create one of these the changes should already be merged into the -``master`` branch. - -#. Create a new ``release/YY.N.Z+1`` branch off of the ``YY.N`` tag using the - command ``git checkout -b release/YY.N.Z+1 YY.N``. -#. Cherry pick the fixed commits off of the ``master`` branch, fixing any - conflicts and moving any changelog entries from the development version's - changelog section to the ``YY.N.Z+1`` section. -#. Push the ``release/YY.N.Z+1`` branch to github and submit a PR for it against - the ``master`` branch and wait for the tests to run. -#. Once tests run, merge the ``release/YY.N.Z+1`` branch into master, and follow - the above release process starting with step 4. - - .. _`mailing list`: https://mail.python.org/mailman/listinfo/distutils-sig .. _`towncrier`: https://pypi.org/project/towncrier/ .. _`Travis CI`: https://travis-ci.org/ diff --git a/docs/development/release-process.rst b/docs/development/release-process.rst new file mode 100644 index 000000000..e17ef0e43 --- /dev/null +++ b/docs/development/release-process.rst @@ -0,0 +1,96 @@ +===================== +pip's release process +===================== + + +Release Cadence +=============== + +The pip project has a release cadence of releasing whatever is on ``master`` +every 3 months. This gives users a predictable pattern for when releases +are going to happen and prevents locking up improvements for fixes for long +periods of time, while still preventing massively fracturing the user base +with version numbers. + +Our release months are January, April, July, October. The release date within +that month will be up to the release manager for that release. If there are +no changes, then that release month is skipped and the next release will be +3 month later. + +The release manager may, at their discretion, choose whether or not there +will be a pre-release period for a release, and if there is may extend that +period into the next month if needed. + +Because releases are made direct from the ``master`` branch, it is essential +that ``master`` is always in a releasable state. It is acceptable to merge +PRs that partially implement a new feature, but only if the partially +implemented version is usable in that state (for example, with reduced +functionality or disabled by default). In the case where a merged PR is found +to need extra work before being released, the release manager always has the +option to back out the partial change prior to a release. The PR can then be +reworked and resubmitted for the next release. + + +Deprecation Policy +================== + +Any change to pip that removes or significantly alters user-visible behaviour +that is described in the pip documentation will be deprecated for a minimum of +one released version before the change occurs. Deprecation will take the form of +a warning being issued by pip when the feature is used. Longer deprecation +periods, or deprecation warnings for behaviour changes that would not normally +be covered by this policy, are also possible depending on circumstances, but +this is at the discretion of the pip developers. + +Note that the documentation is the sole reference for what counts as agreed +behaviour. If something isn't explicitly mentioned in the documentation, it can +be changed without warning, or any deprecation period, in a pip release. +However, we are aware that the documentation isn't always complete - PRs that +document existing behaviour with the intention of covering that behaviour with +the above deprecation process are always acceptable, and will be considered on +their merits. + + +Release Ops +=========== + +Doing a new release +------------------- + +#. On the current pip ``master`` branch, generate a new ``AUTHORS.txt`` by + running ``invoke generate.authors`` and commit the results. +#. On the current pip ``master`` branch, make a new commit which bumps the + version in ``pip/__init__.py`` to the release version and adjust the + ``CHANGES.txt`` file to reflect the current date. The release version should + follow a YY.N scheme, where YY is the two digit year, and N is the Nth + release within that year. +#. On the current pip ``master`` branch, generate a new ``NEWS.rst`` by running + ``invoke generate.news`` and commit the results. +#. Create a signed tag of the ``master`` branch of the form ``X.Y.Z`` using the + command ``git tag -s X.Y.Z``. +#. Checkout the tag using ``git checkout X.Y.Z`` and create the distribution + files using ``python setup.py sdist bdist_wheel``. +#. Upload the distribution files to PyPI using twine + (``twine upload -s dist/*``). The upload should include GPG signatures of + the distribution files. +#. Push all of the changes. +#. Regenerate the ``get-pip.py`` script by running + ``invoke generate.installer`` in the get-pip repository, and committing the + results. + +Doing a bug-fix release +----------------------- + +Sometimes we need to release a bugfix release of the form ``X.Y.Z+1``. In order +to create one of these the changes should already be merged into the +``master`` branch. + +#. Create a new ``release/YY.N.Z+1`` branch off of the ``YY.N`` tag using the + command ``git checkout -b release/YY.N.Z+1 YY.N``. +#. Cherry pick the fixed commits off of the ``master`` branch, fixing any + conflicts and moving any changelog entries from the development version's + changelog section to the ``YY.N.Z+1`` section. +#. Push the ``release/YY.N.Z+1`` branch to github and submit a PR for it against + the ``master`` branch and wait for the tests to run. +#. Once tests run, merge the ``release/YY.N.Z+1`` branch into master, and follow + the above release process starting with step 4. From 8891656e40e899d3fea7e66caee367436db380e2 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 15:19:17 +0530 Subject: [PATCH 044/129] Move and expand contribution related notes --- docs/development.rst | 105 ------------------------------ docs/development/contributing.rst | 103 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 105 deletions(-) create mode 100644 docs/development/contributing.rst diff --git a/docs/development.rst b/docs/development.rst index df2d0ec9b..ca7099bf8 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -48,112 +48,7 @@ within lines. Such changes can be made separately, as a "formatting cleanup" PR, if needed. -Automated Testing -================= - -All pull requests and merges to 'master' branch are tested using `Travis CI`_ -and `Appveyor CI`_ based on our `.travis.yml`_ and `appveyor.yml`_ files. - -You can find the status and results to the CI runs for your PR on GitHub's Web -UI for the pull request. You can also find links to the CI services' pages for -the specific builds in the form of "Details" links, in case the CI run fails -and you wish to view the output. - -To trigger CI to run again for a pull request, you can close and open the pull -request or submit another change to the pull request. If needed, project -maintainers can manually trigger a restart of a job/build. - -Running tests -============= - -OS Requirements: subversion, bazaar, git, and mercurial. - -Python Requirements: tox or install all packages listed in -`tools/test-requirements.txt`_ - -Ways to run the tests locally:: - - $ tox -e py36 # The preferred way to run the tests, can use pyNN to - # run for a particular version or leave off the -e to - # run for all versions. - $ python setup.py test # Using the setuptools test plugin - $ py.test # Using py.test directly - $ tox # Using tox against pip's tox.ini - -If you are missing one of the VCS tools, you can tell ``py.test`` to skip it:: - - # When using tox - $ tox -e py36 -- -k 'not svn' - $ tox -e py36 -- -k 'not (svn or git)' - # Directly with py.test - $ py.test -k 'not svn' - $ py.test -k 'not (svn or git)' - - -Adding a NEWS Entry -=================== - -The ``NEWS.rst`` file is managed using `towncrier`_ and all non trivial changes -must be accompanied by a news entry. - -To add an entry to the news file, first you need to have created an issue -describing the change you want to make. A Pull Request itself *may* function as -such, but it is preferred to have a dedicated issue (for example, in case the -PR ends up rejected due to code quality reasons). - -Once you have an issue or pull request, you take the number and you create a -file inside of the ``news/`` directory named after that issue number with an -extension of ``removal``, ``feature``, ``bugfix``, or ``doc``. Thus if your -issue or PR number is ``1234`` and this change is fixing a bug, then you would -create a file ``news/1234.bugfix``. PRs can span multiple categories by creating -multiple files (for instance, if you added a feature and deprecated/removed the -old feature at the same time, you would create ``news/NNNN.feature`` and -``news/NNNN.removal``). Likewise if a PR touches multiple issues/PRs you may -create a file for each of them with the exact same contents and Towncrier will -deduplicate them. - -The contents of this file are reStructuredText formatted text that will be used -as the content of the news file entry. You do not need to reference the issue -or PR numbers here as towncrier will automatically add a reference to all of -the affected issues when rendering the news file. - -In order to maintain a consistent style in the ``NEWS.rst`` file, it is -preferred to keep the news entry to the point, in sentence case, shorter than -80 characters and in an imperative tone -- an entry should complete the sentence -"This change will ...". In rare cases, where one line is not enough, use a -summary line in an imperative tone followed by a blank line separating it -from a description of the feature/change in one or more paragraphs, each wrapped -at 80 characters. Remember that a news entry is meant for end users and should -only contain details relevant to an end user. - -A trivial change is anything that does not warrant an entry in the news file. -Some examples are: Code refactors that don't change anything as far as the -public is concerned, typo fixes, white space modification, etc. To mark a PR -as trivial a contributor simply needs to add a randomly named, empty file to -the ``news/`` directory with the extension of ``.trivial``. If you are on a -POSIX like operating system, one can be added by running -``touch news/$(uuidgen).trivial``. On Windows, the same result can be achieved -in Powershell using ``New-Item "news/$([guid]::NewGuid()).trivial"``. Core -committers may also add a "trivial" label to the PR which will accomplish the -same thing. - -Upgrading, removing, or adding a new vendored library gets a special mention -using a ``news/.vendor`` file. This is in addition to any features, -bugfixes, or other kinds of news that pulling in this library may have. This -uses the library name as the key so that updating the same library twice doesn't -produce two news file entries. - -Changes to the processes, policies, or other non code related changed that are -otherwise notable can be done using a ``news/.process`` file. This is not -typically used, but can be used for things like changing version schemes, -updating deprecation policy, etc. - - .. _`mailing list`: https://mail.python.org/mailman/listinfo/distutils-sig -.. _`towncrier`: https://pypi.org/project/towncrier/ -.. _`Travis CI`: https://travis-ci.org/ -.. _`Appveyor CI`: https://www.appveyor.com/ -.. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml .. _`appveyor.yml`: https://github.com/pypa/pip/blob/master/appveyor.yml .. _`Travis CI Pull Requests`: https://travis-ci.org/pypa/pip/pull_requests .. _`tools/test-requirements.txt`: https://github.com/pypa/pip/blob/master/tools/test-requirements.txt diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst new file mode 100644 index 000000000..627580295 --- /dev/null +++ b/docs/development/contributing.rst @@ -0,0 +1,103 @@ +=================== +Contributing to pip +=================== + +.. todo + Create a "guide" to pip's internals and link to it from here saying + "you might want to take a look at the guide" + + +Automated Testing +================= + +All pull requests and merges to 'master' branch are tested using `Travis CI`_ +and `Appveyor CI`_ based on our `.travis.yml`_ and `appveyor.yml`_ files. + +You can find the status and results to the CI runs for your PR on GitHub's Web +UI for the pull request. You can also find links to the CI services' pages for +the specific builds in the form of "Details" links, in case the CI run fails +and you wish to view the output. + +To trigger CI to run again for a pull request, you can close and open the pull +request or submit another change to the pull request. If needed, project +maintainers can manually trigger a restart of a job/build. + + +NEWS Entries +============ + +The ``NEWS.rst`` file is managed using `towncrier`_ and all non trivial changes +must be accompanied by a news entry. + +To add an entry to the news file, first you need to have created an issue +describing the change you want to make. A Pull Request itself *may* function as +such, but it is preferred to have a dedicated issue (for example, in case the +PR ends up rejected due to code quality reasons). + +Once you have an issue or pull request, you take the number and you create a +file inside of the ``news/`` directory named after that issue number with an +extension of ``removal``, ``feature``, ``bugfix``, or ``doc``. Thus if your +issue or PR number is ``1234`` and this change is fixing a bug, then you would +create a file ``news/1234.bugfix``. PRs can span multiple categories by creating +multiple files (for instance, if you added a feature and deprecated/removed the +old feature at the same time, you would create ``news/NNNN.feature`` and +``news/NNNN.removal``). Likewise if a PR touches multiple issues/PRs you may +create a file for each of them with the exact same contents and Towncrier will +deduplicate them. + +Contents of a NEWS entry +------------------------ + +The contents of this file are reStructuredText formatted text that will be used +as the content of the news file entry. You do not need to reference the issue +or PR numbers here as towncrier will automatically add a reference to all of +the affected issues when rendering the news file. + +In order to maintain a consistent style in the ``NEWS.rst`` file, it is +preferred to keep the news entry to the point, in sentence case, shorter than +80 characters and in an imperative tone -- an entry should complete the sentence +"This change will ...". In rare cases, where one line is not enough, use a +summary line in an imperative tone followed by a blank line separating it +from a description of the feature/change in one or more paragraphs, each wrapped +at 80 characters. Remember that a news entry is meant for end users and should +only contain details relevant to an end user. + +Choosing the type of NEWS entry +------------------------------- + +A trivial change is anything that does not warrant an entry in the news file. +Some examples are: Code refactors that don't change anything as far as the +public is concerned, typo fixes, white space modification, etc. To mark a PR +as trivial a contributor simply needs to add a randomly named, empty file to +the ``news/`` directory with the extension of ``.trivial``. If you are on a +POSIX like operating system, one can be added by running +``touch news/$(uuidgen).trivial``. On Windows, the same result can be achieved +in Powershell using ``New-Item "news/$([guid]::NewGuid()).trivial"``. Core +committers may also add a "trivial" label to the PR which will accomplish the +same thing. + +Upgrading, removing, or adding a new vendored library gets a special mention +using a ``news/.vendor`` file. This is in addition to any features, +bugfixes, or other kinds of news that pulling in this library may have. This +uses the library name as the key so that updating the same library twice doesn't +produce two news file entries. + +Changes to the processes, policies, or other non code related changed that are +otherwise notable can be done using a ``news/.process`` file. This is not +typically used, but can be used for things like changing version schemes, +updating deprecation policy, etc. + + +Becoming a maintainer +===================== + +If you want to become an official maintainer, start by helping out. + +Later, when you think you're ready, get in touch with one of the maintainers +and they will initiate a vote. + +.. _`Travis CI`: https://travis-ci.org/ +.. _`Appveyor CI`: https://www.appveyor.com/ +.. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml +.. _`appveyor.yml`: https://github.com/pypa/pip/blob/master/appveyor.yml +.. _`towncrier`: https://pypi.org/project/towncrier/ From c1780e03813f6ccce808fc9f0ba4e411400ab7d1 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 15:20:42 +0530 Subject: [PATCH 045/129] Add a new section documenting development processes --- docs/development/getting-started.rst | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 docs/development/getting-started.rst diff --git a/docs/development/getting-started.rst b/docs/development/getting-started.rst new file mode 100644 index 000000000..da68aae58 --- /dev/null +++ b/docs/development/getting-started.rst @@ -0,0 +1,96 @@ +=============== +Getting Started +=============== + +We’re pleased that you are interested in working on pip. + +This document is meant to get you setup to work on pip and to act as a guide and +reference to the the development setup. If you face any issues during this +process, please `open an issue`_ about it on the issue tracker. + +Development tools +================= + +pip uses :pypi:`tox` for testing against multiple different Python environments +and ensuring reproducible environments for linting and building documentation. + +For developing pip, you need to install ``tox`` on your system. Often, you can +just do ``python -m pip install tox`` to install and use it. + +Running Tests +------------- + +pip uses the :pypi:`pytest` test framework, :pypi:`mock` and :pypi:`pretend` +for testing. These are automatically installed by tox for running the tests. + +To run tests locally, run: + +.. code-block:: console + + $ tox -e py36 + +The example above runs tests against Python 3.6. You can also use other +versions like ``py27`` and ``pypy3``. + +pip's test suite requires supported version control tools (subversion, bazaar, +git, and mercurial) to be installed. If you are missing one of the VCS tools, +you can tell pip to skip those tests: + +.. code-block:: console + + $ tox -e py36 -- -k "not svn" + $ tox -e py36 -- -k "not (svn or git)" + +Running Linters +--------------- + +pip uses :pypi:`flake8` and :pypi:`isort` for linting the codebase. These +ensure that the codebase is in compliance with :pep:`8` and the imports are +consistently ordered and styled. + +To use linters locally, run: + +.. code-block:: console + + $ tox -e lint-py2 + $ tox -e lint-py3 + +The above commands run the linters on Python 2 followed by Python 3. + +.. note:: + + Do not silence errors from flake8 with ``# noqa`` comments or otherwise. + The only exception to this is silencing unused-import errors for imports + related to static type checking as currently `flake8 does not understand + PEP 484 type-comments`_. + +Running mypy +------------ + +pip uses :pypi:`mypy` to run static type analysis, which helps catch certain +kinds of bugs. The codebase uses `PEP 484 type-comments`_ due to compatibility +requirements with Python 2.7. + +To run the ``mypy`` type checker, run: + +.. code-block:: console + + $ tox -e mypy + +Building Documentation +---------------------- + +pip's documentation is built using :pypi:`Sphinx`. The documentation is written +in reStructuredText. + +To build it locally, run: + +.. code-block:: console + + $ tox -e docs + +The built documentation can be found in the ``docs/build`` folder. + +.. _`open an issue`: https://github.com/pypa/pip/issues/new?title=Truouble+with+pip+development+environment +.. _`flake8 does not understand PEP 484 type-comments`: https://gitlab.com/pycqa/flake8/issues/118 +.. _`PEP 484 type-comments`: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code From b6cdf5dbf02e1a7f35c9a29a4e96a9ab6c352d13 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 15:41:57 +0530 Subject: [PATCH 046/129] Document how to submit pull requests --- docs/development.rst | 54 ------------------------------- docs/development/contributing.rst | 27 ++++++++++++++++ 2 files changed, 27 insertions(+), 54 deletions(-) delete mode 100644 docs/development.rst diff --git a/docs/development.rst b/docs/development.rst deleted file mode 100644 index ca7099bf8..000000000 --- a/docs/development.rst +++ /dev/null @@ -1,54 +0,0 @@ -=========== -Development -=========== - -Pull Requests -============= - -- Submit Pull Requests against the `master` branch. -- Provide a good description of what you're doing and why. -- Provide tests that cover your changes and try to run the tests locally first. - -**Example**. Assuming you set up GitHub account, forked pip repository from -https://github.com/pypa/pip to your own page via web interface, and your -fork is located at https://github.com/yourname/pip - -:: - - $ git clone git@github.com:pypa/pip.git - $ cd pip - # ... - $ git diff - $ git add ... - $ git status - $ git commit - -You may reference relevant issues in commit messages (like #1259) to -make GitHub link issues and commits together, and with phrase like -"fixes #1259" you can even close relevant issues automatically. Now -push the changes to your fork:: - - $ git push git@github.com:yourname/pip.git - -Open Pull Requests page at https://github.com/yourname/pip/pulls and -click "New pull request" and select your fork. That's it. - -Pull requests should be self-contained, and limited in scope. Before being -merged, a pull request must be reviewed, and keeping individual PRs limited -in scope makes this far easier. In particular, pull requests must not be -treated as "feature branches", with ongoing development work happening -within the PR. Instead, the feature should be broken up into smaller, -independent parts which can be reviewed and merged individually. - -When creating a pull request, avoid including "cosmetic" changes to -code that is unrelated to your change, as these make reviewing the PR -more difficult. Examples include re-flowing text in comments or -documentation, or addition or removal of blank lines or whitespace -within lines. Such changes can be made separately, as a "formatting -cleanup" PR, if needed. - - -.. _`mailing list`: https://mail.python.org/mailman/listinfo/distutils-sig -.. _`appveyor.yml`: https://github.com/pypa/pip/blob/master/appveyor.yml -.. _`Travis CI Pull Requests`: https://travis-ci.org/pypa/pip/pull_requests -.. _`tools/test-requirements.txt`: https://github.com/pypa/pip/blob/master/tools/test-requirements.txt diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index 627580295..ce2357afb 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -7,6 +7,31 @@ Contributing to pip "you might want to take a look at the guide" +Submitting Pull Requests +======================== + +Submit pull requests against the ``master`` branch, providing a good +description of what you're doing and why. Provide tests that cover your changes +and run the tests locally first. + +Pull Requests should be small to facilitate easier review. Keep them +self-contained, and limited in scope. `Studies have shown`_ that review quality +falls off as patch size grows. Sometimes this will result in many small PRs to +land a single large feature. In particular, pull requests must not be treated +as "feature branches", with ongoing development work happening within the PR. +Instead, the feature should be broken up into smaller, independent parts which +can be reviewed and merged individually. + +When creating a pull request, avoid including "cosmetic" changes to code that +is unrelated to your change, as these make reviewing the PR more difficult. +Examples include re-flowing text in comments or documentation, or addition or +removal of blank lines or whitespace within lines. Such changes can be made +separately, as a "formatting cleanup" PR, if needed. + +You must have legal permission to distribute any code you contribute to +pip and it must be available under the MIT License. + + Automated Testing ================= @@ -96,6 +121,8 @@ If you want to become an official maintainer, start by helping out. Later, when you think you're ready, get in touch with one of the maintainers and they will initiate a vote. +.. _`Studies have shown`: https://smartbear.com/smartbear/media/pdfs/wp-cc-11-best-practices-of-peer-code-review.pdf +.. _`resolve merge conflicts`: https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/ .. _`Travis CI`: https://travis-ci.org/ .. _`Appveyor CI`: https://www.appveyor.com/ .. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml From 8344642207b0ae99b395fcd2badaa483deec4f9a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 15:47:25 +0530 Subject: [PATCH 047/129] Document how to rebase a branch --- docs/development/contributing.rst | 106 ++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index ce2357afb..c509b6b22 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -113,6 +113,112 @@ typically used, but can be used for things like changing version schemes, updating deprecation policy, etc. +Updating your branch +==================== + +As you work, you might need to update your local master branch up-to-date with +the ``master`` branch in the main pip repository, which moves forward as the +maintainers merge pull requests. Most people working on the project use the +following workflow. + +This assumes that you have Git configured so that when you run the following +command: + +.. code-block:: console + + git remote -v + +Your output looks like this: + +.. code-block:: console + + origin https://github.com/USERNAME/pip.git (fetch) + origin https://github.com/USERNAME/pip.git (push) + upstream https://github.com/pypa/pip.git (fetch) + upstream https://github.com/pypa/pip.git (push) + +In the example above, ``USERNAME`` is your username on GitHub. + +First, fetch the latest changes from the main pip repository, ``upstream``: + +.. code-block:: console + + git fetch upstream + +Then, check out your local ``master`` branch, and rebase the changes on top of +it: + +.. code-block:: console + + git checkout master + git rebase upstream/master + +At this point, you might have to `resolve merge conflicts`_. Once this is done, +push the updates you have just made to your local ``master`` branch to your +``origin`` repository on GitHub: + +.. code-block:: console + + git checkout master + git push origin master + +Now your local ``master`` branch and the ``master`` branch in your ``origin`` +repo have been updated with the most recent changes from the main pip +repository. + +To keep your branches updated, the process is similar: + +.. code-block:: console + + git checkout awesome-feature + git fetch upstream + git rebase upstream/master + +Now your branch has been updated with the latest changes from the +``master`` branch on the upstream pip repository. + +It's good practice to back up your branches by pushing them to your +``origin`` on GitHub as you are working on them. To push a branch, +run this command: + +.. code-block:: console + + git push origin awesome-feature + +In this example, ```` is the name of your branch. This +will push the branch you are working on to GitHub, but will not +create a PR. + +Once you have pushed your branch to your ``origin``, if you need to +update it again, you will have to force push your changes by running the +following command: + +.. code-block:: console + + git push -f origin awesome-feature + +The ``-f`` (or ``--force``) flag after ``push`` forces updates from your local +branch to update your ``origin`` branch. If you have a PR open on your +branch, force pushing will update your PR. (This is a useful command +when someone requests changes on a PR.) + +If you get an error message like this: + +.. code-block:: console + + ! [rejected] awesome-feature -> awesome-feature (non-fast-forward) + error: failed to push some refs to 'https://github.com/USERNAME/pip.git' + hint: Updates were rejected because the tip of your current branch is behind + hint: its remote counterpart. Integrate the remote changes (e.g. + hint: 'git pull ...') before pushing again. + hint: See the 'Note about fast-forwards' in 'git push --help' for details. + +Try force-pushing your branch with ``push -f``. + +The ``master`` branch in the main pip repository gets updated frequently, so +you might have to update your branch at least once while you are working on it. + + Becoming a maintainer ===================== From ff1cdc7c6e9b6eac8915cf5deb991ee0982af8b3 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 16:16:49 +0530 Subject: [PATCH 048/129] Add shorthand for pypi projects --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index e615e6be8..d01a59e3b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -127,6 +127,7 @@ pygments_style = 'sphinx' extlinks = { 'issue': ('https://github.com/pypa/pip/issues/%s', '#'), 'pull': ('https://github.com/pypa/pip/pull/%s', 'PR #'), + 'pypi': ('https://pypi.org/project/%s', ''), } # -- Options for HTML output -------------------------------------------------- From 2e9e42bc33cfd6f1a131434d65aad5c34ec81aa6 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 16:17:00 +0530 Subject: [PATCH 049/129] Use newer development docs --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index caf0725a6..64bbb79f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,5 +18,5 @@ for installing Python packages. installing user_guide reference/index - development + development/index news From 868dd687bd67abaa3e85a8596eac1547055f0f91 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 23 Jun 2018 21:31:38 +0530 Subject: [PATCH 050/129] Polish the Release Process documentation --- docs/development/release-process.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/development/release-process.rst b/docs/development/release-process.rst index e17ef0e43..041943a12 100644 --- a/docs/development/release-process.rst +++ b/docs/development/release-process.rst @@ -51,11 +51,11 @@ the above deprecation process are always acceptable, and will be considered on their merits. -Release Ops -=========== +Release Process +=============== -Doing a new release -------------------- +Creating a new release +---------------------- #. On the current pip ``master`` branch, generate a new ``AUTHORS.txt`` by running ``invoke generate.authors`` and commit the results. @@ -66,9 +66,9 @@ Doing a new release release within that year. #. On the current pip ``master`` branch, generate a new ``NEWS.rst`` by running ``invoke generate.news`` and commit the results. -#. Create a signed tag of the ``master`` branch of the form ``X.Y.Z`` using the - command ``git tag -s X.Y.Z``. -#. Checkout the tag using ``git checkout X.Y.Z`` and create the distribution +#. Create a signed tag of the ``master`` branch of the form ``YY.N`` using the + command ``git tag -s YY.N``. +#. Checkout the tag using ``git checkout YY.N`` and create the distribution files using ``python setup.py sdist bdist_wheel``. #. Upload the distribution files to PyPI using twine (``twine upload -s dist/*``). The upload should include GPG signatures of @@ -78,11 +78,11 @@ Doing a new release ``invoke generate.installer`` in the get-pip repository, and committing the results. -Doing a bug-fix release ------------------------ +Creating a bug-fix release +-------------------------- -Sometimes we need to release a bugfix release of the form ``X.Y.Z+1``. In order -to create one of these the changes should already be merged into the +Sometimes we need to release a bugfix release of the form ``YY.N.Z+1``. In +order to create one of these the changes should already be merged into the ``master`` branch. #. Create a new ``release/YY.N.Z+1`` branch off of the ``YY.N`` tag using the From 0db7d17ea5c689d199ca693ba393b0b3aab27078 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 2 Jul 2018 19:37:52 +0530 Subject: [PATCH 051/129] Don't mention various ways to contribute --- docs/development/index.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/development/index.rst b/docs/development/index.rst index f7dc0fb5d..219e340b8 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -5,10 +5,6 @@ pip, as an open-source project, welcomes contributions of all forms. The sections below will help you get started with development, testing, and documentation. -Please contribute issues, submit bug reports, file feature requests, and help -triage the issues on our issue tracker on `GitHub`_. If submitting a bug report -for the first time, check out `"What to put in your bug report"`_ for guidance. - You can also join ``#pypa`` (general packaging discussion and user support) and ``#pypa-dev`` (discussion about development of packaging tools) `on Freenode`_, or the `pypa-dev mailing list`_, to ask questions or get involved. @@ -20,7 +16,5 @@ or the `pypa-dev mailing list`_, to ask questions or get involved. contributing release-process -.. _`GitHub`: https://github.com/pypa/pip -.. _`"What to put in your bug report"`: http://www.contribution-guide.org/#what-to-put-in-your-bug-report .. _`on Freenode`: https://webchat.freenode.net/?channels=%23pypa-dev,pypa .. _`pypa-dev mailing list`: https://groups.google.com/forum/#!forum/pypa-dev From 2b57b8dc36207443f02e3d35c86c1c6e1ae1a849 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 2 Jul 2018 19:38:14 +0530 Subject: [PATCH 052/129] Mention that pip is volunteer maintained --- docs/development/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/development/index.rst b/docs/development/index.rst index 219e340b8..0118cb38b 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -1,9 +1,9 @@ Development =========== -pip, as an open-source project, welcomes contributions of all forms. The -sections below will help you get started with development, testing, and -documentation. +pip is a volunteer maintained open source project and we welcome contributions +of all forms. The sections below will help you get started with development, +testing, and documentation. You can also join ``#pypa`` (general packaging discussion and user support) and ``#pypa-dev`` (discussion about development of packaging tools) `on Freenode`_, From 60ef05b01382af2e3c180183d86145b8aab46c4f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 2 Jul 2018 19:38:36 +0530 Subject: [PATCH 053/129] Provide examples of running specific tests using tox --- docs/development/getting-started.rst | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/development/getting-started.rst b/docs/development/getting-started.rst index da68aae58..6a91b605d 100644 --- a/docs/development/getting-started.rst +++ b/docs/development/getting-started.rst @@ -32,9 +32,22 @@ To run tests locally, run: The example above runs tests against Python 3.6. You can also use other versions like ``py27`` and ``pypy3``. -pip's test suite requires supported version control tools (subversion, bazaar, -git, and mercurial) to be installed. If you are missing one of the VCS tools, -you can tell pip to skip those tests: +``tox`` has been configured to any additional arguments it is given to +``pytest``. This enables the use of pytest's `rich CLI`_. As an example, you +can select tests using the various ways that pytest provides: + +.. code-block:: console + + $ # Using file name + $ tox -e py36 -- tests/functional/test_install.py + $ # Using markers + $ tox -e py36 -- -m unit + $ # Using keywords + $ tox -e py36 -- -k "install and not wheel" + +Running pip's test suite requires supported version control tools (subversion, +bazaar, git, and mercurial) to be installed. If you are missing one of the VCS +tools, you can tell pip to skip those tests: .. code-block:: console @@ -91,6 +104,7 @@ To build it locally, run: The built documentation can be found in the ``docs/build`` folder. -.. _`open an issue`: https://github.com/pypa/pip/issues/new?title=Truouble+with+pip+development+environment +.. _`open an issue`: https://github.com/pypa/pip/issues/new?title=Trouble+with+pip+development+environment .. _`flake8 does not understand PEP 484 type-comments`: https://gitlab.com/pycqa/flake8/issues/118 .. _`PEP 484 type-comments`: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code +.. _`rich CLI`: https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests From 98ddd2dddc7d858f649858d88115eea8827a615a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 2 Jul 2018 19:58:13 +0530 Subject: [PATCH 054/129] Improve "Submitting Pull Requests" section - Link to compatibility requirements and require compliance - Move licensing details to come earlier - Reword text on cosmetic changes --- docs/development/contributing.rst | 16 +++++++++------- docs/installing.rst | 12 +++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index c509b6b22..821323755 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -11,8 +11,14 @@ Submitting Pull Requests ======================== Submit pull requests against the ``master`` branch, providing a good -description of what you're doing and why. Provide tests that cover your changes -and run the tests locally first. +description of what you're doing and why. You must have legal permission to +distribute any code you contribute to pip and it must be available under the +MIT License. + +Provide tests that cover your changes and run the tests locally first. pip +:ref:`supports ` multiple Python versions and +operating systems. Any pull request must consider and work on all these +platforms. Pull Requests should be small to facilitate easier review. Keep them self-contained, and limited in scope. `Studies have shown`_ that review quality @@ -22,16 +28,12 @@ as "feature branches", with ongoing development work happening within the PR. Instead, the feature should be broken up into smaller, independent parts which can be reviewed and merged individually. -When creating a pull request, avoid including "cosmetic" changes to code that +Additionally, avoid including "cosmetic" changes to code that is unrelated to your change, as these make reviewing the PR more difficult. Examples include re-flowing text in comments or documentation, or addition or removal of blank lines or whitespace within lines. Such changes can be made separately, as a "formatting cleanup" PR, if needed. -You must have legal permission to distribute any code you contribute to -pip and it must be available under the MIT License. - - Automated Testing ================= diff --git a/docs/installing.rst b/docs/installing.rst index 916c3a3ba..2daa2c108 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -6,10 +6,10 @@ Installation Do I need to install pip? ------------------------- -pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4 -downloaded from `python.org `_ or if you are working -in a :ref:`Virtual Environment ` -created by :ref:`pypug:virtualenv` or :ref:`pyvenv `. +pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4 +downloaded from `python.org `_ or if you are working +in a :ref:`Virtual Environment ` +created by :ref:`pypug:virtualenv` or :ref:`pyvenv `. Just make sure to :ref:`upgrade pip `. @@ -34,7 +34,7 @@ Inspect ``get-pip.py`` for any malevolence. Then run the following:: system or another package manager. ``get-pip.py`` does not coordinate with those tools, and may leave your system in an inconsistent state. -``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel` +``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel` if they are not already. :ref:`pypug:setuptools` is required to install :term:`source distributions `. Both are required in order to build a :ref:`Wheel cache` (which improves installation @@ -104,6 +104,8 @@ On Windows [4]_:: python -m pip install -U pip +.. _compatibility-requirements: + Python and OS Compatibility --------------------------- From 19621988618aa8d961ff2ff58a031acfbb6fa736 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 3 Jul 2018 00:50:10 +0530 Subject: [PATCH 055/129] Remove hack for pyyaml support on 3.7 --- tools/tests-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/tests-requirements.txt b/tools/tests-requirements.txt index a49698b92..8c9bd7592 100644 --- a/tools/tests-requirements.txt +++ b/tools/tests-requirements.txt @@ -6,8 +6,6 @@ pytest-cov pytest-rerunfailures pytest-timeout pytest-xdist +pyyaml scripttest https://github.com/pypa/virtualenv/archive/master.zip#egg=virtualenv -# Revert the following hack after a PyYAML is released supporting Py3.7 -pyyaml; python_version < "3.7" -https://github.com/yaml/pyyaml/archive/master.zip#egg=pyyaml ; python_version >= "3.7" From 4555cd356390f36e40040d5cd7a640520e7131e6 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Tue, 3 Jul 2018 01:09:41 -0700 Subject: [PATCH 056/129] Remove the checkout variable from VersionControl.check_destination(). --- ...B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial | 0 src/pip/_internal/vcs/__init__.py | 37 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 news/82B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial diff --git a/news/82B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial b/news/82B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index fd45fac93..5f32172f0 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -348,7 +348,22 @@ class VersionControl(object): ) response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) - checkout = False + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + return True + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + return True + if response == 's': logger.info( 'Switching %s %s to %s%s', @@ -358,24 +373,10 @@ class VersionControl(object): rev_display, ) self.switch(dest, url, rev_options) - elif response == 'i': - # do nothing - pass - elif response == 'w': - logger.warning('Deleting %s', display_path(dest)) - rmtree(dest) - checkout = True - elif response == 'b': - dest_dir = backup_dir(dest) - logger.warning( - 'Backing up %s to %s', display_path(dest), dest_dir, - ) - shutil.move(dest, dest_dir) - checkout = True - elif response == 'a': - sys.exit(-1) - return checkout + # Do nothing if the response is "i". + + return False def unpack(self, location): """ From 92cc86b5c9617710151151b61865484ae85bdf34 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 5 Jul 2018 11:27:11 +0530 Subject: [PATCH 057/129] Satisfy the linter --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index d15c09f80..2f1d39318 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ import sys from setuptools import find_packages, setup - here = os.path.abspath(os.path.dirname(__file__)) From fde9eb66f072adad73491b00beafddd656f2d9ae Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 5 Jul 2018 18:01:11 -0400 Subject: [PATCH 058/129] Adjust the deprecation policy to be time based --- docs/development.rst | 16 ++++++++-------- news/deprecation.process | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index caac58048..0467bbf8b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -195,19 +195,19 @@ reworked and resubmitted for the next release. Deprecation Policy ================== -Any change to pip that removes or significantly alters user-visible behaviour +Any change to pip that removes or significantly alters user-visible behavior that is described in the pip documentation will be deprecated for a minimum of -one released version before the change occurs. Deprecation will take the form of -a warning being issued by pip when the feature is used. Longer deprecation -periods, or deprecation warnings for behaviour changes that would not normally -be covered by this policy, are also possible depending on circumstances, but -this is at the discretion of the pip developers. +6 months before the change occurs. Deprecation will take the form of a warning +being issued by pip when the feature is used. Longer deprecation periods, or +deprecation warnings for behavior changes that would not normally be covered by +this policy, are also possible depending on circumstances, but this is at the +discretion of the pip developers. Note that the documentation is the sole reference for what counts as agreed -behaviour. If something isn't explicitly mentioned in the documentation, it can +behavior. If something isn't explicitly mentioned in the documentation, it can be changed without warning, or any deprecation period, in a pip release. However, we are aware that the documentation isn't always complete - PRs that -document existing behaviour with the intention of covering that behaviour with +document existing behavior with the intention of covering that behavior with the above deprecation process are always acceptable, and will be considered on their merits. diff --git a/news/deprecation.process b/news/deprecation.process index 0c3e042fe..9545be4ff 100644 --- a/news/deprecation.process +++ b/news/deprecation.process @@ -1 +1,2 @@ -Formally document our deprecation process +Formally document our deprecation process as a minimum of 6 months of deprecation +warnings. From ca5cca342637247f58c005711798cfd7e6adf6a5 Mon Sep 17 00:00:00 2001 From: Oliver Jeeves Date: Mon, 2 Jul 2018 13:14:28 +0100 Subject: [PATCH 059/129] Fix error trying to generate scripts warning when PATH not set fixes #5558 --- news/5558.bugfix | 1 + src/pip/_internal/wheel.py | 2 +- tests/unit/test_wheel.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 news/5558.bugfix diff --git a/news/5558.bugfix b/news/5558.bugfix new file mode 100644 index 000000000..f8965d55a --- /dev/null +++ b/news/5558.bugfix @@ -0,0 +1 @@ +Fix error trying to generate scripts warning when PATH not set diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 7ad7a1b8a..fcf9d3d38 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -163,7 +163,7 @@ def message_about_scripts_not_on_PATH(scripts): # We don't want to warn for directories that are on PATH. not_warn_dirs = [ os.path.normcase(i).rstrip(os.sep) for i in - os.environ["PATH"].split(os.pathsep) + os.environ.get("PATH", "").split(os.pathsep) ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6d3c1c7a7..005fde42c 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -470,3 +470,16 @@ class TestMessageAboutScriptsNotOnPATH(object): scripts=[os.path.join('a', 'b', 'c')] ) assert retval is None + + def test_missing_PATH_env_treated_as_empty_PATH_env(self): + scripts = ['a/b/foo'] + + env = os.environ.copy() + del env['PATH'] + with patch.dict('os.environ', env, clear=True): + retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) + + with patch.dict('os.environ', {'PATH': ''}): + retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) + + assert retval_missing == retval_empty From e834643f5c96b255d80c0858ae73909c316fd350 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 4 Jul 2018 17:48:14 +0530 Subject: [PATCH 060/129] Update 5558.bugfix --- news/5558.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/5558.bugfix b/news/5558.bugfix index f8965d55a..0407d5742 100644 --- a/news/5558.bugfix +++ b/news/5558.bugfix @@ -1 +1 @@ -Fix error trying to generate scripts warning when PATH not set +Fix a crash that occurs when PATH not set, while generating script location warning. From 763d2a761fd8880287dfe3535effd40a033037b1 Mon Sep 17 00:00:00 2001 From: gkdoc <40815324+gkdoc@users.noreply.github.com> Date: Fri, 6 Jul 2018 21:33:14 -0700 Subject: [PATCH 061/129] Remove redundancies in list in user_guide.rst --- docs/user_guide.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 8f5326643..ee6c31e50 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -336,13 +336,13 @@ variable ``PIP_CONFIG_FILE``. If multiple configuration files are found by pip then they are combined in the following order: -1. Firstly the site-wide file is read, then -2. The per-user file is read, and finally -3. The virtualenv-specific file is read. +1. The site-wide file is read +2. The per-user file is read +3. The virtualenv-specific file is read Each file read overrides any values read from previous files, so if the global timeout is specified in both the site-wide file and the per-user file -then the latter value is the one that will be used. +then the latter value will be used. The names of the settings are derived from the long command line option, e.g. if you want to use a different package index (``--index-url``) and set the From a6e2d4d09950725c4f3085ccab30a05770a1c1d7 Mon Sep 17 00:00:00 2001 From: gkdoc <40815324+gkdoc@users.noreply.github.com> Date: Fri, 6 Jul 2018 21:44:10 -0700 Subject: [PATCH 062/129] Remove redundancies in list in user_guide.rst --- news/f86bc245-7633-470a-a05e-925309ba648d.trivial | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/f86bc245-7633-470a-a05e-925309ba648d.trivial diff --git a/news/f86bc245-7633-470a-a05e-925309ba648d.trivial b/news/f86bc245-7633-470a-a05e-925309ba648d.trivial new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/news/f86bc245-7633-470a-a05e-925309ba648d.trivial @@ -0,0 +1 @@ + From 64d205e8962aad6a51e168da5991a206a66dd143 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Sat, 7 Jul 2018 20:47:16 -0700 Subject: [PATCH 063/129] Refactor out a new VersionControl.fetch_new() method. --- ...2A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial | 0 src/pip/_internal/vcs/__init__.py | 11 ++++ src/pip/_internal/vcs/bazaar.py | 21 ++++---- src/pip/_internal/vcs/git.py | 52 ++++++++++--------- src/pip/_internal/vcs/mercurial.py | 23 ++++---- src/pip/_internal/vcs/subversion.py | 21 ++++---- 6 files changed, 76 insertions(+), 52 deletions(-) create mode 100644 news/052A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial diff --git a/news/052A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial b/news/052A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 5f32172f0..97d32859b 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -260,6 +260,17 @@ class VersionControl(object): """ raise NotImplementedError + def fetch_new(self, dest, url, rev_options): + """ + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. + """ + raise NotImplementedError + def switch(self, dest, url, rev_options): """ Switch the repo at ``dest`` to point to ``URL``. diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index b4e46e0e1..71553533e 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -48,6 +48,17 @@ class Bazaar(VersionControl): cwd=temp_dir.path, show_stdout=False, ) + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = ['branch', '-q'] + rev_options.to_args() + [url, dest] + self.run_command(cmd_args) + def switch(self, dest, url, rev_options): self.run_command(['switch', url], cwd=dest) @@ -59,15 +70,7 @@ class Bazaar(VersionControl): url, rev = self.get_url_rev() rev_options = self.make_rev_options(rev) if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Checking out %s%s to %s', - url, - rev_display, - display_path(dest), - ) - cmd_args = ['branch', '-q'] + rev_options.to_args() + [url, dest] - self.run_command(cmd_args) + self.fetch_new(dest, url, rev_options) def get_url_rev(self): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 7ef82d8ac..b2d6e1439 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -155,6 +155,33 @@ class Git(VersionControl): return self.get_revision(dest) == name + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Cloning %s%s to %s', url, rev_display, display_path(dest), + ) + self.run_command(['clone', '-q', url, dest]) + + if rev_options.rev: + # Then a specific revision was requested. + rev_options = self.check_rev_options(dest, rev_options) + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + rev = rev_options.rev + # Only fetch the revision if it's a ref + if rev.startswith('refs/'): + self.run_command( + ['fetch', '-q', url] + rev_options.to_args(), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + rev = 'FETCH_HEAD' + self.run_command(['checkout', '-q', rev], cwd=dest) + + #: repo may contain submodules + self.update_submodules(dest) + def switch(self, dest, url, rev_options): self.run_command(['config', 'remote.origin.url', url], cwd=dest) cmd_args = ['checkout', '-q'] + rev_options.to_args() @@ -180,30 +207,7 @@ class Git(VersionControl): url, rev = self.get_url_rev() rev_options = self.make_rev_options(rev) if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Cloning %s%s to %s', url, rev_display, display_path(dest), - ) - self.run_command(['clone', '-q', url, dest]) - - if rev: - rev_options = self.check_rev_options(dest, rev_options) - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - rev = rev_options.rev - # Only fetch the revision if it's a ref - if rev.startswith('refs/'): - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - rev = 'FETCH_HEAD' - self.run_command(['checkout', '-q', rev], cwd=dest) - - #: repo may contain submodules - self.update_submodules(dest) + self.fetch_new(dest, url, rev_options) def get_url(self, location): """Return URL of the first remote encountered.""" diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 52a1cce54..2ab325a5e 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -31,6 +31,18 @@ class Mercurial(VersionControl): ['archive', location], show_stdout=False, cwd=temp_dir.path ) + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Cloning hg %s%s to %s', + url, + rev_display, + display_path(dest), + ) + self.run_command(['clone', '--noupdate', '-q', url, dest]) + cmd_args = ['update', '-q'] + rev_options.to_args() + self.run_command(cmd_args, cwd=dest) + def switch(self, dest, url, rev_options): repo_config = os.path.join(dest, self.dirname, 'hgrc') config = configparser.SafeConfigParser() @@ -56,16 +68,7 @@ class Mercurial(VersionControl): url, rev = self.get_url_rev() rev_options = self.make_rev_options(rev) if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Cloning hg %s%s to %s', - url, - rev_display, - display_path(dest), - ) - self.run_command(['clone', '--noupdate', '-q', url, dest]) - cmd_args = ['update', '-q'] + rev_options.to_args() - self.run_command(cmd_args, cwd=dest) + self.fetch_new(dest, url, rev_options) def get_url(self, location): url = self.run_command( diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index de448c7c6..3a56fe6a5 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -73,6 +73,17 @@ class Subversion(VersionControl): cmd_args = ['export'] + rev_options.to_args() + [url, location] self.run_command(cmd_args, show_stdout=False) + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = ['checkout', '-q'] + rev_options.to_args() + [url, dest] + self.run_command(cmd_args) + def switch(self, dest, url, rev_options): cmd_args = ['switch'] + rev_options.to_args() + [url, dest] self.run_command(cmd_args) @@ -86,15 +97,7 @@ class Subversion(VersionControl): rev_options = get_rev_options(self, url, rev) url = remove_auth_from_url(url) if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Checking out %s%s to %s', - url, - rev_display, - display_path(dest), - ) - cmd_args = ['checkout', '-q'] + rev_options.to_args() + [url, dest] - self.run_command(cmd_args) + self.fetch_new(dest, url, rev_options) def get_location(self, dist, dependency_links): for url in dependency_links: From e3b06f9e51aa343fc2b2e66227ecee1b1333148e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 9 Jul 2018 07:48:36 +0530 Subject: [PATCH 064/129] Isolate when pyproject.toml does not have build-system.requires --- src/pip/_internal/req/req_install.py | 15 ++++++++------- tests/functional/test_install.py | 4 ++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index d9e3df745..c40e9262a 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -585,16 +585,17 @@ class InstallRequirement(object): if requires is None: logging.warn(template, self, "it is missing.") warnings.warn( - "Future versions of pip will reject packages with " - "pyproject.toml files that do not comply with PEP 518.", + "Future versions of pip may reject packages with " + "pyproject.toml files that do not contain the [build-system]" + "table and the requires key, as specified in PEP 518.", RemovedInPip12Warning, ) - # NOTE: Currently allowing projects to skip this key so that they - # can transition to a PEP 518 compliant pyproject.toml or - # push to update the PEP. - # Come pip 19.0, bring this to compliance with PEP 518. - return None + # Currently, we're isolating the build based on the presence of the + # pyproject.toml file. If the user doesn't specify + # build-system.requires, assume they intended to use setuptools and + # wheel for now. + return ["setuptools", "wheel"] else: # Error out if it's not a list of strings is_list_of_str = isinstance(requires, list) and all( diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index b10251907..c8095c224 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -55,6 +55,10 @@ def test_pep518_allows_but_warns_missing_requires(script, data, common_wheels): ) assert "does not comply with PEP 518" in result.stderr assert "DEPRECATION" in result.stderr + + # We want it to go through isolation for now. + assert "Installing build dependencies" in result.stdout, result.stdout + assert result.returncode == 0 assert result.files_created From 701d010c0298f3ef2cc003d5f0957c4924c2e5ae Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Sun, 8 Jul 2018 04:21:00 -0700 Subject: [PATCH 065/129] Implement VersionControl.obtain() in the base class. --- ...450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial | 0 src/pip/_internal/vcs/__init__.py | 56 +++++++++++-------- src/pip/_internal/vcs/bazaar.py | 6 -- src/pip/_internal/vcs/git.py | 6 -- src/pip/_internal/vcs/mercurial.py | 6 -- src/pip/_internal/vcs/subversion.py | 24 ++++---- tests/unit/test_vcs.py | 18 ++++++ 7 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 news/03450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial diff --git a/news/03450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial b/news/03450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 97d32859b..08bd71fa8 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -232,6 +232,24 @@ class VersionControl(object): url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev + def get_url_rev_args(self, url): + """ + Return the URL and RevOptions "extra arguments" to use in obtain(), + as a tuple (url, extra_args). + """ + return url, [] + + def get_url_rev_options(self): + """ + Return the URL and RevOptions object to use in obtain(), as a tuple + (url, rev_options). + """ + url, rev = self.get_url_rev() + url, extra_args = self.get_url_rev_args(url) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return url, rev_options + def get_info(self, location): """ Returns (url, revision), where both are strings @@ -253,13 +271,6 @@ class VersionControl(object): """ return (self.normalize_url(url1) == self.normalize_url(url2)) - def obtain(self, dest): - """ - Called when installing or updating an editable package, takes the - source path of the checkout. - """ - raise NotImplementedError - def fetch_new(self, dest, url, rev_options): """ Fetch a revision from a repository, in the case that this is the @@ -299,18 +310,19 @@ class VersionControl(object): """ raise NotImplementedError - def check_destination(self, dest, url, rev_options): + def obtain(self, dest): """ - Prepare a location to receive a checkout/clone. - - Return True if the location is ready for (and requires) a - checkout/clone, False otherwise. + Install or update in editable mode the package represented by this + VersionControl object. Args: - rev_options: a RevOptions object. + dest: the repository directory in which to install or update. """ + url, rev_options = self.get_url_rev_options() + if not os.path.exists(dest): - return True + self.fetch_new(dest, url, rev_options) + return rev_display = rev_options.to_display() if os.path.exists(os.path.join(dest, self.dirname)): @@ -332,7 +344,7 @@ class VersionControl(object): self.update(dest, rev_options) else: logger.info('Skipping because already up-to-date.') - return False + return logger.warning( '%s %s in %s exists with URL %s', @@ -365,7 +377,8 @@ class VersionControl(object): if response == 'w': logger.warning('Deleting %s', display_path(dest)) rmtree(dest) - return True + self.fetch_new(dest, url, rev_options) + return if response == 'b': dest_dir = backup_dir(dest) @@ -373,8 +386,10 @@ class VersionControl(object): 'Backing up %s to %s', display_path(dest), dest_dir, ) shutil.move(dest, dest_dir) - return True + self.fetch_new(dest, url, rev_options) + return + # Do nothing if the response is "i". if response == 's': logger.info( 'Switching %s %s to %s%s', @@ -385,10 +400,6 @@ class VersionControl(object): ) self.switch(dest, url, rev_options) - # Do nothing if the response is "i". - - return False - def unpack(self, location): """ Clean up current location and download the url repository @@ -410,7 +421,8 @@ class VersionControl(object): def get_url(self, location): """ Return the url used at location - Used in get_info or check_destination + + This is used in get_info() and obtain(). """ raise NotImplementedError diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index 71553533e..80c14903f 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -66,12 +66,6 @@ class Bazaar(VersionControl): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = self.make_rev_options(rev) - if self.check_destination(dest, url, rev_options): - self.fetch_new(dest, url, rev_options) - def get_url_rev(self): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it url, rev = super(Bazaar, self).get_url_rev() diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index b2d6e1439..6bda129a8 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -203,12 +203,6 @@ class Git(VersionControl): #: update submodules self.update_submodules(dest) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = self.make_rev_options(rev) - if self.check_destination(dest, url, rev_options): - self.fetch_new(dest, url, rev_options) - def get_url(self, location): """Return URL of the first remote encountered.""" remotes = self.run_command( diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 2ab325a5e..2d0750cbb 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -64,12 +64,6 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = self.make_rev_options(rev) - if self.check_destination(dest, url, rev_options): - self.fetch_new(dest, url, rev_options) - def get_url(self, location): url = self.run_command( ['showconfig', 'paths.default'], diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 3a56fe6a5..f42aa2f43 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -61,9 +61,8 @@ class Subversion(VersionControl): def export(self, location): """Export the svn repository at the url to the destination location""" - url, rev = self.get_url_rev() - rev_options = get_rev_options(self, url, rev) - url = remove_auth_from_url(url) + url, rev_options = self.get_url_rev_options() + logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): @@ -92,13 +91,6 @@ class Subversion(VersionControl): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = get_rev_options(self, url, rev) - url = remove_auth_from_url(url) - if self.check_destination(dest, url, rev_options): - self.fetch_new(dest, url, rev_options) - def get_location(self, dist, dependency_links): for url in dependency_links: egg_fragment = Link(url).egg_fragment @@ -147,6 +139,12 @@ class Subversion(VersionControl): url = 'svn+' + url return url, rev + def get_url_rev_args(self, url): + extra_args = get_rev_options_args(url) + url = remove_auth_from_url(url) + + return url, extra_args + def get_url(self, location): # In cases where the source is in a subdirectory, not alongside # setup.py we have to look up in the location until we find a real @@ -225,9 +223,9 @@ class Subversion(VersionControl): return False -def get_rev_options(vcs, url, rev): +def get_rev_options_args(url): """ - Return a RevOptions object. + Return the extra arguments to pass to RevOptions. """ r = urllib_parse.urlsplit(url) if hasattr(r, 'username'): @@ -250,7 +248,7 @@ def get_rev_options(vcs, url, rev): if password: extra_args += ['--password', password] - return vcs.make_rev_options(rev, extra_args=extra_args) + return extra_args vcs.register(Subversion) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 1cfd4c409..1c7836bab 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -177,3 +177,21 @@ def test_bazaar_simple_urls(): def test_get_git_version(): git_version = Git().get_git_version() assert git_version >= parse_version('1.0.0') + + +@pytest.mark.parametrize('url, expected', [ + # Test a basic case. + ('svn+https://svn.myproject.org/MyProject#egg=MyProject', + ('svn+https://svn.myproject.org/MyProject#egg=MyProject', [])), + # Test with username and pass. + ('svn+https://user:pass@svn.myproject.org/MyProject#egg=MyProject', + ('svn+https://svn.myproject.org/MyProject#egg=MyProject', + ['--username', 'user', '--password', 'pass'])), +]) +def test_subversion__get_url_rev_args(url, expected): + """ + Test Subversion.get_url_rev_args(). + """ + vcs = Subversion() + actual = vcs.get_url_rev_args(url) + assert actual == expected From d096c6b3407335929b418fc34b79382126f5822b Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Mon, 9 Jul 2018 23:08:27 -0700 Subject: [PATCH 066/129] Add test_git__get_url_rev_args() to test a non-SVN backend. This addresses a review comment of @pradyunsg. --- tests/unit/test_vcs.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 1c7836bab..46e0d66ef 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -179,19 +179,36 @@ def test_get_git_version(): assert git_version >= parse_version('1.0.0') +# The non-SVN backends all have the same get_url_rev_args() implementation, +# so test with Git as a representative. @pytest.mark.parametrize('url, expected', [ # Test a basic case. - ('svn+https://svn.myproject.org/MyProject#egg=MyProject', - ('svn+https://svn.myproject.org/MyProject#egg=MyProject', [])), - # Test with username and pass. - ('svn+https://user:pass@svn.myproject.org/MyProject#egg=MyProject', - ('svn+https://svn.myproject.org/MyProject#egg=MyProject', + ('git+https://git.example.com/MyProject#egg=MyProject', + ('git+https://git.example.com/MyProject#egg=MyProject', [])), + # Test with username and password. + ('git+https://user:pass@git.example.com/MyProject#egg=MyProject', + ('git+https://user:pass@git.example.com/MyProject#egg=MyProject', [])), +]) +def test_git__get_url_rev_args(url, expected): + """ + Test Git.get_url_rev_args(). + """ + actual = Git().get_url_rev_args(url) + assert actual == expected + + +@pytest.mark.parametrize('url, expected', [ + # Test a basic case. + ('svn+https://svn.example.com/MyProject#egg=MyProject', + ('svn+https://svn.example.com/MyProject#egg=MyProject', [])), + # Test with username and password. + ('svn+https://user:pass@svn.example.com/MyProject#egg=MyProject', + ('svn+https://svn.example.com/MyProject#egg=MyProject', ['--username', 'user', '--password', 'pass'])), ]) def test_subversion__get_url_rev_args(url, expected): """ Test Subversion.get_url_rev_args(). """ - vcs = Subversion() - actual = vcs.get_url_rev_args(url) + actual = Subversion().get_url_rev_args(url) assert actual == expected From 3401f4de51870c0dd7fc03746e3b6793a847e9d9 Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Fri, 6 Jul 2018 14:34:12 +1000 Subject: [PATCH 067/129] Added User Guide section on proxy configuration. Fixed #512. Signed-off-by: Brett Randall --- docs/user_guide.rst | 15 +++++++++++++++ news/512.doc | 1 + news/5574.doc | 1 + 3 files changed, 17 insertions(+) create mode 100644 news/512.doc create mode 100644 news/5574.doc diff --git a/docs/user_guide.rst b/docs/user_guide.rst index ee6c31e50..142bc87a2 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -51,6 +51,21 @@ For more information and examples, see the :ref:`pip install` reference. .. _`Requirements Files`: +Using a Proxy Server +******************** + +When installing packages from `PyPI`_, pip requires internet access, which +in many corporate environments requires an outbound HTTP proxy server. + + +pip can be configured to connect through a proxy server in various ways: + +* using the ``--proxy`` command-line option to specify a proxy in the form + ``[user:passwd@]proxy.server:port`` +* using ``proxy`` in a :ref:`config-file` +* by setting the standard environment-variables ``http_proxy``, ``https_proxy`` + and ``no_proxy``. + Requirements Files ****************** diff --git a/news/512.doc b/news/512.doc new file mode 100644 index 000000000..e0382fbe4 --- /dev/null +++ b/news/512.doc @@ -0,0 +1 @@ +Adding documentation for making pip use a proxy server diff --git a/news/5574.doc b/news/5574.doc new file mode 100644 index 000000000..e0382fbe4 --- /dev/null +++ b/news/5574.doc @@ -0,0 +1 @@ +Adding documentation for making pip use a proxy server From 26457497b8c562eff2c0d349191890e218721d02 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Tue, 10 Jul 2018 00:31:57 -0700 Subject: [PATCH 068/129] Add failing test. --- tests/unit/test_vcs.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 46e0d66ef..67378f263 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -129,7 +129,19 @@ def test_translate_egg_surname(): assert vc.translate_egg_surname("foo/1.2.3") == "foo_1.2.3" -def test_bazaar_simple_urls(): +def test_git__get_url_rev__idempotent(): + """ + Check that Git.get_url_rev() is idempotent for what the code calls + "stub URLs" (i.e. URLs that don't contain "://"). + """ + vcs = Git('git+git@git.example.com:MyProject#egg=MyProject') + result1 = vcs.get_url_rev() + result2 = vcs.get_url_rev() + assert result1 == ('git@git.example.com:MyProject', None) + assert result2 == ('git@git.example.com:MyProject', None) + + +def test_bazaar__get_url_rev(): """ Test bzr url support. From 1f6e1e51ec2dfdbfeba9db12f189e41213bf46e1 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Wed, 11 Jul 2018 00:51:38 -0700 Subject: [PATCH 069/129] Make get_url_rev() accept an url argument. --- ...8CCC8B-0506-40B2-B108-616C18016E59.trivial | 0 src/pip/_internal/vcs/__init__.py | 16 +++++++------- src/pip/_internal/vcs/bazaar.py | 4 ++-- src/pip/_internal/vcs/git.py | 12 +++++----- src/pip/_internal/vcs/subversion.py | 6 ++--- tests/unit/test_vcs.py | 22 +++++++++++-------- 6 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 news/408CCC8B-0506-40B2-B108-616C18016E59.trivial diff --git a/news/408CCC8B-0506-40B2-B108-616C18016E59.trivial b/news/408CCC8B-0506-40B2-B108-616C18016E59.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 08bd71fa8..704f1c3fa 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -213,7 +213,7 @@ class VersionControl(object): """ raise NotImplementedError - def get_url_rev(self): + def get_url_rev(self, url): """ Returns the correct repository URL and revision by parsing the given repository URL @@ -223,8 +223,8 @@ class VersionControl(object): "The format is +://, " "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" ) - assert '+' in self.url, error_message % self.url - url = self.url.split('+', 1)[1] + assert '+' in url, error_message % url + url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) rev = None if '@' in path: @@ -239,12 +239,12 @@ class VersionControl(object): """ return url, [] - def get_url_rev_options(self): + def get_url_rev_options(self, url): """ - Return the URL and RevOptions object to use in obtain(), as a tuple - (url, rev_options). + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). """ - url, rev = self.get_url_rev() + url, rev = self.get_url_rev(url) url, extra_args = self.get_url_rev_args(url) rev_options = self.make_rev_options(rev, extra_args=extra_args) @@ -318,7 +318,7 @@ class VersionControl(object): Args: dest: the repository directory in which to install or update. """ - url, rev_options = self.get_url_rev_options() + url, rev_options = self.get_url_rev_options(self.url) if not os.path.exists(dest): self.fetch_new(dest, url, rev_options) diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index 80c14903f..93b76165a 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -66,9 +66,9 @@ class Bazaar(VersionControl): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url_rev(self): + def get_url_rev(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev() + url, rev = super(Bazaar, self).get_url_rev(url) if url.startswith('ssh://'): url = 'bzr+' + url return url, rev diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 6bda129a8..9ee2e012f 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -265,20 +265,20 @@ class Git(VersionControl): req += '&subdirectory=' + subdirectory return req - def get_url_rev(self): + def get_url_rev(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't work with a ssh:// scheme (e.g. GitHub). But we need a scheme for parsing. Hence we remove it again afterwards and return it as a stub. """ - if '://' not in self.url: - assert 'file:' not in self.url - self.url = self.url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev() + if '://' not in url: + assert 'file:' not in url + url = url.replace('git+', 'git+ssh://') + url, rev = super(Git, self).get_url_rev(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev() + url, rev = super(Git, self).get_url_rev(url) return url, rev diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index f42aa2f43..70787750b 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -61,7 +61,7 @@ class Subversion(VersionControl): def export(self, location): """Export the svn repository at the url to the destination location""" - url, rev_options = self.get_url_rev_options() + url, rev_options = self.get_url_rev_options(self.url) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): @@ -132,9 +132,9 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def get_url_rev(self): + def get_url_rev(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev() + url, rev = super(Subversion, self).get_url_rev(url) if url.startswith('ssh://'): url = 'svn+' + url return url, rev diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 67378f263..c9bb882a9 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -133,10 +133,14 @@ def test_git__get_url_rev__idempotent(): """ Check that Git.get_url_rev() is idempotent for what the code calls "stub URLs" (i.e. URLs that don't contain "://"). + + Also check that it doesn't change self.url. """ - vcs = Git('git+git@git.example.com:MyProject#egg=MyProject') - result1 = vcs.get_url_rev() - result2 = vcs.get_url_rev() + url = 'git+git@git.example.com:MyProject#egg=MyProject' + vcs = Git(url) + result1 = vcs.get_url_rev(url) + assert vcs.url == url + result2 = vcs.get_url_rev(url) assert result1 == ('git@git.example.com:MyProject', None) assert result2 == ('git@git.example.com:MyProject', None) @@ -166,22 +170,22 @@ def test_bazaar__get_url_rev(): url='bzr+lp:MyLaunchpadProject#egg=MyLaunchpadProject' ) - assert http_bzr_repo.get_url_rev() == ( + assert http_bzr_repo.get_url_rev(http_bzr_repo.url) == ( 'http://bzr.myproject.org/MyProject/trunk/', None, ) - assert https_bzr_repo.get_url_rev() == ( + assert https_bzr_repo.get_url_rev(https_bzr_repo.url) == ( 'https://bzr.myproject.org/MyProject/trunk/', None, ) - assert ssh_bzr_repo.get_url_rev() == ( + assert ssh_bzr_repo.get_url_rev(ssh_bzr_repo.url) == ( 'bzr+ssh://bzr.myproject.org/MyProject/trunk/', None, ) - assert ftp_bzr_repo.get_url_rev() == ( + assert ftp_bzr_repo.get_url_rev(ftp_bzr_repo.url) == ( 'ftp://bzr.myproject.org/MyProject/trunk/', None, ) - assert sftp_bzr_repo.get_url_rev() == ( + assert sftp_bzr_repo.get_url_rev(sftp_bzr_repo.url) == ( 'sftp://bzr.myproject.org/MyProject/trunk/', None, ) - assert launchpad_bzr_repo.get_url_rev() == ( + assert launchpad_bzr_repo.get_url_rev(launchpad_bzr_repo.url) == ( 'lp:MyLaunchpadProject', None, ) From ae3705972c91c96d3caa2cbbbd6ba418311e706e Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Wed, 11 Jul 2018 01:03:11 -0700 Subject: [PATCH 070/129] Fix typo: tools/test-requirements.txt. --- docs/development.rst | 4 ++-- news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial | 0 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial diff --git a/docs/development.rst b/docs/development.rst index 0467bbf8b..bc1d36777 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -69,7 +69,7 @@ Running tests OS Requirements: subversion, bazaar, git, and mercurial. Python Requirements: tox or install all packages listed in -`tools/test-requirements.txt`_ +`tools/tests-requirements.txt`_ Ways to run the tests locally:: @@ -262,4 +262,4 @@ to create one of these the changes should already be merged into the .. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml .. _`appveyor.yml`: https://github.com/pypa/pip/blob/master/appveyor.yml .. _`Travis CI Pull Requests`: https://travis-ci.org/pypa/pip/pull_requests -.. _`tools/test-requirements.txt`: https://github.com/pypa/pip/blob/master/tools/test-requirements.txt +.. _`tools/tests-requirements.txt`: https://github.com/pypa/pip/blob/master/tools/tests-requirements.txt diff --git a/news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial b/news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial new file mode 100644 index 000000000..e69de29bb From c71ac73102fb18335172582e713df2d972099233 Mon Sep 17 00:00:00 2001 From: Bstrdsmkr Date: Thu, 5 Jul 2018 15:06:25 -0400 Subject: [PATCH 071/129] Allow PEP508 url dependencies in install_requires --- news/4187.feature | 5 +++++ src/pip/_internal/models/index.py | 10 ++++++++-- src/pip/_internal/req/req_install.py | 15 +++++++++++--- tests/functional/test_install.py | 29 +++++++++++++++++++++++++--- 4 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 news/4187.feature diff --git a/news/4187.feature b/news/4187.feature new file mode 100644 index 000000000..03a874bc3 --- /dev/null +++ b/news/4187.feature @@ -0,0 +1,5 @@ +Allow PEP 508 URL requirements to be used as dependencies. + +As a security measure, pip will raise an exception when installing packages from +PyPI if those packages depend on packages not also hosted on PyPI. +In the future, PyPI will block uploading packages with such external URL dependencies directly. diff --git a/src/pip/_internal/models/index.py b/src/pip/_internal/models/index.py index a7f10c8db..136216325 100644 --- a/src/pip/_internal/models/index.py +++ b/src/pip/_internal/models/index.py @@ -2,14 +2,20 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse class Index(object): - def __init__(self, url): + def __init__(self, url, file_storage_domain): self.url = url self.netloc = urllib_parse.urlsplit(url).netloc self.simple_url = self.url_to_path('simple') self.pypi_url = self.url_to_path('pypi') + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain + def url_to_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = Index('https://pypi.org/', 'files.pythonhosted.org') +TestPyPI = Index('https://test.pypi.org/', 'test-files.pythonhosted.org') diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 043353048..fb7d4f1b8 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -32,6 +32,7 @@ from pip._internal.exceptions import InstallationError from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pip._internal.models.index import PyPI, TestPyPI from pip._internal.req.req_uninstall import UninstallPathSet from pip._internal.utils.deprecation import ( RemovedInPip11Warning, RemovedInPip12Warning, @@ -169,11 +170,19 @@ class InstallRequirement(object): req = Requirement(req) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) - if req.url: + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if req.url and comes_from.link.netloc in domains_not_allowed: + # Explicitly disallow pypi packages that depend on external urls raise InstallationError( - "Direct url requirement (like %s) are not allowed for " - "dependencies" % req + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) ) + return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) @classmethod diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index b10251907..bae5e0fe2 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -8,6 +8,7 @@ from os.path import curdir, join, pardir import pytest from pip._internal import pep425tags +from pip._internal.models.index import PyPI, TestPyPI from pip._internal.status_codes import ERROR from pip._internal.utils.misc import rmtree from tests.lib import ( @@ -1267,9 +1268,31 @@ def test_install_pep508_with_url_in_install_requires(script): 'ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4' ], ) - res = script.pip('install', pkga_path, expect_error=True) - assert "Direct url requirement " in res.stderr, str(res) - assert "are not allowed for dependencies" in res.stderr, str(res) + res = script.pip('install', pkga_path) + assert "Successfully installed packaging-15.3" in str(res), str(res) + + +@pytest.mark.network +@pytest.mark.parametrize('index', (PyPI.simple_url, TestPyPI.simple_url)) +def test_install_from_test_pypi_with_ext_url_dep_is_blocked(script, index): + res = script.pip( + 'install', + '--index-url', + index, + 'pep-508-url-deps', + expect_error=True, + ) + error_message = ( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI." + ) + error_cause = ( + "pep-508-url-deps depends on sampleproject@ " + "https://github.com/pypa/sampleproject/archive/master.zip" + ) + assert res.returncode == 1 + assert error_message in res.stderr, str(res) + assert error_cause in res.stderr, str(res) def test_installing_scripts_outside_path_prints_warning(script): From 549cdebf96289f97c80f66e21b3db0d1f9b9bf30 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 15 Jul 2018 15:55:27 +0530 Subject: [PATCH 072/129] Remove redundant newline --- docs/user_guide.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 142bc87a2..4520c3831 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -57,7 +57,6 @@ Using a Proxy Server When installing packages from `PyPI`_, pip requires internet access, which in many corporate environments requires an outbound HTTP proxy server. - pip can be configured to connect through a proxy server in various ways: * using the ``--proxy`` command-line option to specify a proxy in the form From 7769ddd52c338ad7e369ef1c404e1bd63a3b63f5 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 22 Jun 2018 08:16:41 +0530 Subject: [PATCH 073/129] Rename deprecation warnings to not be versioned --- src/pip/_internal/utils/deprecation.py | 51 +++++++++++--------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 89b66e9ad..a1644c778 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -16,50 +16,43 @@ class PipDeprecationWarning(Warning): pass -class Pending(object): +class PipPendingDeprecationWarning(PipDeprecationWarning): pass -class RemovedInPip12Warning(PipDeprecationWarning, Pending): - pass - - -# Warnings <-> Logging Integration - - _warnings_showwarning = None # type: Any +# Warnings <-> Logging Integration def _showwarning(message, category, filename, lineno, file=None, line=None): if file is not None: if _warnings_showwarning is not None: _warnings_showwarning( message, category, filename, lineno, file, line, ) - else: - if issubclass(category, PipDeprecationWarning): - # We use a specially named logger which will handle all of the - # deprecation messages for pip. - logger = logging.getLogger("pip._internal.deprecations") + elif issubclass(category, PipDeprecationWarning): + # We use a specially named logger which will handle all of the + # deprecation messages for pip. + logger = logging.getLogger("pip._internal.deprecations") - # This is purposely using the % formatter here instead of letting - # the logging module handle the interpolation. This is because we - # want it to appear as if someone typed this entire message out. - log_message = "DEPRECATION: %s" % message + # This is purposely using the % formatter here instead of letting + # the logging module handle the interpolation. This is because we + # want it to appear as if someone typed this entire message out. + log_message = "DEPRECATION: %s" % message - # PipDeprecationWarnings that are Pending still have at least 2 - # versions to go until they are removed so they can just be - # warnings. Otherwise, they will be removed in the very next - # version of pip. We want these to be more obvious so we use the - # ERROR logging level. - if issubclass(category, Pending): - logger.warning(log_message) - else: - logger.error(log_message) + # PipPendingDeprecationWarnings still have at least 2 + # versions to go until they are removed so they can just be + # warnings. Otherwise, they will be removed in the very next + # version of pip. We want these to be more obvious so we use the + # ERROR logging level. + if issubclass(category, PipPendingDeprecationWarning): + logger.warning(log_message) else: - _warnings_showwarning( - message, category, filename, lineno, file, line, - ) + logger.error(log_message) + else: + _warnings_showwarning( + message, category, filename, lineno, file, line, + ) def install_warning_logger(): From a8eaf1e0ff013b9e61e4420b28477f05470ff0ae Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 22 Jun 2018 09:30:17 +0530 Subject: [PATCH 074/129] Add a helper function for deprecation (and use it) --- src/pip/_internal/index.py | 8 ++++---- src/pip/_internal/operations/freeze.py | 8 ++++---- src/pip/_internal/req/req_install.py | 9 +++++---- src/pip/_internal/utils/deprecation.py | 20 +++++++++++++++++++- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 857486cc1..fae26eeff 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -9,7 +9,6 @@ import os import posixpath import re import sys -import warnings from collections import namedtuple from pip._vendor import html5lib, requests, six @@ -29,7 +28,7 @@ from pip._internal.exceptions import ( ) from pip._internal.models.index import PyPI from pip._internal.pep425tags import get_supported -from pip._internal.utils.deprecation import RemovedInPip12Warning +from pip._internal.utils.deprecation import deprecated from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, @@ -212,10 +211,11 @@ class PackageFinder(object): # # dependency_links value # # FIXME: also, we should track comes_from (i.e., use Link) if self.process_dependency_links: - warnings.warn( + deprecated( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - RemovedInPip12Warning, + replacement=None, + issue=4187, ) self.dependency_links.extend(links) diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index 82d0a8c4b..80fdd1044 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -4,7 +4,6 @@ import collections import logging import os import re -import warnings from pip._vendor import pkg_resources, six from pip._vendor.packaging.utils import canonicalize_name @@ -13,7 +12,7 @@ from pip._vendor.pkg_resources import RequirementParseError from pip._internal.exceptions import InstallationError from pip._internal.req import InstallRequirement from pip._internal.req.req_file import COMMENT_RE -from pip._internal.utils.deprecation import RemovedInPip12Warning +from pip._internal.utils.deprecation import deprecated from pip._internal.utils.misc import ( dist_is_editable, get_installed_distributions, ) @@ -216,10 +215,11 @@ class FrozenRequirement(object): 'for this package:' ) else: - warnings.warn( + deprecated( "SVN editable detection based on dependency links " "will be dropped in the future.", - RemovedInPip12Warning, + replacement=None, + issue=4187, ) comments.append( '# Installing as editable to satisfy requirement %s:' % diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 8838dd2eb..6a33e4e42 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -8,7 +8,6 @@ import shutil import sys import sysconfig import traceback -import warnings import zipfile from distutils.util import change_root from email.parser import FeedParser # type: ignore @@ -33,7 +32,7 @@ from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) from pip._internal.req.req_uninstall import UninstallPathSet -from pip._internal.utils.deprecation import RemovedInPip12Warning +from pip._internal.utils.deprecation import deprecated from pip._internal.utils.hashes import Hashes from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( @@ -582,11 +581,13 @@ class InstallRequirement(object): if requires is None: logging.warn(template, self, "it is missing.") - warnings.warn( + deprecated( "Future versions of pip may reject packages with " "pyproject.toml files that do not contain the [build-system]" "table and the requires key, as specified in PEP 518.", - RemovedInPip12Warning, + replacement=None, + issue=5416, + imminent=True, ) # Currently, we're isolating the build based on the presence of the diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index a1644c778..271e6d9fc 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -9,7 +9,7 @@ import warnings from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Optional # noqa: F401 class PipDeprecationWarning(Warning): @@ -64,3 +64,21 @@ def install_warning_logger(): if _warnings_showwarning is None: _warnings_showwarning = warnings.showwarning warnings.showwarning = _showwarning + + +def deprecated(reason, replacement, issue=None, imminent=False): + # type: (str, Optional[str], Optional[int], bool) -> None + if imminent: + category = PipDeprecationWarning + else: + category = PipPendingDeprecationWarning + + # Construct a nice message. + message = reason + if replacement is not None: + message += " An alternative is to {}.".format(replacement) + if issue is not None: + url = "https://github.com/pypa/pip/issues/" + str(issue) + message += " You can find discussion regarding this at {}.".format(url) + + warnings.warn(message, category=category, stacklevel=2) From a1912454b8eb3bbbc3347089da59543b700165a7 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 22 Jun 2018 09:31:34 +0530 Subject: [PATCH 075/129] Rename variable for clarity It doesn't represent warnings.showwarning's current state so, the name shouldn't suggest that. --- src/pip/_internal/utils/deprecation.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 271e6d9fc..94b3caa0a 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -20,14 +20,14 @@ class PipPendingDeprecationWarning(PipDeprecationWarning): pass -_warnings_showwarning = None # type: Any +_original_showwarning = None # type: Any # Warnings <-> Logging Integration def _showwarning(message, category, filename, lineno, file=None, line=None): if file is not None: - if _warnings_showwarning is not None: - _warnings_showwarning( + if _original_showwarning is not None: + _original_showwarning( message, category, filename, lineno, file, line, ) elif issubclass(category, PipDeprecationWarning): @@ -50,7 +50,7 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): else: logger.error(log_message) else: - _warnings_showwarning( + _original_showwarning( message, category, filename, lineno, file, line, ) @@ -59,10 +59,10 @@ def install_warning_logger(): # Enable our Deprecation Warnings warnings.simplefilter("default", PipDeprecationWarning, append=True) - global _warnings_showwarning + global _original_showwarning - if _warnings_showwarning is None: - _warnings_showwarning = warnings.showwarning + if _original_showwarning is None: + _original_showwarning = warnings.showwarning warnings.showwarning = _showwarning From 4c2b268d52e84452e785468dfbfabac80874ee47 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 6 Jul 2018 04:41:19 +0530 Subject: [PATCH 076/129] Add "DEPRECATION" to start of message when composing it --- src/pip/_internal/utils/deprecation.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 94b3caa0a..e8f59705c 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -35,11 +35,6 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): # deprecation messages for pip. logger = logging.getLogger("pip._internal.deprecations") - # This is purposely using the % formatter here instead of letting - # the logging module handle the interpolation. This is because we - # want it to appear as if someone typed this entire message out. - log_message = "DEPRECATION: %s" % message - # PipPendingDeprecationWarnings still have at least 2 # versions to go until they are removed so they can just be # warnings. Otherwise, they will be removed in the very next @@ -74,7 +69,9 @@ def deprecated(reason, replacement, issue=None, imminent=False): category = PipPendingDeprecationWarning # Construct a nice message. - message = reason + # This is purposely eagerly formatted as we want it to appear as if someone + # typed this entire message out. + message = "DEPRECATION: " + reason if replacement is not None: message += " An alternative is to {}.".format(replacement) if issue is not None: From e027cd84494b27da6e404dd03cba1a61bb4a0b60 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 6 Jul 2018 04:47:33 +0530 Subject: [PATCH 077/129] Update deprecation utilities to specify versions Also: - Remove conditional warning/error level logging - Remove now-obsolete class - Update call sites as per new signature --- src/pip/_internal/index.py | 1 + src/pip/_internal/operations/freeze.py | 1 + src/pip/_internal/req/req_install.py | 2 +- src/pip/_internal/utils/deprecation.py | 34 ++++++++++---------------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index fae26eeff..8c0ec82c0 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -215,6 +215,7 @@ class PackageFinder(object): "Dependency Links processing has been deprecated and will be " "removed in a future release.", replacement=None, + gone_in="18.2", issue=4187, ) self.dependency_links.extend(links) diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index 80fdd1044..4bbc27b0a 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -219,6 +219,7 @@ class FrozenRequirement(object): "SVN editable detection based on dependency links " "will be dropped in the future.", replacement=None, + gone_in="18.2", issue=4187, ) comments.append( diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 6a33e4e42..efe96a774 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -586,8 +586,8 @@ class InstallRequirement(object): "pyproject.toml files that do not contain the [build-system]" "table and the requires key, as specified in PEP 518.", replacement=None, + gone_in="18.2", issue=5416, - imminent=True, ) # Currently, we're isolating the build based on the presence of the diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index e8f59705c..73bf69cd8 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -6,6 +6,9 @@ from __future__ import absolute_import import logging import warnings +from pip._vendor.packaging.version import parse + +from pip import __version__ as current_version from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -16,10 +19,6 @@ class PipDeprecationWarning(Warning): pass -class PipPendingDeprecationWarning(PipDeprecationWarning): - pass - - _original_showwarning = None # type: Any @@ -34,16 +33,7 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): # We use a specially named logger which will handle all of the # deprecation messages for pip. logger = logging.getLogger("pip._internal.deprecations") - - # PipPendingDeprecationWarnings still have at least 2 - # versions to go until they are removed so they can just be - # warnings. Otherwise, they will be removed in the very next - # version of pip. We want these to be more obvious so we use the - # ERROR logging level. - if issubclass(category, PipPendingDeprecationWarning): - logger.warning(log_message) - else: - logger.error(log_message) + logger.warning(message) else: _original_showwarning( message, category, filename, lineno, file, line, @@ -61,12 +51,11 @@ def install_warning_logger(): warnings.showwarning = _showwarning -def deprecated(reason, replacement, issue=None, imminent=False): - # type: (str, Optional[str], Optional[int], bool) -> None - if imminent: - category = PipDeprecationWarning - else: - category = PipPendingDeprecationWarning +def deprecated(reason, replacement, gone_in, issue=None): + # type: (str, Optional[str], Optional[str], Optional[int]) -> None + """Helper to deprecate existing functionality. + """ + # NOTE: treat replacement, gone_in, issue as keyword only arguments. # Construct a nice message. # This is purposely eagerly formatted as we want it to appear as if someone @@ -78,4 +67,7 @@ def deprecated(reason, replacement, issue=None, imminent=False): url = "https://github.com/pypa/pip/issues/" + str(issue) message += " You can find discussion regarding this at {}.".format(url) - warnings.warn(message, category=category, stacklevel=2) + # Raise as an error if it has to be removed. + if gone_in is not None and parse(current_version) >= parse(gone_in): + raise PipDeprecationWarning(message) + warnings.warn(message, category=PipDeprecationWarning, stacklevel=2) From 3e0e45ba61955d84dd373585116e948ef86b5dc3 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 6 Jul 2018 04:42:23 +0530 Subject: [PATCH 078/129] Nicer language in line with parameter name --- src/pip/_internal/utils/deprecation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 73bf69cd8..d6a20fd57 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -62,7 +62,7 @@ def deprecated(reason, replacement, gone_in, issue=None): # typed this entire message out. message = "DEPRECATION: " + reason if replacement is not None: - message += " An alternative is to {}.".format(replacement) + message += " A possible replacement is {}.".format(replacement) if issue is not None: url = "https://github.com/pypa/pip/issues/" + str(issue) message += " You can find discussion regarding this at {}.".format(url) From a84dde598297495fe6f0f8b233b3a3761b0df7d4 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 6 Jul 2018 14:17:56 +0530 Subject: [PATCH 079/129] Update test to check newer logic --- tests/functional/test_warning.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/functional/test_warning.py b/tests/functional/test_warning.py index 20f246795..6c097c75b 100644 --- a/tests/functional/test_warning.py +++ b/tests/functional/test_warning.py @@ -1,21 +1,22 @@ +import textwrap + def test_environ(script, tmpdir): """$PYTHONWARNINGS was added in python2.7""" demo = tmpdir.join('warnings_demo.py') - demo.write(''' -from pip._internal.utils import deprecation -deprecation.install_warning_logger() + demo.write(textwrap.dedent(''' + from logging import basicConfig + from pip._internal.utils import deprecation -from logging import basicConfig -basicConfig() + deprecation.install_warning_logger() + basicConfig() -from warnings import warn -warn("deprecated!", deprecation.PipDeprecationWarning) -''') + deprecation.deprecated("deprecated!", replacement=None, gone_in=None) + ''')) result = script.run('python', demo, expect_stderr=True) - assert result.stderr == \ - 'ERROR:pip._internal.deprecations:DEPRECATION: deprecated!\n' + expected = 'WARNING:pip._internal.deprecations:DEPRECATION: deprecated!\n' + assert result.stderr == expected script.environ['PYTHONWARNINGS'] = 'ignore' result = script.run('python', demo) From 28f183e1a26f5f9091b68b32bca0d10e3564b8a5 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 15 Jul 2018 15:47:10 +0530 Subject: [PATCH 080/129] Document parameters of the deprecated helper --- src/pip/_internal/utils/deprecation.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index d6a20fd57..bd744cf2c 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -54,8 +54,24 @@ def install_warning_logger(): def deprecated(reason, replacement, gone_in, issue=None): # type: (str, Optional[str], Optional[str], Optional[int]) -> None """Helper to deprecate existing functionality. + + reason: + Textual reason shown to the user about why this functionality has + been deprecated. + replacement: + Textual suggestion shown to the user about what alternative + functionality they can use. + gone_in: + The version of pip does this functionality should get removed in. + Raises errors if pip's current version is greater than or equal to + this. + issue: + Issue number on the tracker that would serve as a useful place for + users to find related discussion and provide feedback. + + Always pass replacement, gone_in and issue as keyword arguments for clarity + at the call site. """ - # NOTE: treat replacement, gone_in, issue as keyword only arguments. # Construct a nice message. # This is purposely eagerly formatted as we want it to appear as if someone From b6dae7a5949ec67c16678f0a367cc8452f10d3b0 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 15 Jul 2018 15:52:34 +0530 Subject: [PATCH 081/129] Mention the helper along with pip's deprecation policy --- docs/development.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/development.rst b/docs/development.rst index bc1d36777..5b1ad1370 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -211,6 +211,12 @@ document existing behavior with the intention of covering that behavior with the above deprecation process are always acceptable, and will be considered on their merits. +.. note:: + + pip has a helper function for making deprecation easier for pip maintainers. + The supporting documentation can be found in the source code of + ``pip._internal.utils.deprecation.deprecated``. The function is not a part of + pip's public API. Release Process =============== From 6813caadabcb3c1827670145f75957c8168d1047 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 15 Jul 2018 18:37:24 +0530 Subject: [PATCH 082/129] Be factually correct --- src/pip/_vendor/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_vendor/README.rst b/src/pip/_vendor/README.rst index 0b0d27056..739844e2e 100644 --- a/src/pip/_vendor/README.rst +++ b/src/pip/_vendor/README.rst @@ -117,7 +117,7 @@ Debundling As mentioned in the rationale, we, the pip team, would prefer it if pip was not debundled (other than optionally ``pip/_vendor/requests/cacert.pem``) and that pip was left intact. However, if you insist on doing so, we have a -semi-supported method that we do test in our CI, but requires a bit of +semi-supported method (that we don't test in our CI) and requires a bit of extra work on your end in order to solve the problems described above. 1. Delete everything in ``pip/_vendor/`` **except** for From 5a608324fb62180c38ea6621ce8e4dfe54ded68f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 15 Jul 2018 18:37:45 +0530 Subject: [PATCH 083/129] Styling changes --- src/pip/_vendor/README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_vendor/README.rst b/src/pip/_vendor/README.rst index 739844e2e..b5c170bf4 100644 --- a/src/pip/_vendor/README.rst +++ b/src/pip/_vendor/README.rst @@ -22,9 +22,9 @@ Policy Rationale --------- -Historically pip has not had any dependencies except for setuptools itself, +Historically pip has not had any dependencies except for ``setuptools`` itself, choosing instead to implement any functionality it needed to prevent needing -a dependency. However, starting with pip 1.5 we began to replace code that was +a dependency. However, starting with pip 1.5, we began to replace code that was implemented inside of pip with reusable libraries from PyPI. This brought the typical benefits of reusing libraries instead of reinventing the wheel like higher quality and more battle tested code, centralization of bug fixes @@ -43,7 +43,7 @@ way (via ``install_requires``) for pip. These issues are: * **Making other libraries uninstallable.** One of pip's current dependencies is the ``requests`` library, for which pip requires a fairly recent version to run. - If pip dependended on ``requests`` in the traditional manner, then we'd either + If pip depended on ``requests`` in the traditional manner, then we'd either have to maintain compatibility with every ``requests`` version that has ever existed (and ever will), OR allow pip to render certain versions of ``requests`` uninstallable. (The second issue, although technically true for any Python From 490b6fd062e026c197c84632740a2f9601a71623 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 15 Jul 2018 18:38:44 +0530 Subject: [PATCH 084/129] Document new debundling suggestions --- src/pip/_vendor/README.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/pip/_vendor/README.rst b/src/pip/_vendor/README.rst index b5c170bf4..cf0ebddba 100644 --- a/src/pip/_vendor/README.rst +++ b/src/pip/_vendor/README.rst @@ -131,6 +131,14 @@ extra work on your end in order to solve the problems described above. 3. Modify ``pip/_vendor/__init__.py`` so that the ``DEBUNDLED`` variable is ``True``. -4. *(Optional)* If you've placed the wheels in a location other than +4. Upon installation, the ``INSTALLER`` file in pip's own ``dist-info`` + directory should be set to something other than ``pip``, so that pip + can detect that it wasn't installed using itself. + +5. *(optional)* If you've placed the wheels in a location other than ``pip/_vendor/``, then modify ``pip/_vendor/__init__.py`` so that the ``WHEEL_DIR`` variable points to the location you've placed them. + +6. *(optional)* Update the ``pip_version_check`` logic to use the + appropriate logic for determining the latest available version of pip and + prompt the user with the correct upgrade message. From 96f9fcc4a02a1e7521d807899a2723b31559cdde Mon Sep 17 00:00:00 2001 From: Nehal J Wani Date: Sun, 15 Jul 2018 21:24:12 +0530 Subject: [PATCH 085/129] Make pip_installed_by_pip() more generic --- news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial | 0 src/pip/_internal/utils/outdated.py | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial diff --git a/news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial b/news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/utils/outdated.py b/src/pip/_internal/utils/outdated.py index 8c3b50b98..2b3fa95e3 100644 --- a/src/pip/_internal/utils/outdated.py +++ b/src/pip/_internal/utils/outdated.py @@ -58,14 +58,14 @@ class SelfCheckState(object): separators=(",", ":")) -def pip_installed_by_pip(): - """Checks whether pip was installed by pip +def was_installed_by_pip(pkg): + """Checks whether pkg was installed by pip This is used not to display the upgrade message when pip is in fact installed by system package manager, such as dnf on Fedora. """ try: - dist = pkg_resources.get_distribution('pip') + dist = pkg_resources.get_distribution(pkg) return (dist.has_metadata('INSTALLER') and 'pip' in dist.get_metadata_lines('INSTALLER')) except pkg_resources.DistributionNotFound: @@ -125,7 +125,7 @@ def pip_version_check(session, options): # Determine if our pypi_version is older if (pip_version < remote_version and pip_version.base_version != remote_version.base_version and - pip_installed_by_pip()): + was_installed_by_pip('pip')): # Advise "python -m pip" on Windows to avoid issues # with overwriting pip.exe. if WINDOWS: From e37a17cc0987176c83d0fd5a807c61efc7575895 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Sun, 15 Jul 2018 10:39:06 -0700 Subject: [PATCH 086/129] Change VersionControl to parse the URL only once to start. --- ...18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial | 0 src/pip/_internal/vcs/__init__.py | 30 ++-- src/pip/_internal/vcs/bazaar.py | 4 +- src/pip/_internal/vcs/git.py | 6 +- src/pip/_internal/vcs/subversion.py | 68 +++++---- tests/unit/test_vcs.py | 129 ++++++++++++------ 6 files changed, 143 insertions(+), 94 deletions(-) create mode 100644 news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial diff --git a/news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial b/news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 704f1c3fa..3844adb5b 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -213,10 +213,21 @@ class VersionControl(object): """ raise NotImplementedError + def parse_netloc(self, netloc): + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Returns: (netloc, username, password). + """ + return netloc, None, None + def get_url_rev(self, url): """ - Returns the correct repository URL and revision by parsing the given - repository URL + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). """ error_message = ( "Sorry, '%s' is a malformed VCS url. " @@ -226,26 +237,27 @@ class VersionControl(object): assert '+' in url, error_message % url url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + netloc, username, password = self.parse_netloc(netloc) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev + return url, rev, (username, password) - def get_url_rev_args(self, url): + def make_rev_args(self, username, password): """ - Return the URL and RevOptions "extra arguments" to use in obtain(), - as a tuple (url, extra_args). + Return the RevOptions "extra arguments" to use in obtain(). """ - return url, [] + return [] def get_url_rev_options(self, url): """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). """ - url, rev = self.get_url_rev(url) - url, extra_args = self.get_url_rev_args(url) + url, rev, user_auth = self.get_url_rev(url) + username, password = user_auth + extra_args = self.make_rev_args(username, password) rev_options = self.make_rev_options(rev, extra_args=extra_args) return url, rev_options diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index 93b76165a..8470b7740 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -68,10 +68,10 @@ class Bazaar(VersionControl): def get_url_rev(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev(url) + url, rev, user_auth = super(Bazaar, self).get_url_rev(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev + return url, rev, user_auth def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 9ee2e012f..828675392 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -275,12 +275,12 @@ class Git(VersionControl): if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_auth = super(Git, self).get_url_rev(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_auth = super(Git, self).get_url_rev(url) - return url, rev + return url, rev, user_auth def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 70787750b..b318b0c06 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -4,11 +4,9 @@ import logging import os import re -from pip._vendor.six.moves.urllib import parse as urllib_parse - from pip._internal.index import Link from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, remove_auth_from_url, rmtree +from pip._internal.utils.misc import display_path, rmtree from pip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') @@ -132,18 +130,42 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision + def parse_netloc(self, netloc): + """ + Parse out and remove from the netloc the auth information. + """ + if '@' not in netloc: + return netloc, None, None + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (by checking the password + # attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (again by checking the + # password attribute of the return value) + username, password = auth.split(':', 1) + else: + username, password = auth, None + + return netloc, username, password + def get_url_rev(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev(url) + url, rev, user_auth = super(Subversion, self).get_url_rev(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev + return url, rev, user_auth - def get_url_rev_args(self, url): - extra_args = get_rev_options_args(url) - url = remove_auth_from_url(url) + def make_rev_args(self, username, password): + extra_args = [] + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] - return url, extra_args + return extra_args def get_url(self, location): # In cases where the source is in a subdirectory, not alongside @@ -223,32 +245,4 @@ class Subversion(VersionControl): return False -def get_rev_options_args(url): - """ - Return the extra arguments to pass to RevOptions. - """ - r = urllib_parse.urlsplit(url) - if hasattr(r, 'username'): - # >= Python-2.5 - username, password = r.username, r.password - else: - netloc = r[1] - if '@' in netloc: - auth = netloc.split('@')[0] - if ':' in auth: - username, password = auth.split(':', 1) - else: - username, password = auth, None - else: - username, password = None, None - - extra_args = [] - if username: - extra_args += ['--username', username] - if password: - extra_args += ['--password', password] - - return extra_args - - vcs.register(Subversion) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index c9bb882a9..a5a67384c 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -129,6 +129,42 @@ def test_translate_egg_surname(): assert vc.translate_egg_surname("foo/1.2.3") == "foo_1.2.3" +# The non-SVN backends all use the same parse_netloc(), so only test +# Git as a representative. +@pytest.mark.parametrize('netloc, expected', [ + # Test a basic case. + ('example.com', ('example.com', None, None)), + # Test with username and password. + ('user:pass@example.com', ('user:pass@example.com', None, None)), +]) +def test_git__parse_netloc(netloc, expected): + """ + Test VersionControl.parse_netloc(). + """ + actual = Git().parse_netloc(netloc) + assert actual == expected + + +@pytest.mark.parametrize('netloc, expected', [ + # Test a basic case. + ('example.com', ('example.com', None, None)), + # Test with username and no password. + ('user@example.com', ('example.com', 'user', None)), + # Test with username and password. + ('user:pass@example.com', ('example.com', 'user', 'pass')), + # Test the password containing an @ symbol. + ('user:pass@word@example.com', ('example.com', 'user', 'pass@word')), + # Test the password containing a : symbol. + ('user:pass:word@example.com', ('example.com', 'user', 'pass:word')), +]) +def test_subversion__parse_netloc(netloc, expected): + """ + Test Subversion.parse_netloc(). + """ + actual = Subversion().parse_netloc(netloc) + assert actual == expected + + def test_git__get_url_rev__idempotent(): """ Check that Git.get_url_rev() is idempotent for what the code calls @@ -141,8 +177,9 @@ def test_git__get_url_rev__idempotent(): result1 = vcs.get_url_rev(url) assert vcs.url == url result2 = vcs.get_url_rev(url) - assert result1 == ('git@git.example.com:MyProject', None) - assert result2 == ('git@git.example.com:MyProject', None) + expected = ('git@git.example.com:MyProject', None, (None, None)) + assert result1 == expected + assert result2 == expected def test_bazaar__get_url_rev(): @@ -171,60 +208,66 @@ def test_bazaar__get_url_rev(): ) assert http_bzr_repo.get_url_rev(http_bzr_repo.url) == ( - 'http://bzr.myproject.org/MyProject/trunk/', None, + 'http://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) assert https_bzr_repo.get_url_rev(https_bzr_repo.url) == ( - 'https://bzr.myproject.org/MyProject/trunk/', None, + 'https://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) assert ssh_bzr_repo.get_url_rev(ssh_bzr_repo.url) == ( - 'bzr+ssh://bzr.myproject.org/MyProject/trunk/', None, + 'bzr+ssh://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) assert ftp_bzr_repo.get_url_rev(ftp_bzr_repo.url) == ( - 'ftp://bzr.myproject.org/MyProject/trunk/', None, + 'ftp://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) assert sftp_bzr_repo.get_url_rev(sftp_bzr_repo.url) == ( - 'sftp://bzr.myproject.org/MyProject/trunk/', None, + 'sftp://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) assert launchpad_bzr_repo.get_url_rev(launchpad_bzr_repo.url) == ( - 'lp:MyLaunchpadProject', None, + 'lp:MyLaunchpadProject', None, (None, None), + ) + + +# The non-SVN backends all use the same make_rev_args(), so only test +# Git as a representative. +@pytest.mark.parametrize('username, password, expected', [ + (None, None, []), + ('user', None, []), + ('user', 'pass', []), +]) +def test_git__make_rev_args(username, password, expected): + """ + Test VersionControl.make_rev_args(). + """ + actual = Git().make_rev_args(username, password) + assert actual == expected + + +@pytest.mark.parametrize('username, password, expected', [ + (None, None, []), + ('user', None, ['--username', 'user']), + ('user', 'pass', ['--username', 'user', '--password', 'pass']), +]) +def test_subversion__make_rev_args(username, password, expected): + """ + Test Subversion.make_rev_args(). + """ + actual = Subversion().make_rev_args(username, password) + assert actual == expected + + +def test_subversion__get_url_rev_options(): + """ + Test Subversion.get_url_rev_options(). + """ + url = 'svn+https://user:pass@svn.example.com/MyProject@v1.0#egg=MyProject' + url, rev_options = Subversion().get_url_rev_options(url) + assert url == 'https://svn.example.com/MyProject' + assert rev_options.rev == 'v1.0' + assert rev_options.extra_args == ( + ['--username', 'user', '--password', 'pass'] ) def test_get_git_version(): git_version = Git().get_git_version() assert git_version >= parse_version('1.0.0') - - -# The non-SVN backends all have the same get_url_rev_args() implementation, -# so test with Git as a representative. -@pytest.mark.parametrize('url, expected', [ - # Test a basic case. - ('git+https://git.example.com/MyProject#egg=MyProject', - ('git+https://git.example.com/MyProject#egg=MyProject', [])), - # Test with username and password. - ('git+https://user:pass@git.example.com/MyProject#egg=MyProject', - ('git+https://user:pass@git.example.com/MyProject#egg=MyProject', [])), -]) -def test_git__get_url_rev_args(url, expected): - """ - Test Git.get_url_rev_args(). - """ - actual = Git().get_url_rev_args(url) - assert actual == expected - - -@pytest.mark.parametrize('url, expected', [ - # Test a basic case. - ('svn+https://svn.example.com/MyProject#egg=MyProject', - ('svn+https://svn.example.com/MyProject#egg=MyProject', [])), - # Test with username and password. - ('svn+https://user:pass@svn.example.com/MyProject#egg=MyProject', - ('svn+https://svn.example.com/MyProject#egg=MyProject', - ['--username', 'user', '--password', 'pass'])), -]) -def test_subversion__get_url_rev_args(url, expected): - """ - Test Subversion.get_url_rev_args(). - """ - actual = Subversion().get_url_rev_args(url) - assert actual == expected From 901578419da3835c6e91e64cd4cbc6acb1edb5f7 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 17 Jul 2018 12:46:37 +0530 Subject: [PATCH 087/129] Fix a merging artifact that broke the linter --- src/pip/_internal/utils/logging.py | 90 +++++++++++++++--------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index c2a2f9d81..66c1d39d7 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -173,53 +173,53 @@ def setup_logging(verbosity, no_color, user_log_file): } logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": False, - "filters": { - "exclude_warnings": { - "()": "pip._internal.utils.logging.MaxLevelFilter", - "level": logging.WARNING, - }, + "version": 1, + "disable_existing_loggers": False, + "filters": { + "exclude_warnings": { + "()": "pip._internal.utils.logging.MaxLevelFilter", + "level": logging.WARNING, }, - "formatters": { - "indent": { - "()": IndentingFormatter, - "format": "%(message)s", - }, + }, + "formatters": { + "indent": { + "()": IndentingFormatter, + "format": "%(message)s", }, - "handlers": { - "console": { - "level": level, - "class": handler_classes["stream"], - "no_color": no_color, - "stream": log_streams["stdout"], - "filters": ["exclude_warnings"], - "formatter": "indent", - }, - "console_errors": { - "level": "WARNING", - "class": handler_classes["stream"], - "no_color": no_color, - "stream": log_streams["stderr"], - "formatter": "indent", - }, - "user_log": { - "level": "DEBUG", - "class": handler_classes["file"], - "filename": additional_log_file, - "delay": True, - "formatter": "indent", - }, + }, + "handlers": { + "console": { + "level": level, + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stdout"], + "filters": ["exclude_warnings"], + "formatter": "indent", }, - "root": { - "level": root_level, - "handlers": ["console", "console_errors"] + ( - ["user_log"] if include_user_log else [] - ), + "console_errors": { + "level": "WARNING", + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stderr"], + "formatter": "indent", }, - "loggers": { - "pip._vendor": { - "level": vendored_log_level - } + "user_log": { + "level": "DEBUG", + "class": handler_classes["file"], + "filename": additional_log_file, + "delay": True, + "formatter": "indent", }, - }) + }, + "root": { + "level": root_level, + "handlers": ["console", "console_errors"] + ( + ["user_log"] if include_user_log else [] + ), + }, + "loggers": { + "pip._vendor": { + "level": vendored_log_level + } + }, + }) From 3c073f92f40fc41b9c71e13d48c281248f941d88 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 17 Jul 2018 13:59:41 +0530 Subject: [PATCH 088/129] Add the deprecation policy changes from fde9eb66f072adad73491b00beafddd6 --- docs/development/release-process.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/development/release-process.rst b/docs/development/release-process.rst index 041943a12..13c144798 100644 --- a/docs/development/release-process.rst +++ b/docs/development/release-process.rst @@ -34,19 +34,19 @@ reworked and resubmitted for the next release. Deprecation Policy ================== -Any change to pip that removes or significantly alters user-visible behaviour +Any change to pip that removes or significantly alters user-visible behavior that is described in the pip documentation will be deprecated for a minimum of -one released version before the change occurs. Deprecation will take the form of -a warning being issued by pip when the feature is used. Longer deprecation -periods, or deprecation warnings for behaviour changes that would not normally -be covered by this policy, are also possible depending on circumstances, but -this is at the discretion of the pip developers. +6 months before the change occurs. Deprecation will take the form of a warning +being issued by pip when the feature is used. Longer deprecation periods, or +deprecation warnings for behavior changes that would not normally be covered by +this policy, are also possible depending on circumstances, but this is at the +discretion of the pip developers. Note that the documentation is the sole reference for what counts as agreed -behaviour. If something isn't explicitly mentioned in the documentation, it can +behavior. If something isn't explicitly mentioned in the documentation, it can be changed without warning, or any deprecation period, in a pip release. However, we are aware that the documentation isn't always complete - PRs that -document existing behaviour with the intention of covering that behaviour with +document existing behavior with the intention of covering that behavior with the above deprecation process are always acceptable, and will be considered on their merits. From 22acd1fed1f126bdccfa0a11acc0b5aaeb9874f6 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Mon, 16 Jul 2018 20:12:44 -0700 Subject: [PATCH 089/129] Break out VersionControl.is_repository_directory(). --- news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial | 0 src/pip/_internal/vcs/__init__.py | 8 +++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial diff --git a/news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial b/news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 704f1c3fa..4a392b891 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -200,6 +200,12 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive + def is_repository_directory(self, path): + """ + Return whether a directory path is a repository directory. + """ + return os.path.exists(os.path.join(path, self.dirname)) + # See issue #1083 for why this method was introduced: # https://github.com/pypa/pip/issues/1083 def translate_egg_surname(self, surname): @@ -325,7 +331,7 @@ class VersionControl(object): return rev_display = rev_options.to_display() - if os.path.exists(os.path.join(dest, self.dirname)): + if self.is_repository_directory(dest): existing_url = self.get_url(dest) if self.compare_urls(existing_url, url): logger.debug( From 977d0740fbe773a3691b3aeccf29464ffa67ff2b Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Tue, 17 Jul 2018 08:31:19 -0700 Subject: [PATCH 090/129] Add more explanation to parse_netloc()'s docstrings. This addresses @pradyunsg's review comments. --- src/pip/_internal/vcs/__init__.py | 5 +++++ src/pip/_internal/vcs/subversion.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 3844adb5b..ebf9890f6 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -218,6 +218,11 @@ class VersionControl(object): Parse the repository URL's netloc, and return the new netloc to use along with auth information. + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + Returns: (netloc, username, password). """ return netloc, None, None diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index b318b0c06..acbace0de 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -133,6 +133,9 @@ class Subversion(VersionControl): def parse_netloc(self, netloc): """ Parse out and remove from the netloc the auth information. + + This allows the auth information to be provided via the --username + and --password options instead of via the URL. """ if '@' not in netloc: return netloc, None, None From f62440950be90c862a96027db94ef0e60a303013 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Tue, 17 Jul 2018 13:05:26 -0700 Subject: [PATCH 091/129] Use is_repository_directory() inside VersionControl.controls_location(). --- src/pip/_internal/vcs/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 4a392b891..66a3c93cb 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -204,6 +204,8 @@ class VersionControl(object): """ Return whether a directory path is a repository directory. """ + logger.debug('Checking in %s for %s (%s)...', + path, self.dirname, self.name) return os.path.exists(os.path.join(path, self.dirname)) # See issue #1083 for why this method was introduced: @@ -472,10 +474,8 @@ class VersionControl(object): It is meant to be overridden to implement smarter detection mechanisms for specific vcs. """ - logger.debug('Checking in %s for %s (%s)...', - location, cls.dirname, cls.name) - path = os.path.join(location, cls.dirname) - return os.path.exists(path) + vcs = cls() + return vcs.is_repository_directory(location) def get_src_requirement(dist, location): From 814b4bd52a40ae250c32119187b3f5c2fca4d309 Mon Sep 17 00:00:00 2001 From: barneygale Date: Wed, 18 Jul 2018 01:38:12 +0100 Subject: [PATCH 092/129] Remove vestigial `VersionControl.get_info()` method. --- src/pip/_internal/vcs/__init__.py | 10 +--------- src/pip/_internal/vcs/subversion.py | 28 ---------------------------- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 704f1c3fa..78a13eb9a 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -250,14 +250,6 @@ class VersionControl(object): return url, rev_options - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - def normalize_url(self, url): """ Normalize a URL for comparison by unquoting it and removing any @@ -422,7 +414,7 @@ class VersionControl(object): """ Return the url used at location - This is used in get_info() and obtain(). + This is used in obtain(). """ raise NotImplementedError diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 70787750b..ec2526475 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -31,34 +31,6 @@ class Subversion(VersionControl): def get_base_rev_args(self, rev): return ['-r', rev] - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - output = self.run_command( - ['info', location], - show_stdout=False, - extra_environ={'LANG': 'C'}, - ) - match = _svn_url_re.search(output) - if not match: - logger.warning( - 'Cannot determine URL of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warning( - 'Cannot determine revision of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return url, None - return url, match.group(1) - def export(self, location): """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(self.url) From c6a18767dd2b0cabe2ef0176ce96b637cdbb8f36 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Tue, 17 Jul 2018 20:53:42 -0700 Subject: [PATCH 093/129] Make is_repository_directory() a class method. This addresses a review suggestion of @xavfernandez. --- src/pip/_internal/vcs/__init__.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 66a3c93cb..e8b4deb1d 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -200,14 +200,6 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - def is_repository_directory(self, path): - """ - Return whether a directory path is a repository directory. - """ - logger.debug('Checking in %s for %s (%s)...', - path, self.dirname, self.name) - return os.path.exists(os.path.join(path, self.dirname)) - # See issue #1083 for why this method was introduced: # https://github.com/pypa/pip/issues/1083 def translate_egg_surname(self, surname): @@ -467,15 +459,26 @@ class VersionControl(object): else: raise # re-raise exception if a different error occurred + @classmethod + def is_repository_directory(cls, path): + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + @classmethod def controls_location(cls, location): """ Check if a location is controlled by the vcs. It is meant to be overridden to implement smarter detection mechanisms for specific vcs. + + This can do more than is_repository_directory() alone. For example, + the Git override checks that Git is actually available. """ - vcs = cls() - return vcs.is_repository_directory(location) + return cls.is_repository_directory(location) def get_src_requirement(dist, location): From c6e65a0e402e21e8f1b0bed4513e48a1c0d67384 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Thu, 19 Jul 2018 09:13:05 -0700 Subject: [PATCH 094/129] Rename methods to get_netloc_and_auth() and get_url_rev_and_auth(). This addresses review comments by @xavfernandez. --- src/pip/_internal/vcs/__init__.py | 16 +++++----- src/pip/_internal/vcs/bazaar.py | 6 ++-- src/pip/_internal/vcs/git.py | 8 ++--- src/pip/_internal/vcs/subversion.py | 16 +++++----- tests/unit/test_vcs.py | 48 ++++++++++++++--------------- 5 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index ebf9890f6..9cb490f42 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -213,7 +213,7 @@ class VersionControl(object): """ raise NotImplementedError - def parse_netloc(self, netloc): + def get_netloc_and_auth(self, netloc): """ Parse the repository URL's netloc, and return the new netloc to use along with auth information. @@ -223,11 +223,11 @@ class VersionControl(object): instead of through the URL. For other subclasses like Git without such an option, auth information must stay in the URL. - Returns: (netloc, username, password). + Returns: (netloc, (username, password)). """ - return netloc, None, None + return netloc, (None, None) - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): """ Parse the repository URL to use, and return the URL, revision, and auth info to use. @@ -242,12 +242,12 @@ class VersionControl(object): assert '+' in url, error_message % url url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) - netloc, username, password = self.parse_netloc(netloc) + netloc, user_pass = self.get_netloc_and_auth(netloc) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev, (username, password) + return url, rev, user_pass def make_rev_args(self, username, password): """ @@ -260,8 +260,8 @@ class VersionControl(object): Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). """ - url, rev, user_auth = self.get_url_rev(url) - username, password = user_auth + url, rev, user_pass = self.get_url_rev_and_auth(url) + username, password = user_pass extra_args = self.make_rev_args(username, password) rev_options = self.make_rev_options(rev, extra_args=extra_args) diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index 8470b7740..d83812000 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -66,12 +66,12 @@ class Bazaar(VersionControl): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev, user_auth = super(Bazaar, self).get_url_rev(url) + url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev, user_auth + return url, rev, user_pass def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 828675392..5376caa32 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -265,7 +265,7 @@ class Git(VersionControl): req += '&subdirectory=' + subdirectory return req - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't @@ -275,12 +275,12 @@ class Git(VersionControl): if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev, user_auth = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev, user_auth = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) - return url, rev, user_auth + return url, rev, user_pass def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index acbace0de..b4773ed69 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -130,7 +130,7 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def parse_netloc(self, netloc): + def get_netloc_and_auth(self, netloc): """ Parse out and remove from the netloc the auth information. @@ -138,7 +138,7 @@ class Subversion(VersionControl): and --password options instead of via the URL. """ if '@' not in netloc: - return netloc, None, None + return netloc, (None, None) # Split from the right because that's how urllib.parse.urlsplit() # behaves if more than one @ is present (by checking the password @@ -148,18 +148,18 @@ class Subversion(VersionControl): # Split from the left because that's how urllib.parse.urlsplit() # behaves if more than one : is present (again by checking the # password attribute of the return value) - username, password = auth.split(':', 1) + user_pass = tuple(auth.split(':', 1)) else: - username, password = auth, None + user_pass = auth, None - return netloc, username, password + return netloc, user_pass - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev, user_auth = super(Subversion, self).get_url_rev(url) + url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev, user_auth + return url, rev, user_pass def make_rev_args(self, username, password): extra_args = [] diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index a5a67384c..939dcc3cb 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -129,60 +129,60 @@ def test_translate_egg_surname(): assert vc.translate_egg_surname("foo/1.2.3") == "foo_1.2.3" -# The non-SVN backends all use the same parse_netloc(), so only test +# The non-SVN backends all use the same get_netloc_and_auth(), so only test # Git as a representative. @pytest.mark.parametrize('netloc, expected', [ # Test a basic case. - ('example.com', ('example.com', None, None)), + ('example.com', ('example.com', (None, None))), # Test with username and password. - ('user:pass@example.com', ('user:pass@example.com', None, None)), + ('user:pass@example.com', ('user:pass@example.com', (None, None))), ]) -def test_git__parse_netloc(netloc, expected): +def test_git__get_netloc_and_auth(netloc, expected): """ - Test VersionControl.parse_netloc(). + Test VersionControl.get_netloc_and_auth(). """ - actual = Git().parse_netloc(netloc) + actual = Git().get_netloc_and_auth(netloc) assert actual == expected @pytest.mark.parametrize('netloc, expected', [ # Test a basic case. - ('example.com', ('example.com', None, None)), + ('example.com', ('example.com', (None, None))), # Test with username and no password. - ('user@example.com', ('example.com', 'user', None)), + ('user@example.com', ('example.com', ('user', None))), # Test with username and password. - ('user:pass@example.com', ('example.com', 'user', 'pass')), + ('user:pass@example.com', ('example.com', ('user', 'pass'))), # Test the password containing an @ symbol. - ('user:pass@word@example.com', ('example.com', 'user', 'pass@word')), + ('user:pass@word@example.com', ('example.com', ('user', 'pass@word'))), # Test the password containing a : symbol. - ('user:pass:word@example.com', ('example.com', 'user', 'pass:word')), + ('user:pass:word@example.com', ('example.com', ('user', 'pass:word'))), ]) -def test_subversion__parse_netloc(netloc, expected): +def test_subversion__get_netloc_and_auth(netloc, expected): """ - Test Subversion.parse_netloc(). + Test Subversion.get_netloc_and_auth(). """ - actual = Subversion().parse_netloc(netloc) + actual = Subversion().get_netloc_and_auth(netloc) assert actual == expected def test_git__get_url_rev__idempotent(): """ - Check that Git.get_url_rev() is idempotent for what the code calls + Check that Git.get_url_rev_and_auth() is idempotent for what the code calls "stub URLs" (i.e. URLs that don't contain "://"). Also check that it doesn't change self.url. """ url = 'git+git@git.example.com:MyProject#egg=MyProject' vcs = Git(url) - result1 = vcs.get_url_rev(url) + result1 = vcs.get_url_rev_and_auth(url) assert vcs.url == url - result2 = vcs.get_url_rev(url) + result2 = vcs.get_url_rev_and_auth(url) expected = ('git@git.example.com:MyProject', None, (None, None)) assert result1 == expected assert result2 == expected -def test_bazaar__get_url_rev(): +def test_bazaar__get_url_rev_and_auth(): """ Test bzr url support. @@ -207,22 +207,22 @@ def test_bazaar__get_url_rev(): url='bzr+lp:MyLaunchpadProject#egg=MyLaunchpadProject' ) - assert http_bzr_repo.get_url_rev(http_bzr_repo.url) == ( + assert http_bzr_repo.get_url_rev_and_auth(http_bzr_repo.url) == ( 'http://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert https_bzr_repo.get_url_rev(https_bzr_repo.url) == ( + assert https_bzr_repo.get_url_rev_and_auth(https_bzr_repo.url) == ( 'https://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert ssh_bzr_repo.get_url_rev(ssh_bzr_repo.url) == ( + assert ssh_bzr_repo.get_url_rev_and_auth(ssh_bzr_repo.url) == ( 'bzr+ssh://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert ftp_bzr_repo.get_url_rev(ftp_bzr_repo.url) == ( + assert ftp_bzr_repo.get_url_rev_and_auth(ftp_bzr_repo.url) == ( 'ftp://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert sftp_bzr_repo.get_url_rev(sftp_bzr_repo.url) == ( + assert sftp_bzr_repo.get_url_rev_and_auth(sftp_bzr_repo.url) == ( 'sftp://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert launchpad_bzr_repo.get_url_rev(launchpad_bzr_repo.url) == ( + assert launchpad_bzr_repo.get_url_rev_and_auth(launchpad_bzr_repo.url) == ( 'lp:MyLaunchpadProject', None, (None, None), ) From 6acdceaf395ec1e03cfc9aa337e191ee3c029a46 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 20 Jul 2018 10:07:46 +0530 Subject: [PATCH 095/129] :art: --- src/pip/_internal/req/req_tracker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py index 4869bb3a9..0a86f4cd3 100644 --- a/src/pip/_internal/req/req_tracker.py +++ b/src/pip/_internal/req/req_tracker.py @@ -8,7 +8,6 @@ import os from pip._internal.utils.temp_dir import TempDirectory - logger = logging.getLogger(__name__) From b10ae89ebf95643bcea0a4c46b6cc94263964794 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 20 Jul 2018 10:30:21 +0530 Subject: [PATCH 096/129] Add periods to all NEWS fragments --- news/3651.removal | 2 +- news/3654.removal | 2 +- news/4174.removal | 2 +- news/5127.bugfix | 2 +- news/5196.bugfix | 2 +- news/5249.bugfix | 2 +- news/5293.bugfix | 2 +- news/5311.bugfix | 2 +- news/5312.bugfix | 2 +- news/5319.feature | 2 +- news/cachecontrol.vendor | 2 +- news/certifi.vendor | 2 +- news/distro.vendor | 2 +- news/idna.vendor | 2 +- news/ipaddress.vendor | 2 +- news/pkg_resources.vendor | 2 +- news/progress.vendor | 2 +- news/pytoml.vendor | 2 +- news/requests.vendor | 2 +- news/urllib3.vendor | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/news/3651.removal b/news/3651.removal index d7cbb0362..ed69b2c86 100644 --- a/news/3651.removal +++ b/news/3651.removal @@ -1 +1 @@ -Remove the legacy format from pip list +Remove the legacy format from pip list. diff --git a/news/3654.removal b/news/3654.removal index d7cbb0362..ed69b2c86 100644 --- a/news/3654.removal +++ b/news/3654.removal @@ -1 +1 @@ -Remove the legacy format from pip list +Remove the legacy format from pip list. diff --git a/news/4174.removal b/news/4174.removal index a1362cf71..30fd3f0e9 100644 --- a/news/4174.removal +++ b/news/4174.removal @@ -1 +1 @@ -Remove support for cleaning up #egg fragment postfixes +Remove support for cleaning up #egg fragment postfixes. diff --git a/news/5127.bugfix b/news/5127.bugfix index 4c03f450a..8a7880a37 100644 --- a/news/5127.bugfix +++ b/news/5127.bugfix @@ -1 +1 @@ -Speed up printing of newly installed package versions +Speed up printing of newly installed package versions. diff --git a/news/5196.bugfix b/news/5196.bugfix index 02c31e25f..405cc2a46 100644 --- a/news/5196.bugfix +++ b/news/5196.bugfix @@ -1,4 +1,4 @@ -Restrict install time dependency warnings to directly-dependant packages +Restrict install time dependency warnings to directly-dependant packages. Warning about the entire package set has resulted in users getting confused as to why pip is printing these warnings. diff --git a/news/5249.bugfix b/news/5249.bugfix index 32f8a9721..7b6ea48da 100644 --- a/news/5249.bugfix +++ b/news/5249.bugfix @@ -1 +1 @@ -Remove username/password from log message when using index with basic auth +Remove username/password from log message when using index with basic auth. diff --git a/news/5293.bugfix b/news/5293.bugfix index dc1379ba1..bff0d8daa 100644 --- a/news/5293.bugfix +++ b/news/5293.bugfix @@ -1 +1 @@ -Remove trailing os.sep from PATH directories to avoid false negatives \ No newline at end of file +Remove trailing os.sep from PATH directories to avoid false negatives. diff --git a/news/5311.bugfix b/news/5311.bugfix index b9f3d74b1..dfa4663f1 100644 --- a/news/5311.bugfix +++ b/news/5311.bugfix @@ -1 +1 @@ -Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check +Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. diff --git a/news/5312.bugfix b/news/5312.bugfix index b9f3d74b1..dfa4663f1 100644 --- a/news/5312.bugfix +++ b/news/5312.bugfix @@ -1 +1 @@ -Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check +Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. diff --git a/news/5319.feature b/news/5319.feature index 25041dcdb..08fd1a7d4 100644 --- a/news/5319.feature +++ b/news/5319.feature @@ -1 +1 @@ -Improve status message when upgrade is skipped due to only-if-needed strategy +Improve status message when upgrade is skipped due to only-if-needed strategy. diff --git a/news/cachecontrol.vendor b/news/cachecontrol.vendor index 1affedb83..a4c1b5ff5 100644 --- a/news/cachecontrol.vendor +++ b/news/cachecontrol.vendor @@ -1 +1 @@ -Update CacheControl to 0.12.5 +Update CacheControl to 0.12.5. diff --git a/news/certifi.vendor b/news/certifi.vendor index e03c3ae1c..751544d7a 100644 --- a/news/certifi.vendor +++ b/news/certifi.vendor @@ -1 +1 @@ -Update certifi to 2018.4.16 +Update certifi to 2018.4.16. diff --git a/news/distro.vendor b/news/distro.vendor index a21167d38..347a6f6ab 100644 --- a/news/distro.vendor +++ b/news/distro.vendor @@ -1 +1 @@ -Update distro to 1.3.0 +Update distro to 1.3.0. diff --git a/news/idna.vendor b/news/idna.vendor index e0491b2d6..c959331aa 100644 --- a/news/idna.vendor +++ b/news/idna.vendor @@ -1 +1 @@ -Update idna to 2.7 +Update idna to 2.7. diff --git a/news/ipaddress.vendor b/news/ipaddress.vendor index 4df2e8374..091d12ad5 100644 --- a/news/ipaddress.vendor +++ b/news/ipaddress.vendor @@ -1 +1 @@ -Update ipaddress to 1.0.22 +Update ipaddress to 1.0.22. diff --git a/news/pkg_resources.vendor b/news/pkg_resources.vendor index ebde0d1dd..f04b46c47 100644 --- a/news/pkg_resources.vendor +++ b/news/pkg_resources.vendor @@ -1 +1 @@ -Update pkg_resources to 39.2.0 (via setuptools) +Update pkg_resources to 39.2.0 (via setuptools). diff --git a/news/progress.vendor b/news/progress.vendor index 583728427..513f57ef8 100644 --- a/news/progress.vendor +++ b/news/progress.vendor @@ -1 +1 @@ -Update progress to 1.4 +Update progress to 1.4. diff --git a/news/pytoml.vendor b/news/pytoml.vendor index 122491173..f7c6faaa2 100644 --- a/news/pytoml.vendor +++ b/news/pytoml.vendor @@ -1 +1 @@ -Update pytoml to 0.1.16 +Update pytoml to 0.1.16. diff --git a/news/requests.vendor b/news/requests.vendor index 1144da4c6..b6a9c0bad 100644 --- a/news/requests.vendor +++ b/news/requests.vendor @@ -1 +1 @@ -Update requests to 2.19.1 +Update requests to 2.19.1. diff --git a/news/urllib3.vendor b/news/urllib3.vendor index d854f4703..e89ae636c 100644 --- a/news/urllib3.vendor +++ b/news/urllib3.vendor @@ -1 +1 @@ -Update urllib3 to 1.23 +Update urllib3 to 1.23. From c682d98a9879db89c810510241d9bcad3979d907 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 20 Jul 2018 10:30:40 +0530 Subject: [PATCH 097/129] Remove not useful NEWS fragments --- news/5325.bugfix | 1 - news/5347.bugfix | 1 - 2 files changed, 2 deletions(-) delete mode 100644 news/5325.bugfix delete mode 100644 news/5347.bugfix diff --git a/news/5325.bugfix b/news/5325.bugfix deleted file mode 100644 index 0d79eb054..000000000 --- a/news/5325.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add test for PR 5293: Remove trailing os.sep to avoid false negatives diff --git a/news/5347.bugfix b/news/5347.bugfix deleted file mode 100644 index 195710772..000000000 --- a/news/5347.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the revendoring script no to rewrite unrelated import that starts suspicious. \ No newline at end of file From 7071efe2a6209cf3e423d884d99fc70d05d1c9f1 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 20 Jul 2018 10:31:20 +0530 Subject: [PATCH 098/129] Rewrite certain NEWS fragments --- news/3905.bugfix | 2 +- news/512.doc | 2 +- news/5261.doc | 2 +- news/5381.bugfix | 2 +- news/5416.bugfix | 2 +- news/5457.bugfix | 2 +- news/5512.bugfix | 2 +- news/5574.doc | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/news/3905.bugfix b/news/3905.bugfix index a719866fc..37fcaf273 100644 --- a/news/3905.bugfix +++ b/news/3905.bugfix @@ -1 +1 @@ -Adjust path to selfcheck.json - remove virtualenv specific path and honor cache-dir in pip.conf +Update pip's self-check logic to not use a virtualenv specific file and honor cache-dir. diff --git a/news/512.doc b/news/512.doc index e0382fbe4..6c35db573 100644 --- a/news/512.doc +++ b/news/512.doc @@ -1 +1 @@ -Adding documentation for making pip use a proxy server +Document how to use pip with a proxy server. diff --git a/news/5261.doc b/news/5261.doc index 61b2918db..01f48a36a 100644 --- a/news/5261.doc +++ b/news/5261.doc @@ -1 +1 @@ -Clarify that the output of pip show is in RFC-compliant mail header format for people who want to parse the ouput. +Document that the output of pip show is in RFC-compliant mail header format. diff --git a/news/5381.bugfix b/news/5381.bugfix index 899fd4908..c50b2212a 100644 --- a/news/5381.bugfix +++ b/news/5381.bugfix @@ -1 +1 @@ -Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. +Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. diff --git a/news/5416.bugfix b/news/5416.bugfix index 2f7707e0c..f5aff2abe 100644 --- a/news/5416.bugfix +++ b/news/5416.bugfix @@ -1 +1 @@ -Start refusing to install packages with non PEP-518 compliant pyproject.toml +Warn when a project's pyproject.toml file does not contain ``build-system.requires``. diff --git a/news/5457.bugfix b/news/5457.bugfix index 02c31e25f..405cc2a46 100644 --- a/news/5457.bugfix +++ b/news/5457.bugfix @@ -1,4 +1,4 @@ -Restrict install time dependency warnings to directly-dependant packages +Restrict install time dependency warnings to directly-dependant packages. Warning about the entire package set has resulted in users getting confused as to why pip is printing these warnings. diff --git a/news/5512.bugfix b/news/5512.bugfix index 2f7707e0c..f5aff2abe 100644 --- a/news/5512.bugfix +++ b/news/5512.bugfix @@ -1 +1 @@ -Start refusing to install packages with non PEP-518 compliant pyproject.toml +Warn when a project's pyproject.toml file does not contain ``build-system.requires``. diff --git a/news/5574.doc b/news/5574.doc index e0382fbe4..6c35db573 100644 --- a/news/5574.doc +++ b/news/5574.doc @@ -1 +1 @@ -Adding documentation for making pip use a proxy server +Document how to use pip with a proxy server. From 05eacbcf63745deef31433a3d92cc4bbf6919bbb Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 20 Jul 2018 22:48:12 +0100 Subject: [PATCH 099/129] Address review feedback --- src/pip/_internal/vcs/__init__.py | 2 -- src/pip/_internal/vcs/subversion.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 78a13eb9a..082c92cc4 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -413,8 +413,6 @@ class VersionControl(object): def get_url(self, location): """ Return the url used at location - - This is used in obtain(). """ raise NotImplementedError diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index ec2526475..545825dfb 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -13,8 +13,6 @@ from pip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'(.*)') From c18f19a6a400bcb84fe3b75b5edf016fea44d308 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 12:58:27 +0530 Subject: [PATCH 100/129] Don't warn if pyproject.toml does not contain build-system.requires --- news/5416.bugfix | 1 - news/5512.bugfix | 1 - src/pip/_internal/req/req_install.py | 29 ++++++++-------------------- tests/functional/test_install.py | 6 +++--- 4 files changed, 11 insertions(+), 26 deletions(-) delete mode 100644 news/5416.bugfix delete mode 100644 news/5512.bugfix diff --git a/news/5416.bugfix b/news/5416.bugfix deleted file mode 100644 index f5aff2abe..000000000 --- a/news/5416.bugfix +++ /dev/null @@ -1 +0,0 @@ -Warn when a project's pyproject.toml file does not contain ``build-system.requires``. diff --git a/news/5512.bugfix b/news/5512.bugfix deleted file mode 100644 index f5aff2abe..000000000 --- a/news/5512.bugfix +++ /dev/null @@ -1 +0,0 @@ -Warn when a project's pyproject.toml file does not contain ``build-system.requires``. diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index efe96a774..b9b6ab3a0 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -32,7 +32,6 @@ from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) from pip._internal.req.req_uninstall import UninstallPathSet -from pip._internal.utils.deprecation import deprecated from pip._internal.utils.hashes import Hashes from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( @@ -574,26 +573,10 @@ class InstallRequirement(object): # Extract the build requirements requires = pp_toml.get("build-system", {}).get("requires", None) - template = ( - "%s does not comply with PEP 518 since pyproject.toml " - "does not contain a valid '[build-system].requires' key: %s" - ) - if requires is None: - logging.warn(template, self, "it is missing.") - deprecated( - "Future versions of pip may reject packages with " - "pyproject.toml files that do not contain the [build-system]" - "table and the requires key, as specified in PEP 518.", - replacement=None, - gone_in="18.2", - issue=5416, - ) - - # Currently, we're isolating the build based on the presence of the - # pyproject.toml file. If the user doesn't specify - # build-system.requires, assume they intended to use setuptools and - # wheel for now. + # We isolate on the presence of the pyproject.toml file. + # If build-system.requires is not specified, treat it as if it was + # specified as ["setuptools", "wheel"] return ["setuptools", "wheel"] else: # Error out if it's not a list of strings @@ -601,8 +584,12 @@ class InstallRequirement(object): isinstance(req, six.string_types) for req in requires ) if not is_list_of_str: + template = ( + "{} does not comply with PEP 518 since pyproject.toml " + "does not contain a valid build-system.requires key: {}" + ) raise InstallationError( - template % (self, "it is not a list of strings.") + template.format(self, "it is not a list of strings.") ) # If control flow reaches here, we're good to go. diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 201d9a6a9..9ccaf9e8c 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -47,14 +47,14 @@ def test_pep518_refuses_invalid_requires(script, data, common_wheels): assert "does not comply with PEP 518" in result.stderr -def test_pep518_allows_but_warns_missing_requires(script, data, common_wheels): +def test_pep518_allows_missing_requires(script, data, common_wheels): result = script.pip( 'install', '-f', common_wheels, data.src.join("pep518_missing_requires"), expect_stderr=True ) - assert "does not comply with PEP 518" in result.stderr - assert "DEPRECATION" in result.stderr + # Make sure we don't warn when this occurs. + assert "does not comply with PEP 518" not in result.stderr # We want it to go through isolation for now. assert "Installing build dependencies" in result.stdout, result.stdout From 9a076e73abd1eb30f10d6e095f1ae310ebdd8d3e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 13:08:42 +0530 Subject: [PATCH 101/129] Add test for rejecting pyproject.toml with empty build-system table --- tests/data/src/pep518_invalid_build_system/MANIFEST.in | 1 + tests/data/src/pep518_invalid_build_system/pep518.py | 1 + .../src/pep518_invalid_build_system/pyproject.toml | 2 ++ tests/data/src/pep518_invalid_build_system/setup.py | 8 ++++++++ tests/functional/test_install.py | 10 ++++++++++ 5 files changed, 22 insertions(+) create mode 100644 tests/data/src/pep518_invalid_build_system/MANIFEST.in create mode 100644 tests/data/src/pep518_invalid_build_system/pep518.py create mode 100644 tests/data/src/pep518_invalid_build_system/pyproject.toml create mode 100644 tests/data/src/pep518_invalid_build_system/setup.py diff --git a/tests/data/src/pep518_invalid_build_system/MANIFEST.in b/tests/data/src/pep518_invalid_build_system/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/tests/data/src/pep518_invalid_build_system/pep518.py b/tests/data/src/pep518_invalid_build_system/pep518.py new file mode 100644 index 000000000..7986d1137 --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/pep518.py @@ -0,0 +1 @@ +#dummy diff --git a/tests/data/src/pep518_invalid_build_system/pyproject.toml b/tests/data/src/pep518_invalid_build_system/pyproject.toml new file mode 100644 index 000000000..86fcebfa8 --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +# This table is intentionally empty. diff --git a/tests/data/src/pep518_invalid_build_system/setup.py b/tests/data/src/pep518_invalid_build_system/setup.py new file mode 100644 index 000000000..ba23cf24a --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/setup.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +from setuptools import setup + +setup( + name='pep518_invalid_build_system', + version='1.0.0', + py_modules=['pep518'], +) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 9ccaf9e8c..5767cdd37 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -47,6 +47,16 @@ def test_pep518_refuses_invalid_requires(script, data, common_wheels): assert "does not comply with PEP 518" in result.stderr +def test_pep518_refuses_invalid_build_system(script, data, common_wheels): + result = script.pip( + 'install', '-f', common_wheels, + data.src.join("pep518_invalid_build_system"), + expect_error=True + ) + assert result.returncode == 1 + assert "does not comply with PEP 518" in result.stderr + + def test_pep518_allows_missing_requires(script, data, common_wheels): result = script.pip( 'install', '-f', common_wheels, From 7fe759bdb1bf46b42414aaaa085ad0e7f6db5270 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 13:09:54 +0530 Subject: [PATCH 102/129] Cleanup existing PEP 518 logic --- src/pip/_internal/req/req_install.py | 43 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index b9b6ab3a0..cd64ef565 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -564,35 +564,29 @@ class InstallRequirement(object): specified as per PEP 518 within the package. If `pyproject.toml` is not present, returns None to signify not using the same. """ + # If pyproject.toml does not exist, don't do anything. if not os.path.isfile(self.pyproject_toml): return None + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + with io.open(self.pyproject_toml, encoding="utf-8") as f: pp_toml = pytoml.load(f) - # Extract the build requirements - requires = pp_toml.get("build-system", {}).get("requires", None) - - if requires is None: - # We isolate on the presence of the pyproject.toml file. - # If build-system.requires is not specified, treat it as if it was - # specified as ["setuptools", "wheel"] + # If there is no build-system table, just use setuptools and wheel. + if "build-system" not in pp_toml: return ["setuptools", "wheel"] - else: - # Error out if it's not a list of strings - is_list_of_str = isinstance(requires, list) and all( - isinstance(req, six.string_types) for req in requires - ) - if not is_list_of_str: - template = ( - "{} does not comply with PEP 518 since pyproject.toml " - "does not contain a valid build-system.requires key: {}" - ) - raise InstallationError( - template.format(self, "it is not a list of strings.") - ) - # If control flow reaches here, we're good to go. + # Error out if it's not a list of strings + requires = build_system.get("requires", []) + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + self, "'build-system.requires' is not a list of strings." + )) + return requires def run_egg_info(self): @@ -1128,3 +1122,10 @@ def deduce_helpful_msg(req): else: msg += " File '%s' does not exist." % (req) return msg + + +def _is_list_of_str(obj): + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) From 4d772e6cf6eaa1ffcdb84f6c5fae9d933e1bb08d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 13:10:42 +0530 Subject: [PATCH 103/129] Start rejecting pyproject.toml files with an empty build-system table --- src/pip/_internal/req/req_install.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index cd64ef565..77526283d 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -580,8 +580,18 @@ class InstallRequirement(object): if "build-system" not in pp_toml: return ["setuptools", "wheel"] + # Specifying the build-system table but not the requires key is invalid + build_system = pp_toml["build-system"] + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=self, reason=( + "it has a 'build-system' table but not the " + "'build-system.requires' which is mandatory in the table" + )) + ) + # Error out if it's not a list of strings - requires = build_system.get("requires", []) + requires = build_system["requires"] if not _is_list_of_str(requires): raise InstallationError(error_template.format( self, "'build-system.requires' is not a list of strings." From 2dd3a28bacaa47d53fc8537acaf06795e2971f50 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 16:26:15 +0530 Subject: [PATCH 104/129] Fix message nit pointed out by @pfmoore --- src/pip/_internal/req/req_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 77526283d..450d05090 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -585,7 +585,7 @@ class InstallRequirement(object): if "requires" not in build_system: raise InstallationError( error_template.format(package=self, reason=( - "it has a 'build-system' table but not the " + "it has a 'build-system' table but not " "'build-system.requires' which is mandatory in the table" )) ) From b8107a39f75993cad59cc5fb4e8b928e836e063b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 16:27:15 +0530 Subject: [PATCH 105/129] Fix message generation which caused a test failure --- src/pip/_internal/req/req_install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 450d05090..462c80aa1 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -594,7 +594,8 @@ class InstallRequirement(object): requires = build_system["requires"] if not _is_list_of_str(requires): raise InstallationError(error_template.format( - self, "'build-system.requires' is not a list of strings." + package=self, + reason="'build-system.requires' is not a list of strings.", )) return requires From 6b1d412948d9ace9ea1b5a75f9176fb46f7a4900 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 21 Jul 2018 16:36:34 +0530 Subject: [PATCH 106/129] :newspaper: --- news/5627.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/5627.bugfix diff --git a/news/5627.bugfix b/news/5627.bugfix new file mode 100644 index 000000000..d26364d8b --- /dev/null +++ b/news/5627.bugfix @@ -0,0 +1 @@ +Disallow packages with ``pyproject.toml`` files that have an empty build-system table. From dd8cbf8da3effb654c6827775e0664571e3bc1ab Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 21 Jul 2018 18:49:08 +0100 Subject: [PATCH 107/129] Fix a typo in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 46f0fb3e0..d2116e934 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = docs, packaging, lint-py2, lint-py3, mypy, - py27, py34, py35, py67, py37, pypy, pypy3 + py27, py34, py35, py36, py37, pypy, pypy3 [testenv] passenv = CI GIT_SSL_CAINFO From aa84f6b360e056be8dbe74978bd9e024dff0132d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 22 Jul 2018 11:06:29 +0530 Subject: [PATCH 108/129] Bump version for release --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index edcc99ba3..9227d0ead 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1 +1 @@ -__version__ = "18.0.dev0" +__version__ = "18.0" From 5d413b2094eb548a3ef91038da20b872e7e8d741 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 22 Jul 2018 11:05:00 +0530 Subject: [PATCH 109/129] Update AUTHORS.txt for 18.0 --- AUTHORS.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/AUTHORS.txt b/AUTHORS.txt index fd8ed6d64..f02621e55 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -1,16 +1,20 @@ Adam Chainz Adam Wentz +Adrien Morison Alan Yee Aleks Bunin Alex Gaynor Alex Grönholm Alex Morega Alex Stachowiak +Alexander Shtyrov Alexandre Conrad Alli Anatoly Techtonik Andrei Geacar Andrey Bulgakov +Andrés Delfino <34587441+andresdelfino@users.noreply.github.com> +Andrés Delfino Andy Freeland Andy Kluger Anish Tambe @@ -33,6 +37,7 @@ Atsushi Odagiri Avner Cohen Baptiste Mispelon Bartek Ogryczak +Bastian Venthur Ben Darnell Ben Hoyt Ben Rosser @@ -45,6 +50,7 @@ Bogdan Opanchuk Brad Erickson Bradley Ayers Brandon L. Reiss +Brett Randall Brian Rosner BrownTruck Bruno Oliveira @@ -81,6 +87,7 @@ Craig Kerstiens Cristian Sorinel Curtis Doty Damian Quiroga +Dan Black Dan Savilonis Dan Sully daniel @@ -88,7 +95,9 @@ Daniel Collins Daniel Hahler Daniel Holth Daniel Jost +Daniel Shaulov Daniele Procida +Danny Hermes Dav Clark Dave Abrahams David Aguilar @@ -106,10 +115,12 @@ Domen Kožar Donald Stufft Dongweiming Douglas Thor +DrFeathers Dustin Ingram Dwayne Bailey Ed Morley <501702+edmorley@users.noreply.github.com> Ed Morley +Eli Schwartz Emil Styrke Endoh Takanao enoch @@ -118,6 +129,8 @@ Eric Hanchrow Eric Hopper Erik M. Bray Erik Rose +Ernest W Durbin III +Ernest W. Durbin III Erwin Janssen Eugene Vereshchagin fiber-space @@ -135,6 +148,7 @@ George Song Georgi Valkov Giftlin Rajaiah gizmoguy1 +gkdoc <40815324+gkdoc@users.noreply.github.com> GOTO Hayato <3532528+gh640@users.noreply.github.com> Guilherme Espada Guy Rozendorn @@ -171,6 +185,7 @@ Jay Graves Jeff Barber Jeff Dairiki Jeremy Stanley +Jeremy Zafran Jim Garrison Jivan Amara John-Scott Atlakson @@ -190,6 +205,7 @@ jwg4 Jyrki Pulliainen Kamal Bin Mustafa kaustav haldar +keanemind Kelsey Hightower Kenneth Belitzky Kenneth Reitz @@ -197,6 +213,7 @@ Kenneth Reitz Kevin Burke Kevin Carter Kevin Frommelt +Kexuan Sun Kit Randel kpinc Kumar McMillan @@ -246,16 +263,19 @@ Miguel Araujo Perez Mihir Singh Min RK MinRK +Miro Hrončok montefra Monty Taylor Nate Coraor Nathaniel J. Smith +Nehal J Wani Nick Coghlan Nick Stenning Nikhil Benesch Nowell Strite nvdv Ofekmeister +Oliver Jeeves Oliver Tonnhofer Olivier Girardot Olivier Grisel @@ -281,6 +301,7 @@ Phaneendra Chiruvella Phil Freo Phil Pennock Phil Whelan +Philip Molloy Philippe Ombredanne Pi Delport Pierre-Yves Rofes @@ -323,6 +344,7 @@ Sebastian Schaetz Segev Finer Sergey Vasilyev Seth Woodworth +Shlomi Fish Simeon Visser Simon Cross Simon Pichugin @@ -340,6 +362,7 @@ Stéphane Bidoul (ACSONE) Stéphane Bidoul Stéphane Klein Takayuki SHIMIZUKAWA +Thijs Triemstra Thomas Fenzl Thomas Grainger Thomas Guettler @@ -351,6 +374,7 @@ Tim Harder Tim Heap tim smith tinruufu +Tom Freudenheim Tom V Tomer Chachamu Tony Zhaocheng Tan @@ -380,4 +404,5 @@ Zearin Zearin Zhiping Deng Zvezdan Petkovic +Łukasz Langa Семён Марьясин From 43fe0c1e70ff25dd25f89f367ab9dfd82a728162 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 22 Jul 2018 11:12:38 +0530 Subject: [PATCH 110/129] Generate NEWS for 18.0 --- NEWS.rst | 77 +++++++++++++++++++ ...450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial | 0 ...b68f4b-6e66-4e94-acbf-0228ca04b2b6.trivial | 0 ...2A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial | 0 ...E9718B-0155-413F-AC2A-1B75AA2934BC.trivial | 0 ...1a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial | 0 ...9004e2-70dd-49df-91c3-29deb506cfbe.trivial | 0 news/3651.removal | 1 - news/3654.removal | 1 - news/3785.feature | 1 - news/3796.removal | 1 - news/3905.bugfix | 1 - ...8CCC8B-0506-40B2-B108-616C18016E59.trivial | 0 news/4174.removal | 1 - ...8e7c9b-3755-4983-9c91-9fd47b1b178c.trivial | 0 news/4471.bugfix | 1 - news/4842.feature | 2 - ...CF6244-41AE-4085-9E20-7C2A0336FB1D.trivial | 0 news/512.doc | 1 - news/5125.feature | 2 - news/5127.bugfix | 1 - news/5196.bugfix | 4 - news/5229.feature | 1 - news/5230.bugfix | 1 - news/5249.bugfix | 1 - news/5261.doc | 1 - news/5265.bugfix | 1 - news/5293.bugfix | 1 - news/5311.bugfix | 1 - news/5312.bugfix | 1 - news/5319.feature | 1 - news/5346.bugfix | 5 -- ...4bb179-538c-4aaa-9688-ac93f21b47f7.trivial | 0 news/5366.bugfix | 1 - news/5381.bugfix | 1 - news/5457.bugfix | 4 - news/5482.bugfix | 1 - news/5520.removal | 4 - news/5558.bugfix | 1 - news/5574.doc | 1 - news/5627.bugfix | 1 - ...b003f8-f701-4df9-a8b5-a869a652327e.trivial | 0 ...C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial | 0 ...6692fe-c2b3-41af-867b-ee11411e0756.trivial | 0 ...B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial | 0 ...1c2359-ff29-4ed8-ba91-2e82dff5e5ad.trivial | 0 ...DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial | 0 ...227e37-7d24-4ecd-9ddd-78bbf0b2b456.trivial | 0 ...d0f55a-2fac-4e6f-b417-4d9d556027a4.trivial | 0 news/cachecontrol.vendor | 1 - news/calver.process | 1 - news/certifi.vendor | 1 - ...66ca44-097a-4ffb-8417-aa046e85eab7.trivial | 0 news/deprecation.process | 2 - news/distro.vendor | 1 - ...19045b-46b8-4ac0-a98a-60d2afa2c675.trivial | 0 ...6bc245-7633-470a-a05e-925309ba648d.trivial | 1 - news/idna.vendor | 1 - news/ipaddress.vendor | 1 - news/news-file-content.process | 1 - news/pkg_resources.vendor | 1 - news/progress.vendor | 1 - news/pytoml.vendor | 1 - news/release-cadence.process | 1 - news/requests.vendor | 1 - news/urllib3.vendor | 1 - 66 files changed, 77 insertions(+), 61 deletions(-) delete mode 100644 news/03450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial delete mode 100644 news/03b68f4b-6e66-4e94-acbf-0228ca04b2b6.trivial delete mode 100644 news/052A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial delete mode 100644 news/07E9718B-0155-413F-AC2A-1B75AA2934BC.trivial delete mode 100644 news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial delete mode 100644 news/2b9004e2-70dd-49df-91c3-29deb506cfbe.trivial delete mode 100644 news/3651.removal delete mode 100644 news/3654.removal delete mode 100644 news/3785.feature delete mode 100644 news/3796.removal delete mode 100644 news/3905.bugfix delete mode 100644 news/408CCC8B-0506-40B2-B108-616C18016E59.trivial delete mode 100644 news/4174.removal delete mode 100644 news/438e7c9b-3755-4983-9c91-9fd47b1b178c.trivial delete mode 100644 news/4471.bugfix delete mode 100644 news/4842.feature delete mode 100644 news/4BCF6244-41AE-4085-9E20-7C2A0336FB1D.trivial delete mode 100644 news/512.doc delete mode 100644 news/5125.feature delete mode 100644 news/5127.bugfix delete mode 100644 news/5196.bugfix delete mode 100644 news/5229.feature delete mode 100644 news/5230.bugfix delete mode 100644 news/5249.bugfix delete mode 100644 news/5261.doc delete mode 100644 news/5265.bugfix delete mode 100644 news/5293.bugfix delete mode 100644 news/5311.bugfix delete mode 100644 news/5312.bugfix delete mode 100644 news/5319.feature delete mode 100644 news/5346.bugfix delete mode 100644 news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial delete mode 100644 news/5366.bugfix delete mode 100644 news/5381.bugfix delete mode 100644 news/5457.bugfix delete mode 100644 news/5482.bugfix delete mode 100644 news/5520.removal delete mode 100644 news/5558.bugfix delete mode 100644 news/5574.doc delete mode 100644 news/5627.bugfix delete mode 100644 news/64b003f8-f701-4df9-a8b5-a869a652327e.trivial delete mode 100644 news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial delete mode 100644 news/776692fe-c2b3-41af-867b-ee11411e0756.trivial delete mode 100644 news/82B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial delete mode 100644 news/951c2359-ff29-4ed8-ba91-2e82dff5e5ad.trivial delete mode 100644 news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial delete mode 100644 news/bb227e37-7d24-4ecd-9ddd-78bbf0b2b456.trivial delete mode 100644 news/bbd0f55a-2fac-4e6f-b417-4d9d556027a4.trivial delete mode 100644 news/cachecontrol.vendor delete mode 100644 news/calver.process delete mode 100644 news/certifi.vendor delete mode 100644 news/dd66ca44-097a-4ffb-8417-aa046e85eab7.trivial delete mode 100644 news/deprecation.process delete mode 100644 news/distro.vendor delete mode 100644 news/ed19045b-46b8-4ac0-a98a-60d2afa2c675.trivial delete mode 100644 news/f86bc245-7633-470a-a05e-925309ba648d.trivial delete mode 100644 news/idna.vendor delete mode 100644 news/ipaddress.vendor delete mode 100644 news/news-file-content.process delete mode 100644 news/pkg_resources.vendor delete mode 100644 news/progress.vendor delete mode 100644 news/pytoml.vendor delete mode 100644 news/release-cadence.process delete mode 100644 news/requests.vendor delete mode 100644 news/urllib3.vendor diff --git a/NEWS.rst b/NEWS.rst index 2457576e0..bae67ef80 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,83 @@ .. towncrier release notes start +18.0 (2018-07-22) +================= + +Process +------- + +- Switch to a Calendar based versioning scheme. +- Formally document our deprecation process as a minimum of 6 months of deprecation + warnings. +- Adopt and document NEWS fragment writing style +- Switch to releasing a new, non bug fix version of pip every 3 months. + +Deprecations and Removals +------------------------- + +- Remove the legacy format from pip list. (#3651, #3654) +- Dropped support for Python 3.3. (#3796) +- Remove support for cleaning up #egg fragment postfixes. (#4174) +- Remove the shim for the old get-pip.py location. + + For the past 2 years, it's only been redirecting users to use the newer + https://bootstrap.pypa.io/get-pip.py location. (#5520) + +Features +-------- + +- Introduce a new --prefer-binary flag, to prefer older wheels over newer source packages. (#3785) +- Improve autocompletion function on file name completion after options + which have ````, ```` or ```` as metavar. (#4842, #5125) +- Add support for installing PEP 518 build dependencies from source. (#5229) +- Improve status message when upgrade is skipped due to only-if-needed strategy. (#5319) + +Bug Fixes +--------- + +- Update pip's self-check logic to not use a virtualenv specific file and honor cache-dir. (#3905) +- Remove compiled pyo files for wheel packages. (#4471) +- Speed up printing of newly installed package versions. (#5127) +- Restrict install time dependency warnings to directly-dependant packages. + + Warning about the entire package set has resulted in users getting confused as + to why pip is printing these warnings. (#5196, #5457) +- Improve handling of PEP 518 build requirements: support environment markers and extras. (#5230, #5265) +- Remove username/password from log message when using index with basic auth. (#5249) +- Remove trailing os.sep from PATH directories to avoid false negatives. (#5293) +- Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. (#5311, #5312) +- Disable pip's version check (and upgrade message) when installed by a different package manager. + + This works better with Linux distributions where pip's upgrade message may + result in users running pip in a manner that modifies files that should be + managed by the OS's package manager. (#5346) +- Check for file existence and unlink first when clobbering existing files during a wheel install. (#5366) +- Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. (#5381) +- Always read ``pyproject.toml`` as UTF-8. This fixes Unicode handling on Windows and Python 2. (#5482) +- Fix a crash that occurs when PATH not set, while generating script location warning. (#5558) +- Disallow packages with ``pyproject.toml`` files that have an empty build-system table. (#5627) + +Vendored Libraries +------------------ + +- Update CacheControl to 0.12.5. +- Update certifi to 2018.4.16. +- Update distro to 1.3.0. +- Update idna to 2.7. +- Update ipaddress to 1.0.22. +- Update pkg_resources to 39.2.0 (via setuptools). +- Update progress to 1.4. +- Update pytoml to 0.1.16. +- Update requests to 2.19.1. +- Update urllib3 to 1.23. + +Improved Documentation +---------------------- + +- Document how to use pip with a proxy server. (#512, #5574) +- Document that the output of pip show is in RFC-compliant mail header format. (#5261) + 10.0.1 (2018-04-19) =================== diff --git a/news/03450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial b/news/03450A0C-9B6C-4AAB-8171-AB92935AE2CA.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/03b68f4b-6e66-4e94-acbf-0228ca04b2b6.trivial b/news/03b68f4b-6e66-4e94-acbf-0228ca04b2b6.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/052A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial b/news/052A794E-8CDC-4F3D-9B06-D195A6C35FD4.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/07E9718B-0155-413F-AC2A-1B75AA2934BC.trivial b/news/07E9718B-0155-413F-AC2A-1B75AA2934BC.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial b/news/271a34fa-97be-4ab4-afcf-e755f3fffdf9.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/2b9004e2-70dd-49df-91c3-29deb506cfbe.trivial b/news/2b9004e2-70dd-49df-91c3-29deb506cfbe.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/3651.removal b/news/3651.removal deleted file mode 100644 index ed69b2c86..000000000 --- a/news/3651.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the legacy format from pip list. diff --git a/news/3654.removal b/news/3654.removal deleted file mode 100644 index ed69b2c86..000000000 --- a/news/3654.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the legacy format from pip list. diff --git a/news/3785.feature b/news/3785.feature deleted file mode 100644 index ca5463ec8..000000000 --- a/news/3785.feature +++ /dev/null @@ -1 +0,0 @@ -Introduce a new --prefer-binary flag, to prefer older wheels over newer source packages. diff --git a/news/3796.removal b/news/3796.removal deleted file mode 100644 index c4c4bbb17..000000000 --- a/news/3796.removal +++ /dev/null @@ -1 +0,0 @@ -Dropped support for Python 3.3. diff --git a/news/3905.bugfix b/news/3905.bugfix deleted file mode 100644 index 37fcaf273..000000000 --- a/news/3905.bugfix +++ /dev/null @@ -1 +0,0 @@ -Update pip's self-check logic to not use a virtualenv specific file and honor cache-dir. diff --git a/news/408CCC8B-0506-40B2-B108-616C18016E59.trivial b/news/408CCC8B-0506-40B2-B108-616C18016E59.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/4174.removal b/news/4174.removal deleted file mode 100644 index 30fd3f0e9..000000000 --- a/news/4174.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for cleaning up #egg fragment postfixes. diff --git a/news/438e7c9b-3755-4983-9c91-9fd47b1b178c.trivial b/news/438e7c9b-3755-4983-9c91-9fd47b1b178c.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/4471.bugfix b/news/4471.bugfix deleted file mode 100644 index 997e59f59..000000000 --- a/news/4471.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove compiled pyo files for wheel packages. diff --git a/news/4842.feature b/news/4842.feature deleted file mode 100644 index f96a668ec..000000000 --- a/news/4842.feature +++ /dev/null @@ -1,2 +0,0 @@ -Improve autocompletion function on file name completion after options -which have ````, ```` or ```` as metavar. diff --git a/news/4BCF6244-41AE-4085-9E20-7C2A0336FB1D.trivial b/news/4BCF6244-41AE-4085-9E20-7C2A0336FB1D.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/512.doc b/news/512.doc deleted file mode 100644 index 6c35db573..000000000 --- a/news/512.doc +++ /dev/null @@ -1 +0,0 @@ -Document how to use pip with a proxy server. diff --git a/news/5125.feature b/news/5125.feature deleted file mode 100644 index f96a668ec..000000000 --- a/news/5125.feature +++ /dev/null @@ -1,2 +0,0 @@ -Improve autocompletion function on file name completion after options -which have ````, ```` or ```` as metavar. diff --git a/news/5127.bugfix b/news/5127.bugfix deleted file mode 100644 index 8a7880a37..000000000 --- a/news/5127.bugfix +++ /dev/null @@ -1 +0,0 @@ -Speed up printing of newly installed package versions. diff --git a/news/5196.bugfix b/news/5196.bugfix deleted file mode 100644 index 405cc2a46..000000000 --- a/news/5196.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Restrict install time dependency warnings to directly-dependant packages. - -Warning about the entire package set has resulted in users getting confused as -to why pip is printing these warnings. diff --git a/news/5229.feature b/news/5229.feature deleted file mode 100644 index 8df75ad16..000000000 --- a/news/5229.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for installing PEP 518 build dependencies from source. diff --git a/news/5230.bugfix b/news/5230.bugfix deleted file mode 100644 index 9f34eddf5..000000000 --- a/news/5230.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve handling of PEP 518 build requirements: support environment markers and extras. diff --git a/news/5249.bugfix b/news/5249.bugfix deleted file mode 100644 index 7b6ea48da..000000000 --- a/news/5249.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove username/password from log message when using index with basic auth. diff --git a/news/5261.doc b/news/5261.doc deleted file mode 100644 index 01f48a36a..000000000 --- a/news/5261.doc +++ /dev/null @@ -1 +0,0 @@ -Document that the output of pip show is in RFC-compliant mail header format. diff --git a/news/5265.bugfix b/news/5265.bugfix deleted file mode 100644 index 9f34eddf5..000000000 --- a/news/5265.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve handling of PEP 518 build requirements: support environment markers and extras. diff --git a/news/5293.bugfix b/news/5293.bugfix deleted file mode 100644 index bff0d8daa..000000000 --- a/news/5293.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove trailing os.sep from PATH directories to avoid false negatives. diff --git a/news/5311.bugfix b/news/5311.bugfix deleted file mode 100644 index dfa4663f1..000000000 --- a/news/5311.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. diff --git a/news/5312.bugfix b/news/5312.bugfix deleted file mode 100644 index dfa4663f1..000000000 --- a/news/5312.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. diff --git a/news/5319.feature b/news/5319.feature deleted file mode 100644 index 08fd1a7d4..000000000 --- a/news/5319.feature +++ /dev/null @@ -1 +0,0 @@ -Improve status message when upgrade is skipped due to only-if-needed strategy. diff --git a/news/5346.bugfix b/news/5346.bugfix deleted file mode 100644 index 3a652ac55..000000000 --- a/news/5346.bugfix +++ /dev/null @@ -1,5 +0,0 @@ -Disable pip's version check (and upgrade message) when installed by a different package manager. - -This works better with Linux distributions where pip's upgrade message may -result in users running pip in a manner that modifies files that should be -managed by the OS's package manager. diff --git a/news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial b/news/534bb179-538c-4aaa-9688-ac93f21b47f7.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/5366.bugfix b/news/5366.bugfix deleted file mode 100644 index af37e3369..000000000 --- a/news/5366.bugfix +++ /dev/null @@ -1 +0,0 @@ -Check for file existence and unlink first when clobbering existing files during a wheel install. diff --git a/news/5381.bugfix b/news/5381.bugfix deleted file mode 100644 index c50b2212a..000000000 --- a/news/5381.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. diff --git a/news/5457.bugfix b/news/5457.bugfix deleted file mode 100644 index 405cc2a46..000000000 --- a/news/5457.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Restrict install time dependency warnings to directly-dependant packages. - -Warning about the entire package set has resulted in users getting confused as -to why pip is printing these warnings. diff --git a/news/5482.bugfix b/news/5482.bugfix deleted file mode 100644 index dd8d65e0b..000000000 --- a/news/5482.bugfix +++ /dev/null @@ -1 +0,0 @@ -Always read ``pyproject.toml`` as UTF-8. This fixes Unicode handling on Windows and Python 2. diff --git a/news/5520.removal b/news/5520.removal deleted file mode 100644 index 49fe4369f..000000000 --- a/news/5520.removal +++ /dev/null @@ -1,4 +0,0 @@ -Remove the shim for the old get-pip.py location. - -For the past 2 years, it's only been redirecting users to use the newer -https://bootstrap.pypa.io/get-pip.py location. diff --git a/news/5558.bugfix b/news/5558.bugfix deleted file mode 100644 index 0407d5742..000000000 --- a/news/5558.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a crash that occurs when PATH not set, while generating script location warning. diff --git a/news/5574.doc b/news/5574.doc deleted file mode 100644 index 6c35db573..000000000 --- a/news/5574.doc +++ /dev/null @@ -1 +0,0 @@ -Document how to use pip with a proxy server. diff --git a/news/5627.bugfix b/news/5627.bugfix deleted file mode 100644 index d26364d8b..000000000 --- a/news/5627.bugfix +++ /dev/null @@ -1 +0,0 @@ -Disallow packages with ``pyproject.toml`` files that have an empty build-system table. diff --git a/news/64b003f8-f701-4df9-a8b5-a869a652327e.trivial b/news/64b003f8-f701-4df9-a8b5-a869a652327e.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial b/news/67C1ECA3-A0C6-4DA6-B987-BE46C7629AF2.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/776692fe-c2b3-41af-867b-ee11411e0756.trivial b/news/776692fe-c2b3-41af-867b-ee11411e0756.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/82B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial b/news/82B89AA2-C6E5-480E-AD03-2D55E913A01E.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/951c2359-ff29-4ed8-ba91-2e82dff5e5ad.trivial b/news/951c2359-ff29-4ed8-ba91-2e82dff5e5ad.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial b/news/B5DC829C-8E61-468C-A9CD-DCCC79D26F85.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/bb227e37-7d24-4ecd-9ddd-78bbf0b2b456.trivial b/news/bb227e37-7d24-4ecd-9ddd-78bbf0b2b456.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/bbd0f55a-2fac-4e6f-b417-4d9d556027a4.trivial b/news/bbd0f55a-2fac-4e6f-b417-4d9d556027a4.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/cachecontrol.vendor b/news/cachecontrol.vendor deleted file mode 100644 index a4c1b5ff5..000000000 --- a/news/cachecontrol.vendor +++ /dev/null @@ -1 +0,0 @@ -Update CacheControl to 0.12.5. diff --git a/news/calver.process b/news/calver.process deleted file mode 100644 index bfd7ba7a5..000000000 --- a/news/calver.process +++ /dev/null @@ -1 +0,0 @@ -Switch to a Calendar based versioning scheme. diff --git a/news/certifi.vendor b/news/certifi.vendor deleted file mode 100644 index 751544d7a..000000000 --- a/news/certifi.vendor +++ /dev/null @@ -1 +0,0 @@ -Update certifi to 2018.4.16. diff --git a/news/dd66ca44-097a-4ffb-8417-aa046e85eab7.trivial b/news/dd66ca44-097a-4ffb-8417-aa046e85eab7.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/deprecation.process b/news/deprecation.process deleted file mode 100644 index 9545be4ff..000000000 --- a/news/deprecation.process +++ /dev/null @@ -1,2 +0,0 @@ -Formally document our deprecation process as a minimum of 6 months of deprecation -warnings. diff --git a/news/distro.vendor b/news/distro.vendor deleted file mode 100644 index 347a6f6ab..000000000 --- a/news/distro.vendor +++ /dev/null @@ -1 +0,0 @@ -Update distro to 1.3.0. diff --git a/news/ed19045b-46b8-4ac0-a98a-60d2afa2c675.trivial b/news/ed19045b-46b8-4ac0-a98a-60d2afa2c675.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/f86bc245-7633-470a-a05e-925309ba648d.trivial b/news/f86bc245-7633-470a-a05e-925309ba648d.trivial deleted file mode 100644 index 8b1378917..000000000 --- a/news/f86bc245-7633-470a-a05e-925309ba648d.trivial +++ /dev/null @@ -1 +0,0 @@ - diff --git a/news/idna.vendor b/news/idna.vendor deleted file mode 100644 index c959331aa..000000000 --- a/news/idna.vendor +++ /dev/null @@ -1 +0,0 @@ -Update idna to 2.7. diff --git a/news/ipaddress.vendor b/news/ipaddress.vendor deleted file mode 100644 index 091d12ad5..000000000 --- a/news/ipaddress.vendor +++ /dev/null @@ -1 +0,0 @@ -Update ipaddress to 1.0.22. diff --git a/news/news-file-content.process b/news/news-file-content.process deleted file mode 100644 index cd89445fa..000000000 --- a/news/news-file-content.process +++ /dev/null @@ -1 +0,0 @@ -Adopt and document NEWS fragment writing style diff --git a/news/pkg_resources.vendor b/news/pkg_resources.vendor deleted file mode 100644 index f04b46c47..000000000 --- a/news/pkg_resources.vendor +++ /dev/null @@ -1 +0,0 @@ -Update pkg_resources to 39.2.0 (via setuptools). diff --git a/news/progress.vendor b/news/progress.vendor deleted file mode 100644 index 513f57ef8..000000000 --- a/news/progress.vendor +++ /dev/null @@ -1 +0,0 @@ -Update progress to 1.4. diff --git a/news/pytoml.vendor b/news/pytoml.vendor deleted file mode 100644 index f7c6faaa2..000000000 --- a/news/pytoml.vendor +++ /dev/null @@ -1 +0,0 @@ -Update pytoml to 0.1.16. diff --git a/news/release-cadence.process b/news/release-cadence.process deleted file mode 100644 index f8f86f0e1..000000000 --- a/news/release-cadence.process +++ /dev/null @@ -1 +0,0 @@ -Switch to releasing a new, non bug fix version of pip every 3 months. diff --git a/news/requests.vendor b/news/requests.vendor deleted file mode 100644 index b6a9c0bad..000000000 --- a/news/requests.vendor +++ /dev/null @@ -1 +0,0 @@ -Update requests to 2.19.1. diff --git a/news/urllib3.vendor b/news/urllib3.vendor deleted file mode 100644 index e89ae636c..000000000 --- a/news/urllib3.vendor +++ /dev/null @@ -1 +0,0 @@ -Update urllib3 to 1.23. From 3ae2d381b9e92222e1454fd704fd4a090d700c81 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 22 Jul 2018 11:15:14 +0530 Subject: [PATCH 111/129] Slightly cleanup NEWS - Move issue numbers to summary line for NEWS entries - Add a period at the end of an entry --- NEWS.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index bae67ef80..6cd81393b 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -16,7 +16,7 @@ Process - Switch to a Calendar based versioning scheme. - Formally document our deprecation process as a minimum of 6 months of deprecation warnings. -- Adopt and document NEWS fragment writing style +- Adopt and document NEWS fragment writing style. - Switch to releasing a new, non bug fix version of pip every 3 months. Deprecations and Removals @@ -25,10 +25,10 @@ Deprecations and Removals - Remove the legacy format from pip list. (#3651, #3654) - Dropped support for Python 3.3. (#3796) - Remove support for cleaning up #egg fragment postfixes. (#4174) -- Remove the shim for the old get-pip.py location. +- Remove the shim for the old get-pip.py location. (#5520) For the past 2 years, it's only been redirecting users to use the newer - https://bootstrap.pypa.io/get-pip.py location. (#5520) + https://bootstrap.pypa.io/get-pip.py location. Features -------- @@ -45,19 +45,19 @@ Bug Fixes - Update pip's self-check logic to not use a virtualenv specific file and honor cache-dir. (#3905) - Remove compiled pyo files for wheel packages. (#4471) - Speed up printing of newly installed package versions. (#5127) -- Restrict install time dependency warnings to directly-dependant packages. +- Restrict install time dependency warnings to directly-dependant packages. (#5196, #5457) Warning about the entire package set has resulted in users getting confused as - to why pip is printing these warnings. (#5196, #5457) + to why pip is printing these warnings. - Improve handling of PEP 518 build requirements: support environment markers and extras. (#5230, #5265) - Remove username/password from log message when using index with basic auth. (#5249) - Remove trailing os.sep from PATH directories to avoid false negatives. (#5293) - Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. (#5311, #5312) -- Disable pip's version check (and upgrade message) when installed by a different package manager. +- Disable pip's version check (and upgrade message) when installed by a different package manager. (#5346) This works better with Linux distributions where pip's upgrade message may result in users running pip in a manner that modifies files that should be - managed by the OS's package manager. (#5346) + managed by the OS's package manager. - Check for file existence and unlink first when clobbering existing files during a wheel install. (#5366) - Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. (#5381) - Always read ``pyproject.toml`` as UTF-8. This fixes Unicode handling on Windows and Python 2. (#5482) From 8be6062f13a7b2818b69f8027e65fe6602ccdc3b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 22 Jul 2018 15:05:04 +0530 Subject: [PATCH 112/129] Bump for development of 18.1 (#5630) --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 9227d0ead..2e8c4e43d 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1 +1 @@ -__version__ = "18.0" +__version__ = "18.1.dev0" From 47e8703241076897809e16ece17496ed7545efde Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 22 Jul 2018 18:34:01 +0530 Subject: [PATCH 113/129] Add the deprecation helper information from b6dae7a5949ec67c16678f0a367cc8452f10d3b0 --- docs/development/release-process.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/development/release-process.rst b/docs/development/release-process.rst index 13c144798..c90c08f71 100644 --- a/docs/development/release-process.rst +++ b/docs/development/release-process.rst @@ -50,6 +50,13 @@ document existing behavior with the intention of covering that behavior with the above deprecation process are always acceptable, and will be considered on their merits. +.. note:: + + pip has a helper function for making deprecation easier for pip maintainers. + The supporting documentation can be found in the source code of + ``pip._internal.utils.deprecation.deprecated``. The function is not a part of + pip's public API. + Release Process =============== From 9c92c618a48326da06642c2c3fb41dd4f3e2555f Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Sun, 22 Jul 2018 14:13:34 -0400 Subject: [PATCH 114/129] user_guide: Fix "Requirements Files" reference --- docs/user_guide.rst | 5 +++-- news/user_guide_fix_requirements_file_ref.doc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 news/user_guide_fix_requirements_file_ref.doc diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 4520c3831..d89ad5376 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -49,8 +49,6 @@ For more information and examples, see the :ref:`pip install` reference. .. _PyPI: https://pypi.org/ -.. _`Requirements Files`: - Using a Proxy Server ******************** @@ -65,6 +63,9 @@ pip can be configured to connect through a proxy server in various ways: * by setting the standard environment-variables ``http_proxy``, ``https_proxy`` and ``no_proxy``. + +.. _`Requirements Files`: + Requirements Files ****************** diff --git a/news/user_guide_fix_requirements_file_ref.doc b/news/user_guide_fix_requirements_file_ref.doc new file mode 100644 index 000000000..07112d375 --- /dev/null +++ b/news/user_guide_fix_requirements_file_ref.doc @@ -0,0 +1 @@ +Fix "Requirements Files" reference in User Guide From a1e32752873081b14ce53961404235499b451fea Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Sun, 22 Jul 2018 11:26:44 -0700 Subject: [PATCH 115/129] Remove the unused VersionControl.translate_egg_surname(). --- news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial | 0 src/pip/_internal/vcs/__init__.py | 6 ------ tests/unit/test_vcs.py | 9 +-------- 3 files changed, 1 insertion(+), 14 deletions(-) create mode 100644 news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial diff --git a/news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial b/news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 9082fa2b7..768be3c97 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -200,12 +200,6 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - # See issue #1083 for why this method was introduced: - # https://github.com/pypa/pip/issues/1083 - def translate_egg_surname(self, surname): - # For example, Django has branches of the form "stable/1.7.x". - return surname.replace('/', '_') - def export(self, location): """ Export the repository at the url to the destination location diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 939dcc3cb..e6b473dae 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -2,7 +2,7 @@ import pytest from mock import Mock from pip._vendor.packaging.version import parse as parse_version -from pip._internal.vcs import RevOptions, VersionControl +from pip._internal.vcs import RevOptions from pip._internal.vcs.bazaar import Bazaar from pip._internal.vcs.git import Git, looks_like_hash from pip._internal.vcs.mercurial import Mercurial @@ -122,13 +122,6 @@ def test_git_is_commit_id_equal(git, rev_name, result): assert git.is_commit_id_equal('/path', rev_name) is result -def test_translate_egg_surname(): - vc = VersionControl() - assert vc.translate_egg_surname("foo") == "foo" - assert vc.translate_egg_surname("foo/bar") == "foo_bar" - assert vc.translate_egg_surname("foo/1.2.3") == "foo_1.2.3" - - # The non-SVN backends all use the same get_netloc_and_auth(), so only test # Git as a representative. @pytest.mark.parametrize('netloc, expected', [ From ae58decefe0afe0f4a8767df64f77f844d312b26 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 23 Jul 2018 10:47:43 +0530 Subject: [PATCH 116/129] Inline a super-simple function --- src/pip/_internal/__init__.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 250ed1d90..4a0390d92 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -274,15 +274,6 @@ def parseopts(args): return cmd_name, cmd_args -def check_isolated(args): - isolated = False - - if "--isolated" in args: - isolated = True - - return isolated - - def main(args=None): if args is None: args = sys.argv[1:] @@ -306,5 +297,5 @@ def main(args=None): except locale.Error as e: # setlocale can apparently crash if locale are uninitialized logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) + command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) return command.main(cmd_args) From ab0644479fe4152ebdf23414a40747446183e112 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 23 Jul 2018 13:11:57 +0100 Subject: [PATCH 117/129] Report files in original (filesystem) case --- src/pip/_internal/req/req_uninstall.py | 7 +++++-- tests/unit/test_req_uninstall.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index f7ec3a8f8..52ea17b2d 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -120,6 +120,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) + _normcased_files = set(map(os.path.normcase, files)) + folders = compact(folders) # This walks the tree using os.walk to not miss extra folders @@ -130,8 +132,9 @@ def compress_for_output_listing(paths): if fname.endswith(".pyc"): continue - file_ = os.path.normcase(os.path.join(dirpath, fname)) - if os.path.isfile(file_) and file_ not in files: + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index dc8517961..8f9994579 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -50,9 +50,9 @@ def test_compressed_listing(tmpdir): def in_tmpdir(paths): li = [] for path in paths: - li.append(str(os.path.normcase( - os.path.join(tmpdir, path.replace("/", os.path.sep)) - ))) + li.append( + str(os.path.join(tmpdir, path.replace("/", os.path.sep))) + ) return li sample = in_tmpdir([ From f1033288a3e63b9c219e28256bface746d9780d5 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 23 Jul 2018 13:26:05 +0100 Subject: [PATCH 118/129] Placate flake8's moaning --- src/pip/_internal/req/req_uninstall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index 52ea17b2d..32ac6b2f7 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -134,7 +134,7 @@ def compress_for_output_listing(paths): file_ = os.path.join(dirpath, fname) if (os.path.isfile(file_) and - os.path.normcase(file_) not in _normcased_files): + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) From dddbbebdeca85b2bf9371f96d2d51732c8c2c5ee Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 23 Jul 2018 16:09:27 +0100 Subject: [PATCH 119/129] Implement check for doc-only PRs on Appveyor --- appveyor.yml | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 078b4db0f..2a0a9d9a1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,10 +28,37 @@ cache: - '%LOCALAPPDATA%\pip\Cache' test_script: - # Shorten paths, workaround https://bugs.python.org/issue18199 - - "subst T: %TEMP%" - - "set TEMP=T:\\" - - "set TMP=T:\\" - - "tox -e py -- -m unit -n 3" - - "if \"%RUN_INTEGRATION_TESTS%\" == \"True\" ( - tox -e py -- -m integration -n 3 --duration=5 )" + - ps: | + function should_run_tests { + if ("$env:APPVEYOR_PULL_REQUEST_NUMBER" -eq "") { + Write-Host "Not a pull request - running tests" + return $true + } + Write-Host "Pull request $env:APPVEYOR_PULL_REQUEST_NUMBER based on branch $env:APPVEYOR_REPO_BRANCH" + git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH + $changes = (git diff --name-only HEAD (git merge-base HEAD FETCH_HEAD)) + Write-Host "Files changed:" + Write-Host $changes + $important = $changes | Where-Object { $_ -NotLike "*.rst" } | + Where-Object { $_ -NotLike "docs*" } | + Where-Object { $_ -NotLike "news*" } | + Where-Object { $_ -NotLike ".github*" } + if (!$important) { + Write-Host "Only documentation changes - skipping tests" + return $false + } + + Write-Host "Pull request $env:APPVEYOR_PULL_REQUEST_NUMBER alters code - running tests" + return $true + } + + if (should_run_tests) { + # Shorten paths, workaround https://bugs.python.org/issue18199 + subst T: $env:TEMP + $env:TEMP = "T:\" + $env:TMP = "T:\" + tox -e py -- -m unit -n 3 + if ($env:RUN_INTEGRATION_TESTS -eq "True") { + tox -e py -- -m integration -n 3 --duration=5 + } + } From 468e38fcf9cfc9c6e21156955cc070fa40e9e78f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 00:58:20 +0530 Subject: [PATCH 120/129] Add a note saying older references might be broken --- docs/development/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/development/index.rst b/docs/development/index.rst index 0118cb38b..4124d831f 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -16,5 +16,10 @@ or the `pypa-dev mailing list`_, to ask questions or get involved. contributing release-process +.. note:: + + pip's development documentation has been rearranged and some older + references might be broken. + .. _`on Freenode`: https://webchat.freenode.net/?channels=%23pypa-dev,pypa .. _`pypa-dev mailing list`: https://groups.google.com/forum/#!forum/pypa-dev From 0d74b20acd9cf90e0b1964426b7a558bb3c2c3e8 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 00:59:39 +0530 Subject: [PATCH 121/129] Update the link in the pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a556ac046..ada7086d9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,5 +6,5 @@ require either a news file fragment or a marker to indicate they don't require one. To read more about adding a news file fragment for your PR, please check out -our documentation at: https://pip.pypa.io/en/latest/development/#adding-a-news-entry +our documentation at: https://pip.pypa.io/en/latest/development/contributing/#adding-a-news-entry --> From 0734255da39c57c06eab320c4293e82fb0729530 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 01:18:12 +0530 Subject: [PATCH 122/129] Don't mention pip in the headers of development docs pages --- docs/development/contributing.rst | 6 +++--- docs/development/release-process.rst | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index 821323755..51d8a5d58 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -1,6 +1,6 @@ -=================== -Contributing to pip -=================== +============ +Contributing +============ .. todo Create a "guide" to pip's internals and link to it from here saying diff --git a/docs/development/release-process.rst b/docs/development/release-process.rst index c90c08f71..2a076fdca 100644 --- a/docs/development/release-process.rst +++ b/docs/development/release-process.rst @@ -1,6 +1,6 @@ -===================== -pip's release process -===================== +=============== +Release process +=============== Release Cadence From 35c66f3ade85b6b7b4e19c95b0d6a09e53b12bee Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 07:38:00 +0530 Subject: [PATCH 123/129] Fix a mistake made while merging --- src/pip/_internal/models/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/models/index.py b/src/pip/_internal/models/index.py index 14ddde9c9..d3bbec393 100644 --- a/src/pip/_internal/models/index.py +++ b/src/pip/_internal/models/index.py @@ -17,7 +17,7 @@ class PackageIndex(object): # block such packages themselves self.file_storage_domain = file_storage_domain - def url_to_path(self, path): + def _url_to_path(self, path): return urllib_parse.urljoin(self.url, path) From 0b1b4797c0a925cf31c23cedfdee58b7d6b69b7a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 08:08:43 +0530 Subject: [PATCH 124/129] Fix issues that occured due to merging --- src/pip/_internal/models/index.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/index.py b/src/pip/_internal/models/index.py index d3bbec393..870a315ed 100644 --- a/src/pip/_internal/models/index.py +++ b/src/pip/_internal/models/index.py @@ -17,9 +17,13 @@ class PackageIndex(object): # block such packages themselves self.file_storage_domain = file_storage_domain - def _url_to_path(self, path): + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = PackageIndex('https://pypi.org/', 'files.pythonhosted.org') -TestPyPI = PackageIndex('https://test.pypi.org/', 'test-files.pythonhosted.org') +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) From 3e86bf786f8fa242910e54083cad8a4c23060560 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 08:09:17 +0530 Subject: [PATCH 125/129] Sort Imports --- src/pip/_internal/req/req_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index bc2b98b91..e90daf159 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -31,8 +31,8 @@ from pip._internal.exceptions import InstallationError from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) -from pip._internal.models.link import Link from pip._internal.models.index import PyPI, TestPyPI +from pip._internal.models.link import Link from pip._internal.req.req_uninstall import UninstallPathSet from pip._internal.utils.hashes import Hashes from pip._internal.utils.logging import indent_log From 68112c826d09627286cf93779223c18680ecd2be Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 08:33:40 +0530 Subject: [PATCH 126/129] Update PackageIndex tests for the addition of TestPyPI --- tests/unit/test_models.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 1c8aed12b..ab9a2c9dd 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -12,10 +12,13 @@ class TestPackageIndex(object): def test_gives_right_urls(self): url = "https://mypypi.internal/path/" - pack_index = index.PackageIndex(url) + file_storage_domain = "files.mypypi.internal" + pack_index = index.PackageIndex(url, file_storage_domain) + + assert pack_index.url == url + assert pack_index.file_storage_domain == file_storage_domain assert pack_index.netloc == "mypypi.internal" - assert pack_index.url == url assert pack_index.simple_url == url + "simple" assert pack_index.pypi_url == url + "pypi" @@ -26,6 +29,16 @@ class TestPackageIndex(object): assert pack_index.url == "https://pypi.org/" assert pack_index.simple_url == "https://pypi.org/simple" assert pack_index.pypi_url == "https://pypi.org/pypi" + assert pack_index.file_storage_domain == "files.pythonhosted.org" + + def test_TestPyPI_urls_are_correct(self): + pack_index = index.TestPyPI + + assert pack_index.netloc == "test.pypi.org" + assert pack_index.url == "https://test.pypi.org/" + assert pack_index.simple_url == "https://test.pypi.org/simple" + assert pack_index.pypi_url == "https://test.pypi.org/pypi" + assert pack_index.file_storage_domain == "test-files.pythonhosted.org" class TestInstallationCandidate(object): From 4e8c2471547a328caa0f0a30117b2f09474e3f08 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 10:54:51 +0530 Subject: [PATCH 127/129] Document platforms/tooling new pip maintainers can be given access to --- docs/development/contributing.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index 51d8a5d58..3dcf74726 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -229,6 +229,17 @@ If you want to become an official maintainer, start by helping out. Later, when you think you're ready, get in touch with one of the maintainers and they will initiate a vote. +.. note:: + + Upon becoming a maintainer, a person should be given access to various + pip-related tooling across multiple platforms. These are noted here for + future reference by the maintainers: + + - GitHub Push Access + - PyPI Publishing Access + - CI Administration capabilities + - ReadTheDocs Administration capabilities + .. _`Studies have shown`: https://smartbear.com/smartbear/media/pdfs/wp-cc-11-best-practices-of-peer-code-review.pdf .. _`resolve merge conflicts`: https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/ .. _`Travis CI`: https://travis-ci.org/ From bcc8fa80693aa5a196b875dafa9335a756186b11 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 13:03:19 +0530 Subject: [PATCH 128/129] Remove redundant files from docs/ directory --- docs/Makefile | 177 ------------------------------- docs/docutils.conf | 2 - docs/make.bat | 253 --------------------------------------------- 3 files changed, 432 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/docutils.conf delete mode 100644 docs/make.bat diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index bbf7afad0..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pip-installer.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pip-installer.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/pip-installer" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pip-installer" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/docutils.conf b/docs/docutils.conf deleted file mode 100644 index dfd80b177..000000000 --- a/docs/docutils.conf +++ /dev/null @@ -1,2 +0,0 @@ -[restructuredtext parser] -smart_quotes = no diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 40c23cc7c..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,253 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -REM Check if sphinx-build is available and fallback to Python version if any -%SPHINXBUILD% 2> nul -if errorlevel 9009 goto sphinx_python -goto sphinx_ok - -:sphinx_python - -set SPHINXBUILD=python -m sphinx.__init__ -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -:sphinx_ok - - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pip-installer.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pip-installer.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end From 16cd301abb46f0d8828c29c04a94d677a1db0ba6 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 24 Jul 2018 13:03:36 +0530 Subject: [PATCH 129/129] :art: for tox.ini --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d2116e934..6701dd27f 100644 --- a/tox.ini +++ b/tox.ini @@ -22,8 +22,8 @@ commands = pytest --timeout 300 --cov=pip --cov-report=term-missing --cov-report deps = -r{toxinidir}/tools/docs-requirements.txt basepython = python2.7 commands = - sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/build/html - sphinx-build -W -b man -d {envtmpdir}/doctrees docs docs/build/man + sphinx-build -W -d {envtmpdir}/doctrees -b html docs docs/build/html + sphinx-build -W -d {envtmpdir}/doctrees -b man docs docs/build/man [testenv:packaging] deps =