Merge pull request #9371 from hugovk/tidy-urllib-imports

Clean up urllib imports

src/pip/_internal/index/collector.py

@@ -9,9 +9,9 @@ import logging
 import mimetypes
 import os
 import re
+import urllib.parse
+import urllib.request
 from collections import OrderedDict
-from urllib import parse as urllib_parse
-from urllib import request as urllib_request
 
 from pip._vendor import html5lib, requests
 from pip._vendor.distlib.compat import unescape
@@ -94,7 +94,7 @@ def _ensure_html_response(url, session):
     Raises `_NotHTTP` if the URL is not available for a HEAD request, or
     `_NotHTML` if the content type is not text/html.
     """
-    scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
+    scheme, netloc, path, query, fragment = urllib.parse.urlsplit(url)
     if scheme not in {'http', 'https'}:
         raise _NotHTTP()
 
@@ -192,7 +192,7 @@ def _clean_url_path_part(part):
     Clean a "part" of a URL path (i.e. after splitting on "@" characters).
     """
     # We unquote prior to quoting to make sure nothing is double quoted.
-    return urllib_parse.quote(urllib_parse.unquote(part))
+    return urllib.parse.quote(urllib.parse.unquote(part))
 
 
 def _clean_file_url_path(part):
@@ -206,7 +206,7 @@ def _clean_file_url_path(part):
     # should not be quoted. On Linux where drive letters do not
     # exist, the colon should be quoted. We rely on urllib.request
     # to do the right thing here.
-    return urllib_request.pathname2url(urllib_request.url2pathname(part))
+    return urllib.request.pathname2url(urllib.request.url2pathname(part))
 
 
 # percent-encoded:                   /
@@ -245,11 +245,11 @@ def _clean_link(url):
     """
     # Split the URL into parts according to the general structure
     # `scheme://netloc/path;parameters?query#fragment`.
-    result = urllib_parse.urlparse(url)
+    result = urllib.parse.urlparse(url)
     # If the netloc is empty, then the URL refers to a local filesystem path.
     is_local_path = not result.netloc
     path = _clean_url_path(result.path, is_local_path=is_local_path)
-    return urllib_parse.urlunparse(result._replace(path=path))
+    return urllib.parse.urlunparse(result._replace(path=path))
 
 
 def _create_link_from_element(
@@ -265,7 +265,7 @@ def _create_link_from_element(
     if not href:
         return None
 
-    url = _clean_link(urllib_parse.urljoin(base_url, href))
+    url = _clean_link(urllib.parse.urljoin(base_url, href))
     pyrequire = anchor.get('data-requires-python')
     pyrequire = unescape(pyrequire) if pyrequire else None
 
@@ -416,13 +416,13 @@ def _get_html_page(link, session=None):
         return None
 
     # Tack index.html onto file:// URLs that point to directories
-    scheme, _, path, _, _, _ = urllib_parse.urlparse(url)
-    if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))):
+    scheme, _, path, _, _, _ = urllib.parse.urlparse(url)
+    if (scheme == 'file' and os.path.isdir(urllib.request.url2pathname(path))):
         # add trailing slash if not present so urljoin doesn't trim
         # final segment
         if not url.endswith('/'):
             url += '/'
-        url = urllib_parse.urljoin(url, 'index.html')
+        url = urllib.parse.urljoin(url, 'index.html')
         logger.debug(' file: URL is directory, getting %s', url)
 
     try:

src/pip/_internal/models/direct_url.py

@@ -1,7 +1,7 @@
 """ PEP 610 """
 import json
 import re
-from urllib import parse as urllib_parse
+import urllib.parse
 
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
@@ -194,9 +194,9 @@ class DirectUrl:
         environment variables as specified in PEP 610, or it is ``git``
         in the case of a git URL.
         """
-        purl = urllib_parse.urlsplit(self.url)
+        purl = urllib.parse.urlsplit(self.url)
         netloc = self._remove_auth_from_netloc(purl.netloc)
-        surl = urllib_parse.urlunsplit(
+        surl = urllib.parse.urlunsplit(
             (purl.scheme, netloc, purl.path, purl.query, purl.fragment)
         )
         return surl

src/pip/_internal/models/index.py

@@ -1,4 +1,4 @@
-from urllib import parse as urllib_parse
+import urllib.parse
 
 
 class PackageIndex:
@@ -12,7 +12,7 @@ class PackageIndex:
         # type: (str, str) -> None
         super().__init__()
         self.url = url
-        self.netloc = urllib_parse.urlsplit(url).netloc
+        self.netloc = urllib.parse.urlsplit(url).netloc
         self.simple_url = self._url_for_path('simple')
         self.pypi_url = self._url_for_path('pypi')
 
@@ -23,7 +23,7 @@ class PackageIndex:
 
     def _url_for_path(self, path):
         # type: (str) -> str
-        return urllib_parse.urljoin(self.url, path)
+        return urllib.parse.urljoin(self.url, path)
 
 
 PyPI = PackageIndex(

src/pip/_internal/models/link.py

@@ -1,7 +1,7 @@
 import os
 import posixpath
 import re
-from urllib import parse as urllib_parse
+import urllib.parse
 
 from pip._internal.utils.filetypes import WHEEL_EXTENSION
 from pip._internal.utils.misc import (
@@ -67,7 +67,7 @@ class Link(KeyBasedCompareMixin):
         if url.startswith('\\\\'):
             url = path_to_url(url)
 
-        self._parsed_url = urllib_parse.urlsplit(url)
+        self._parsed_url = urllib.parse.urlsplit(url)
         # Store the url as a private attribute to prevent accidentally
         # trying to set a new value.
         self._url = url
@@ -112,7 +112,7 @@ class Link(KeyBasedCompareMixin):
             netloc, user_pass = split_auth_from_netloc(self.netloc)
             return netloc
 
-        name = urllib_parse.unquote(name)
+        name = urllib.parse.unquote(name)
         assert name, (
             'URL {self._url!r} produced no filename'.format(**locals()))
         return name
@@ -138,7 +138,7 @@ class Link(KeyBasedCompareMixin):
     @property
     def path(self):
         # type: () -> str
-        return urllib_parse.unquote(self._parsed_url.path)
+        return urllib.parse.unquote(self._parsed_url.path)
 
     def splitext(self):
         # type: () -> Tuple[str, str]
@@ -153,7 +153,7 @@ class Link(KeyBasedCompareMixin):
     def url_without_fragment(self):
         # type: () -> str
         scheme, netloc, path, query, fragment = self._parsed_url
-        return urllib_parse.urlunsplit((scheme, netloc, path, query, None))
+        return urllib.parse.urlunsplit((scheme, netloc, path, query, None))
 
     _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)')
 

src/pip/_internal/models/search_scope.py

@@ -2,7 +2,7 @@ import itertools
 import logging
 import os
 import posixpath
-from urllib import parse as urllib_parse
+import urllib.parse
 
 from pip._vendor.packaging.utils import canonicalize_name
 
@@ -53,7 +53,7 @@ class SearchScope:
         # relies on TLS.
         if not has_tls():
             for link in itertools.chain(index_urls, built_find_links):
-                parsed = urllib_parse.urlparse(link)
+                parsed = urllib.parse.urlparse(link)
                 if parsed.scheme == 'https':
                     logger.warning(
                         'pip is configured with locations that require '
@@ -86,7 +86,7 @@ class SearchScope:
                 redacted_index_url = redact_auth_from_url(url)
 
                 # Parse the URL
-                purl = urllib_parse.urlsplit(redacted_index_url)
+                purl = urllib.parse.urlsplit(redacted_index_url)
 
                 # URL is generally invalid if scheme and netloc is missing
                 # there are issues with Python and URL parsing, so this test
@@ -122,7 +122,7 @@ class SearchScope:
             # type: (str) -> str
             loc = posixpath.join(
                 url,
-                urllib_parse.quote(canonicalize_name(project_name)))
+                urllib.parse.quote(canonicalize_name(project_name)))
             # For maximum compatibility with easy_install, ensure the path
             # ends in a trailing slash.  Although this isn't in the spec
             # (and PyPI can handle it without the slash) some other index

src/pip/_internal/network/auth.py

@@ -5,7 +5,7 @@ providing credentials in the context of network requests.
 """
 
 import logging
-from urllib import parse as urllib_parse
+import urllib.parse
 
 from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
 from pip._vendor.requests.utils import get_netrc_auth
@@ -250,7 +250,7 @@ class MultiDomainBasicAuth(AuthBase):
         if not self.prompting:
             return resp
 
-        parsed = urllib_parse.urlparse(resp.url)
+        parsed = urllib.parse.urlparse(resp.url)
 
         # Prompt the user for a new username and password
         username, password, save = self._prompt_for_password(parsed.netloc)

src/pip/_internal/network/session.py

@@ -13,8 +13,8 @@ import mimetypes
 import os
 import platform
 import sys
+import urllib.parse
 import warnings
-from urllib import parse as urllib_parse
 
 from pip._vendor import requests, six, urllib3
 from pip._vendor.cachecontrol import CacheControlAdapter
@@ -347,7 +347,7 @@ class PipSession(requests.Session):
     def is_secure_origin(self, location):
         # type: (Link) -> bool
         # Determine if this url used a secure transport mechanism
-        parsed = urllib_parse.urlparse(str(location))
+        parsed = urllib.parse.urlparse(str(location))
         origin_protocol, origin_host, origin_port = (
             parsed.scheme, parsed.hostname, parsed.port,
         )

src/pip/_internal/network/xmlrpc.py

@@ -2,7 +2,7 @@
 """
 
 import logging
-from urllib import parse as urllib_parse
+import urllib.parse
 
 # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
 #       why we ignore the type on this import
@@ -29,14 +29,14 @@ class PipXmlrpcTransport(xmlrpc_client.Transport):
     def __init__(self, index_url, session, use_datetime=False):
         # type: (str, PipSession, bool) -> None
         super().__init__(use_datetime)
-        index_parts = urllib_parse.urlparse(index_url)
+        index_parts = urllib.parse.urlparse(index_url)
         self._scheme = index_parts.scheme
         self._session = session
 
     def request(self, host, handler, request_body, verbose=False):
         # type: (str, str, Dict[str, str], bool) -> None
         parts = (self._scheme, host, handler, None, None, None)
-        url = urllib_parse.urlunparse(parts)
+        url = urllib.parse.urlunparse(parts)
         try:
             headers = {'Content-Type': 'text/xml'}
             response = self._session.post(url, data=request_body,

src/pip/_internal/req/req_file.py

@@ -6,7 +6,7 @@ import optparse
 import os
 import re
 import shlex
-from urllib import parse as urllib_parse
+import urllib.parse
 
 from pip._internal.cli import cmdoptions
 from pip._internal.exceptions import InstallationError, RequirementsFileParseError
@@ -354,7 +354,7 @@ class RequirementsFileParser:
                 # original file is over http
                 if SCHEME_RE.search(filename):
                     # do a url join so relative paths work
-                    req_path = urllib_parse.urljoin(filename, req_path)
+                    req_path = urllib.parse.urljoin(filename, req_path)
                 # original file and nested file are paths
                 elif not SCHEME_RE.search(req_path):
                     # do a join so relative paths work

src/pip/_internal/utils/misc.py

@@ -13,11 +13,10 @@ import posixpath
 import shutil
 import stat
 import sys
+import urllib.parse
 from collections import deque
 from io import StringIO
 from itertools import filterfalse, tee, zip_longest
-from urllib import parse as urllib_parse
-from urllib.parse import unquote as urllib_unquote
 
 from pip._vendor import pkg_resources
 from pip._vendor.packaging.utils import canonicalize_name
@@ -697,7 +696,7 @@ def parse_netloc(netloc):
     Return the host-port pair from a netloc.
     """
     url = build_url_from_netloc(netloc)
-    parsed = urllib_parse.urlparse(url)
+    parsed = urllib.parse.urlparse(url)
     return parsed.hostname, parsed.port
 
 
@@ -723,7 +722,7 @@ def split_auth_from_netloc(netloc):
         user_pass = auth, None
 
     user_pass = tuple(
-        None if x is None else urllib_unquote(x) for x in user_pass
+        None if x is None else urllib.parse.unquote(x) for x in user_pass
     )
 
     return netloc, user_pass
@@ -745,7 +744,7 @@ def redact_netloc(netloc):
         user = '****'
         password = ''
     else:
-        user = urllib_parse.quote(user)
+        user = urllib.parse.quote(user)
         password = ':****'
     return '{user}{password}@{netloc}'.format(user=user,
                                               password=password,
@@ -762,13 +761,13 @@ def _transform_url(url, transform_netloc):
     Returns a tuple containing the transformed url as item 0 and the
     original tuple returned by transform_netloc as item 1.
     """
-    purl = urllib_parse.urlsplit(url)
+    purl = urllib.parse.urlsplit(url)
     netloc_tuple = transform_netloc(purl.netloc)
     # stripped url
     url_pieces = (
         purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment
     )
-    surl = urllib_parse.urlunsplit(url_pieces)
+    surl = urllib.parse.urlunsplit(url_pieces)
     return surl, netloc_tuple
 
 

src/pip/_internal/utils/urls.py

@@ -1,7 +1,7 @@
 import os
 import sys
-from urllib import parse as urllib_parse
-from urllib import request as urllib_request
+import urllib.parse
+import urllib.request
 
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
@@ -23,7 +23,7 @@ def path_to_url(path):
     quoted path parts.
     """
     path = os.path.normpath(os.path.abspath(path))
-    url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
+    url = urllib.parse.urljoin('file:', urllib.request.pathname2url(path))
     return url
 
 
@@ -36,7 +36,7 @@ def url_to_path(url):
         "You can only turn file: urls into filenames (not {url!r})"
         .format(**locals()))
 
-    _, netloc, path, _, _ = urllib_parse.urlsplit(url)
+    _, netloc, path, _, _ = urllib.parse.urlsplit(url)
 
     if not netloc or netloc == 'localhost':
         # According to RFC 8089, same as empty authority.
@@ -50,5 +50,5 @@ def url_to_path(url):
             .format(**locals())
         )
 
-    path = urllib_request.url2pathname(netloc + path)
+    path = urllib.request.url2pathname(netloc + path)
     return path

src/pip/_internal/vcs/git.py

@@ -4,8 +4,8 @@
 import logging
 import os.path
 import re
-from urllib import parse as urllib_parse
-from urllib import request as urllib_request
+import urllib.parse
+import urllib.request
 
 from pip._vendor.packaging.version import parse as parse_version
 
@@ -28,8 +28,8 @@ if MYPY_CHECK_RUNNING:
     from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions
 
 
-urlsplit = urllib_parse.urlsplit
-urlunsplit = urllib_parse.urlunsplit
+urlsplit = urllib.parse.urlsplit
+urlunsplit = urllib.parse.urlunsplit
 
 
 logger = logging.getLogger(__name__)
@@ -383,7 +383,7 @@ class Git(VersionControl):
             initial_slashes = path[:-len(path.lstrip('/'))]
             newpath = (
                 initial_slashes +
-                urllib_request.url2pathname(path)
+                urllib.request.url2pathname(path)
                 .replace('\\', '/').lstrip('/')
             )
             after_plus = scheme.find('+') + 1

src/pip/_internal/vcs/versioncontrol.py

@@ -5,7 +5,7 @@ import os
 import shutil
 import subprocess
 import sys
-from urllib import parse as urllib_parse
+import urllib.parse
 
 from pip._vendor import pkg_resources
 
@@ -284,7 +284,7 @@ class VcsSupport:
         # type: () -> None
         # Register more schemes with urlparse for various version control
         # systems
-        urllib_parse.uses_netloc.extend(self.schemes)
+        urllib.parse.uses_netloc.extend(self.schemes)
         super().__init__()
 
     def __iter__(self):
@@ -518,7 +518,7 @@ class VersionControl:
 
         Returns: (url, rev, (username, password)).
         """
-        scheme, netloc, path, query, frag = urllib_parse.urlsplit(url)
+        scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
         if '+' not in scheme:
             raise ValueError(
                 "Sorry, {!r} is a malformed VCS url. "
@@ -537,7 +537,7 @@ class VersionControl:
                     "which is not supported. Include a revision after @ "
                     "or remove @ from the URL.".format(url)
                 )
-        url = urllib_parse.urlunsplit((scheme, netloc, path, query, ''))
+        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))
         return url, rev, user_pass
 
     @staticmethod
@@ -571,7 +571,7 @@ class VersionControl:
         Normalize a URL for comparison by unquoting it and removing any
         trailing slash.
         """
-        return urllib_parse.unquote(url).rstrip('/')
+        return urllib.parse.unquote(url).rstrip('/')
 
     @classmethod
     def compare_urls(cls, url1, url2):

tests/functional/test_install_index.py

@@ -1,6 +1,6 @@
 import os
 import textwrap
-from urllib import parse as urllib_parse
+import urllib.parse
 
 
 def test_find_links_relative_path(script, data, with_wheel):
@@ -58,7 +58,7 @@ def test_file_index_url_quoting(script, data, with_wheel):
     """
     Test url quoting of file index url with a space
     """
-    index_url = data.index_url(urllib_parse.quote("in dex"))
+    index_url = data.index_url(urllib.parse.quote("in dex"))
     result = script.pip(
         'install', '-vvv', '--index-url', index_url, 'simple'
     )

tests/lib/local_repos.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-from urllib import request as urllib_request
+import urllib.request
 
 from pip._internal.utils.misc import hide_url
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
@@ -18,7 +18,7 @@ def _create_svn_initools_repo(initools_dir):
     directory = os.path.dirname(initools_dir)
     subprocess.check_call('svnadmin create INITools'.split(), cwd=directory)
 
-    filename, _ = urllib_request.urlretrieve(
+    filename, _ = urllib.request.urlretrieve(
         'http://bitbucket.org/hltbra/pip-initools-dump/raw/8b55c908a320/'
         'INITools_modified.dump'
     )

tests/unit/test_collector.py

@@ -1,9 +1,9 @@
 import logging
 import os.path
 import re
+import urllib.request
 import uuid
 from textwrap import dedent
-from urllib import request as urllib_request
 
 import mock
 import pretend
@@ -568,7 +568,7 @@ def test_get_html_page_directory_append_index(tmpdir):
     dirpath = tmpdir / "something"
     dirpath.mkdir()
     dir_url = "file:///{}".format(
-        urllib_request.pathname2url(dirpath).lstrip("/"),
+        urllib.request.pathname2url(dirpath).lstrip("/"),
     )
     expected_url = "{}/index.html".format(dir_url.rstrip("/"))
 

tests/unit/test_urls.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from urllib import request as urllib_request
+import urllib.request
 
 import pytest
 
@@ -21,7 +21,7 @@ def test_get_url_scheme(url, expected):
 def test_path_to_url_unix():
     assert path_to_url('/tmp/file') == 'file:///tmp/file'
     path = os.path.join(os.getcwd(), 'file')
-    assert path_to_url('file') == 'file://' + urllib_request.pathname2url(path)
+    assert path_to_url('file') == 'file://' + urllib.request.pathname2url(path)
 
 
 @pytest.mark.skipif("sys.platform != 'win32'")
@@ -30,7 +30,7 @@ def test_path_to_url_win():
     assert path_to_url('c:\\tmp\\file') == 'file:///C:/tmp/file'
     assert path_to_url(r'\\unc\as\path') == 'file://unc/as/path'
     path = os.path.join(os.getcwd(), 'file')
-    assert path_to_url('file') == 'file:' + urllib_request.pathname2url(path)
+    assert path_to_url('file') == 'file:' + urllib.request.pathname2url(path)
 
 
 @pytest.mark.parametrize("url,win_expected,non_win_expected", [