From 3303c1182874f8853c436709cb8bff383511185e Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Tue, 10 Sep 2019 03:13:16 -0700 Subject: [PATCH] Remove index.py and req/constructors.py's dependence on download.py. --- src/pip/_internal/download.py | 60 ++----------------------- src/pip/_internal/index.py | 4 +- src/pip/_internal/operations/prepare.py | 8 +--- src/pip/_internal/req/constructors.py | 20 +++++++-- src/pip/_internal/utils/urls.py | 42 +++++++++++++++++ src/pip/_internal/vcs/__init__.py | 2 +- src/pip/_internal/vcs/versioncontrol.py | 12 +++++ tests/unit/test_download.py | 44 ------------------ tests/unit/test_urls.py | 48 ++++++++++++++++++++ 9 files changed, 128 insertions(+), 112 deletions(-) create mode 100644 src/pip/_internal/utils/urls.py create mode 100644 tests/unit/test_urls.py diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 15d6fd56f..7190702cb 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -26,7 +26,6 @@ from pip._vendor.six import PY2 # why we ignore the type on this import from pip._vendor.six.moves import xmlrpc_client # type: ignore from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._vendor.six.moves.urllib import request as urllib_request import pip from pip._internal.exceptions import HashMismatch, InstallationError @@ -37,7 +36,6 @@ from pip._internal.utils.encoding import auto_decode from pip._internal.utils.filesystem import check_path_owner, copy2_fixed from pip._internal.utils.glibc import libc_ver from pip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, ask, ask_input, ask_password, @@ -61,6 +59,7 @@ from pip._internal.utils.misc import ( from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import DownloadProgressProvider +from pip._internal.utils.urls import get_url_scheme, url_to_path from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: @@ -101,8 +100,8 @@ if MYPY_CHECK_RUNNING: __all__ = ['get_file_content', - 'is_url', 'url_to_path', 'path_to_url', - 'is_archive_file', 'unpack_vcs_link', + 'path_to_url', + 'unpack_vcs_link', 'unpack_file_url', 'is_file_url', 'unpack_http_url', 'unpack_url', 'parse_content_disposition', 'sanitize_content_filename'] @@ -787,7 +786,7 @@ def get_file_content(url, comes_from=None, session=None): "get_file_content() missing 1 required keyword argument: 'session'" ) - scheme = _get_url_scheme(url) + scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors @@ -824,57 +823,6 @@ def get_file_content(url, comes_from=None, session=None): _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) -def _get_url_scheme(url): - # type: (Union[str, Text]) -> Optional[Text] - if ':' not in url: - return None - return url.split(':', 1)[0].lower() - - -def is_url(name): - # type: (Union[str, Text]) -> bool - """Returns true if the name looks like a URL""" - scheme = _get_url_scheme(name) - if scheme is None: - return False - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def url_to_path(url): - # type: (str) -> str - """ - Convert a file: URL to a path. - """ - assert url.startswith('file:'), ( - "You can only turn file: urls into filenames (not %r)" % url) - - _, netloc, path, _, _ = urllib_parse.urlsplit(url) - - if not netloc or netloc == 'localhost': - # According to RFC 8089, same as empty authority. - netloc = '' - elif sys.platform == 'win32': - # If we have a UNC path, prepend UNC share notation. - netloc = '\\\\' + netloc - else: - raise ValueError( - 'non-local file URIs are not supported on this platform: %r' - % url - ) - - path = urllib_request.url2pathname(netloc + path) - return path - - -def is_archive_file(name): - # type: (str) -> bool - """Return True if `name` is a considered as an archive file.""" - ext = splitext(name)[1].lower() - if ext in ARCHIVE_EXTENSIONS: - return True - return False - - def unpack_vcs_link(link, location): # type: (Link, str) -> None vcs_backend = _get_used_vcs_backend(link) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index ed08e0272..dc9904eee 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -21,7 +21,6 @@ from pip._vendor.requests.exceptions import HTTPError, RetryError, SSLError from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves.urllib import request as urllib_request -from pip._internal.download import is_url, url_to_path from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, @@ -44,6 +43,8 @@ from pip._internal.utils.misc import ( ) from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import url_to_path +from pip._internal.vcs import is_url, vcs from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: @@ -79,7 +80,6 @@ def _match_vcs_scheme(url): Returns the matched VCS scheme, or None if there's no match. """ - from pip._internal.vcs import vcs for scheme in vcs.schemes: if url.lower().startswith(scheme) and url[len(scheme)] in '+:': return scheme diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 20e0301be..9bcab9b99 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,12 +13,7 @@ from pip._internal.distributions import ( make_distribution_for_install_requirement, ) from pip._internal.distributions.installed import InstalledDistribution -from pip._internal.download import ( - is_dir_url, - is_file_url, - unpack_url, - url_to_path, -) +from pip._internal.download import is_dir_url, is_file_url, unpack_url from pip._internal.exceptions import ( DirectoryUrlHashUnsupported, HashUnpinned, @@ -32,6 +27,7 @@ from pip._internal.utils.logging import indent_log from pip._internal.utils.marker_files import write_delete_marker_file from pip._internal.utils.misc import display_path, normalize_path from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: from typing import Optional diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 28cef9332..0d910dbc4 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -20,15 +20,20 @@ from pip._vendor.packaging.requirements import InvalidRequirement, Requirement from pip._vendor.packaging.specifiers import Specifier from pip._vendor.pkg_resources import RequirementParseError, parse_requirements -from pip._internal.download import is_archive_file, is_url, url_to_path from pip._internal.exceptions import InstallationError from pip._internal.models.index import PyPI, TestPyPI from pip._internal.models.link import Link from pip._internal.pyproject import make_pyproject_path from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.misc import is_installable_dir, path_to_url +from pip._internal.utils.misc import ( + ARCHIVE_EXTENSIONS, + is_installable_dir, + path_to_url, + splitext, +) from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.vcs import vcs +from pip._internal.utils.urls import url_to_path +from pip._internal.vcs import is_url, vcs from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: @@ -47,6 +52,15 @@ logger = logging.getLogger(__name__) operators = Specifier._operators.keys() +def is_archive_file(name): + # type: (str) -> bool + """Return True if `name` is a considered as an archive file.""" + ext = splitext(name)[1].lower() + if ext in ARCHIVE_EXTENSIONS: + return True + return False + + def _strip_extras(path): # type: (str) -> Tuple[str, Optional[str]] m = re.match(r'^(.+)(\[[^\]]+\])$', path) diff --git a/src/pip/_internal/utils/urls.py b/src/pip/_internal/utils/urls.py new file mode 100644 index 000000000..9c5385044 --- /dev/null +++ b/src/pip/_internal/utils/urls.py @@ -0,0 +1,42 @@ +import sys + +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Union + + +def get_url_scheme(url): + # type: (Union[str, Text]) -> Optional[Text] + if ':' not in url: + return None + return url.split(':', 1)[0].lower() + + +def url_to_path(url): + # type: (str) -> str + """ + Convert a file: URL to a path. + """ + assert url.startswith('file:'), ( + "You can only turn file: urls into filenames (not %r)" % url) + + _, netloc, path, _, _ = urllib_parse.urlsplit(url) + + if not netloc or netloc == 'localhost': + # According to RFC 8089, same as empty authority. + netloc = '' + elif sys.platform == 'win32': + # If we have a UNC path, prepend UNC share notation. + netloc = '\\\\' + netloc + else: + raise ValueError( + 'non-local file URIs are not supported on this platform: %r' + % url + ) + + path = urllib_request.url2pathname(netloc + path) + return path diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index cb573ab6d..75b5589c5 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -3,7 +3,7 @@ # (The test directory and imports protected by MYPY_CHECK_RUNNING may # still need to import from a vcs sub-package.) from pip._internal.vcs.versioncontrol import ( # noqa: F401 - RemoteNotFoundError, make_vcs_requirement_url, vcs, + RemoteNotFoundError, is_url, make_vcs_requirement_url, vcs, ) # Import all vcs modules to register each VCS in the VcsSupport object. import pip._internal.vcs.bazaar diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 40740e978..27610602f 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -22,6 +22,7 @@ from pip._internal.utils.misc import ( rmtree, ) from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme if MYPY_CHECK_RUNNING: from typing import ( @@ -39,6 +40,17 @@ __all__ = ['vcs'] logger = logging.getLogger(__name__) +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): """ Return the URL for a VCS requirement. diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 9c0ccc2cf..07588d1f4 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -19,12 +19,10 @@ from pip._internal.download import ( SafeFileCache, _copy_source_tree, _download_http_url, - _get_url_scheme, parse_content_disposition, sanitize_content_filename, unpack_file_url, unpack_http_url, - url_to_path, ) from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link @@ -301,48 +299,6 @@ def test_download_http_url__no_directory_traversal(tmpdir): assert actual == ['out_dir_file'] -@pytest.mark.parametrize("url,expected", [ - ('http://localhost:8080/', 'http'), - ('file:c:/path/to/file', 'file'), - ('file:/dev/null', 'file'), - ('', None), -]) -def test__get_url_scheme(url, expected): - assert _get_url_scheme(url) == expected - - -@pytest.mark.parametrize("url,win_expected,non_win_expected", [ - ('file:tmp', 'tmp', 'tmp'), - ('file:c:/path/to/file', r'C:\path\to\file', 'c:/path/to/file'), - ('file:/path/to/file', r'\path\to\file', '/path/to/file'), - ('file://localhost/tmp/file', r'\tmp\file', '/tmp/file'), - ('file://localhost/c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), - ('file://somehost/tmp/file', r'\\somehost\tmp\file', None), - ('file:///tmp/file', r'\tmp\file', '/tmp/file'), - ('file:///c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), -]) -def test_url_to_path(url, win_expected, non_win_expected): - if sys.platform == 'win32': - expected_path = win_expected - else: - expected_path = non_win_expected - - if expected_path is None: - with pytest.raises(ValueError): - url_to_path(url) - else: - assert url_to_path(url) == expected_path - - -@pytest.mark.skipif("sys.platform != 'win32'") -def test_url_to_path_path_to_url_symmetry_win(): - path = r'C:\tmp\file' - assert url_to_path(path_to_url(path)) == path - - unc_path = r'\\unc\share\path' - assert url_to_path(path_to_url(unc_path)) == unc_path - - @pytest.fixture def clean_project(tmpdir_factory, data): tmpdir = Path(str(tmpdir_factory.mktemp("clean_project"))) diff --git a/tests/unit/test_urls.py b/tests/unit/test_urls.py new file mode 100644 index 000000000..68d544072 --- /dev/null +++ b/tests/unit/test_urls.py @@ -0,0 +1,48 @@ +import sys + +import pytest + +from pip._internal.utils.misc import path_to_url +from pip._internal.utils.urls import get_url_scheme, url_to_path + + +@pytest.mark.parametrize("url,expected", [ + ('http://localhost:8080/', 'http'), + ('file:c:/path/to/file', 'file'), + ('file:/dev/null', 'file'), + ('', None), +]) +def test_get_url_scheme(url, expected): + assert get_url_scheme(url) == expected + + +@pytest.mark.parametrize("url,win_expected,non_win_expected", [ + ('file:tmp', 'tmp', 'tmp'), + ('file:c:/path/to/file', r'C:\path\to\file', 'c:/path/to/file'), + ('file:/path/to/file', r'\path\to\file', '/path/to/file'), + ('file://localhost/tmp/file', r'\tmp\file', '/tmp/file'), + ('file://localhost/c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), + ('file://somehost/tmp/file', r'\\somehost\tmp\file', None), + ('file:///tmp/file', r'\tmp\file', '/tmp/file'), + ('file:///c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), +]) +def test_url_to_path(url, win_expected, non_win_expected): + if sys.platform == 'win32': + expected_path = win_expected + else: + expected_path = non_win_expected + + if expected_path is None: + with pytest.raises(ValueError): + url_to_path(url) + else: + assert url_to_path(url) == expected_path + + +@pytest.mark.skipif("sys.platform != 'win32'") +def test_url_to_path_path_to_url_symmetry_win(): + path = r'C:\tmp\file' + assert url_to_path(path_to_url(path)) == path + + unc_path = r'\\unc\share\path' + assert url_to_path(path_to_url(unc_path)) == unc_path