From 6d94944efd254adf800f883d7ec127cd3850a2d8 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Sat, 14 Sep 2019 11:08:32 -0700 Subject: [PATCH] Change PackageFinder.create() to accept a LinkCollector. --- src/pip/_internal/cli/req_command.py | 7 ++-- src/pip/_internal/commands/list.py | 7 ++-- src/pip/_internal/index.py | 17 +------- src/pip/_internal/utils/outdated.py | 23 ++++++++--- tests/lib/__init__.py | 29 ++++++++++---- tests/unit/test_build_env.py | 9 +++-- tests/unit/test_collector.py | 29 ++------------ tests/unit/test_index.py | 59 ++++++++++++++++++---------- tests/unit/test_unit_outdated.py | 22 ++++++++--- 9 files changed, 110 insertions(+), 92 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 6f42a6824..9a2d4196f 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -23,7 +23,7 @@ from pip._internal.req.constructors import ( ) from pip._internal.req.req_file import parse_requirements from pip._internal.utils.misc import normalize_path -from pip._internal.utils.outdated import make_search_scope, pip_version_check +from pip._internal.utils.outdated import make_link_collector, pip_version_check from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -282,7 +282,7 @@ class RequirementCommand(IndexGroupCommand): :param ignore_requires_python: Whether to ignore incompatible "Requires-Python" values in links. Defaults to False. """ - search_scope = make_search_scope(options) + link_collector = make_link_collector(session, options=options) selection_prefs = SelectionPreferences( allow_yanked=True, format_control=options.format_control, @@ -292,8 +292,7 @@ class RequirementCommand(IndexGroupCommand): ) return PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, target_python=target_python, ) diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 0d3bf55d1..d6e38db86 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -16,7 +16,7 @@ from pip._internal.utils.misc import ( get_installed_distributions, write_output, ) -from pip._internal.utils.outdated import make_search_scope +from pip._internal.utils.outdated import make_link_collector from pip._internal.utils.packaging import get_installer logger = logging.getLogger(__name__) @@ -116,7 +116,7 @@ class ListCommand(IndexGroupCommand): """ Create a package finder appropriate to this list command. """ - search_scope = make_search_scope(options) + link_collector = make_link_collector(session, options=options) # Pass allow_yanked=False to ignore yanked versions. selection_prefs = SelectionPreferences( @@ -125,9 +125,8 @@ class ListCommand(IndexGroupCommand): ) return PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, ) def run(self, options, args): diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 697b99351..4db14eb1a 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -12,7 +12,6 @@ from pip._vendor.packaging import specifiers from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import parse as parse_version -from pip._internal.collector import LinkCollector from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, @@ -38,9 +37,9 @@ if MYPY_CHECK_RUNNING: Any, FrozenSet, Iterable, List, Optional, Set, Text, Tuple, ) from pip._vendor.packaging.version import _BaseVersion + from pip._internal.collector import LinkCollector from pip._internal.models.search_scope import SearchScope from pip._internal.req import InstallRequirement - from pip._internal.download import PipSession from pip._internal.pep425tags import Pep425Tag from pip._internal.utils.hashes import Hashes @@ -638,9 +637,8 @@ class PackageFinder(object): @classmethod def create( cls, - search_scope, # type: SearchScope + link_collector, # type: LinkCollector selection_prefs, # type: SelectionPreferences - session=None, # type: Optional[PipSession] target_python=None, # type: Optional[TargetPython] ): # type: (...) -> PackageFinder @@ -648,16 +646,10 @@ class PackageFinder(object): :param selection_prefs: The candidate selection preferences, as a SelectionPreferences object. - :param session: The Session to use to make requests. :param target_python: The target Python interpreter to use when checking compatibility. If None (the default), a TargetPython object will be constructed from the running Python. """ - if session is None: - raise TypeError( - "PackageFinder.create() missing 1 required keyword argument: " - "'session'" - ) if target_python is None: target_python = TargetPython() @@ -666,11 +658,6 @@ class PackageFinder(object): allow_all_prereleases=selection_prefs.allow_all_prereleases, ) - link_collector = LinkCollector( - session=session, - search_scope=search_scope, - ) - return cls( candidate_prefs=candidate_prefs, link_collector=link_collector, diff --git a/src/pip/_internal/utils/outdated.py b/src/pip/_internal/utils/outdated.py index 03d16fd2f..9383fc2e0 100644 --- a/src/pip/_internal/utils/outdated.py +++ b/src/pip/_internal/utils/outdated.py @@ -11,6 +11,7 @@ from pip._vendor import pkg_resources from pip._vendor.packaging import version as packaging_version from pip._vendor.six import ensure_binary +from pip._internal.collector import LinkCollector from pip._internal.index import PackageFinder from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences @@ -41,9 +42,14 @@ SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" logger = logging.getLogger(__name__) -def make_search_scope(options, suppress_no_index=False): - # type: (Values, bool) -> SearchScope +def make_link_collector( + session, # type: PipSession + options, # type: Values + suppress_no_index=False, # type: bool +): + # type: (...) -> LinkCollector """ + :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. """ @@ -62,7 +68,9 @@ def make_search_scope(options, suppress_no_index=False): find_links=find_links, index_urls=index_urls, ) - return search_scope + link_collector = LinkCollector(session=session, search_scope=search_scope) + + return link_collector def _get_statefile_name(key): @@ -176,7 +184,11 @@ def pip_version_check(session, options): # Refresh the version if we need to or just see if we need to warn if pypi_version is None: # Lets use PackageFinder to see what the latest pip version is - search_scope = make_search_scope(options, suppress_no_index=True) + link_collector = make_link_collector( + session, + options=options, + suppress_no_index=True, + ) # Pass allow_yanked=False so we don't suggest upgrading to a # yanked version. @@ -186,9 +198,8 @@ def pip_version_check(session, options): ) finder = PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, ) best_candidate = finder.find_best_candidate("pip").best_candidate if best_candidate is None: diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b7046b34b..a1ddd23f1 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -13,6 +13,7 @@ import subprocess import pytest from scripttest import FoundDir, TestFileEnvironment +from pip._internal.collector import LinkCollector from pip._internal.download import PipSession from pip._internal.index import PackageFinder from pip._internal.locations import get_major_minor_version @@ -89,11 +90,28 @@ def make_test_search_scope( if index_urls is None: index_urls = [] - return SearchScope.create( + return SearchScope.create(find_links=find_links, index_urls=index_urls) + + +def make_test_link_collector( + find_links=None, # type: Optional[List[str]] + index_urls=None, # type: Optional[List[str]] + session=None, # type: Optional[PipSession] +): + # type: (...) -> LinkCollector + """ + Create a LinkCollector object for testing purposes. + """ + if session is None: + session = PipSession() + + search_scope = make_test_search_scope( find_links=find_links, index_urls=index_urls, ) + return LinkCollector(session=session, search_scope=search_scope) + def make_test_finder( find_links=None, # type: Optional[List[str]] @@ -106,12 +124,10 @@ def make_test_finder( """ Create a PackageFinder for testing purposes. """ - if session is None: - session = PipSession() - - search_scope = make_test_search_scope( + link_collector = make_test_link_collector( find_links=find_links, index_urls=index_urls, + session=session, ) selection_prefs = SelectionPreferences( allow_yanked=True, @@ -119,9 +135,8 @@ def make_test_finder( ) return PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, target_python=target_python, ) diff --git a/tests/unit/test_build_env.py b/tests/unit/test_build_env.py index 9bfd19aa2..3e3c7ce9f 100644 --- a/tests/unit/test_build_env.py +++ b/tests/unit/test_build_env.py @@ -22,6 +22,7 @@ def run_with_build_env(script, setup_script_contents, import sys from pip._internal.build_env import BuildEnvironment + from pip._internal.collector import LinkCollector from pip._internal.download import PipSession from pip._internal.index import PackageFinder from pip._internal.models.search_scope import SearchScope @@ -29,14 +30,16 @@ def run_with_build_env(script, setup_script_contents, SelectionPreferences ) - search_scope = SearchScope.create([%r], []) + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope.create([%r], []), + ) selection_prefs = SelectionPreferences( allow_yanked=True, ) finder = PackageFinder.create( - search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) build_env = BuildEnvironment() diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 059ba40fd..1e90948eb 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -10,7 +10,6 @@ from pip._vendor.six.moves.urllib import request as urllib_request from pip._internal.collector import ( HTMLPage, - LinkCollector, _clean_link, _determine_base_url, _get_html_page, @@ -22,11 +21,7 @@ from pip._internal.collector import ( from pip._internal.download import PipSession from pip._internal.models.index import PyPI from pip._internal.models.link import Link -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from tests.lib import make_test_search_scope - -if MYPY_CHECK_RUNNING: - from typing import List, Optional +from tests.lib import make_test_link_collector @pytest.mark.parametrize( @@ -388,25 +383,6 @@ def make_fake_html_page(url): return HTMLPage(content, url=url, headers=headers) -def make_test_link_collector( - find_links=None, # type: Optional[List[str]] -): - # type: (...) -> LinkCollector - """ - Create a LinkCollector object for testing purposes. - """ - session = PipSession() - search_scope = make_test_search_scope( - find_links=find_links, - index_urls=[PyPI.simple_url], - ) - - return LinkCollector( - session=session, - search_scope=search_scope, - ) - - def check_links_include(links, names): """ Assert that the given list of Link objects includes, for each of the @@ -428,7 +404,8 @@ class TestLinkCollector(object): mock_get_html_response.return_value = fake_page link_collector = make_test_link_collector( - find_links=[data.find_links] + find_links=[data.find_links], + index_urls=[PyPI.simple_url], ) actual = link_collector.collect_links('twine') diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index b6d8afa26..bf1c457c4 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -3,12 +3,12 @@ import logging import pytest from pip._vendor.packaging.specifiers import SpecifierSet +from pip._internal.collector import LinkCollector from pip._internal.download import PipSession from pip._internal.index import ( CandidateEvaluator, CandidatePreferences, FormatControl, - LinkCollector, LinkEvaluator, PackageFinder, _check_link_requires_python, @@ -564,15 +564,18 @@ class TestPackageFinder: """ Test that the _candidate_prefs attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) selection_prefs = SelectionPreferences( allow_yanked=True, allow_all_prereleases=allow_all_prereleases, prefer_binary=prefer_binary, ) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) candidate_prefs = finder._candidate_prefs assert candidate_prefs.allow_all_prereleases == allow_all_prereleases @@ -582,27 +585,29 @@ class TestPackageFinder: """ Test that the _link_collector attribute is set correctly. """ - search_scope = SearchScope([], []) - session = PipSession() + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) finder = PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), - session=session, ) - actual_link_collector = finder._link_collector - assert actual_link_collector.search_scope is search_scope - assert actual_link_collector.session is session + assert finder._link_collector is link_collector def test_create__target_python(self): """ Test that the _target_python attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) target_python = TargetPython(py_version_info=(3, 7, 3)) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), - session=PipSession(), target_python=target_python, ) actual_target_python = finder._target_python @@ -615,10 +620,13 @@ class TestPackageFinder: """ Test passing target_python=None. """ - finder = PackageFinder.create( - search_scope=SearchScope([], []), - selection_prefs=SelectionPreferences(allow_yanked=True), + link_collector = LinkCollector( session=PipSession(), + search_scope=SearchScope([], []), + ) + finder = PackageFinder.create( + link_collector=link_collector, + selection_prefs=SelectionPreferences(allow_yanked=True), target_python=None, ) # Spot-check the default TargetPython object. @@ -631,11 +639,14 @@ class TestPackageFinder: """ Test that the _allow_yanked attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) selection_prefs = SelectionPreferences(allow_yanked=allow_yanked) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) assert finder._allow_yanked == allow_yanked @@ -644,14 +655,17 @@ class TestPackageFinder: """ Test that the _ignore_requires_python attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) selection_prefs = SelectionPreferences( allow_yanked=True, ignore_requires_python=ignore_requires_python, ) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) assert finder._ignore_requires_python == ignore_requires_python @@ -659,15 +673,18 @@ class TestPackageFinder: """ Test that the format_control attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) format_control = FormatControl(set(), {':all:'}) selection_prefs = SelectionPreferences( allow_yanked=True, format_control=format_control, ) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) actual_format_control = finder.format_control assert actual_format_control is format_control diff --git a/tests/unit/test_unit_outdated.py b/tests/unit/test_unit_outdated.py index d4f052479..4dd5b16cb 100644 --- a/tests/unit/test_unit_outdated.py +++ b/tests/unit/test_unit_outdated.py @@ -9,12 +9,13 @@ import pytest from mock import patch from pip._vendor import pkg_resources +from pip._internal.download import PipSession from pip._internal.index import InstallationCandidate from pip._internal.utils import outdated from pip._internal.utils.outdated import ( SelfCheckState, logger, - make_search_scope, + make_link_collector, pip_version_check, ) from tests.lib.path import Path @@ -32,26 +33,33 @@ from tests.lib.path import Path (False, False, False, ([], ['default_url', 'url1', 'url2'])), ], ) -def test_make_search_scope(find_links, no_index, suppress_no_index, expected): +def test_make_link_collector( + find_links, no_index, suppress_no_index, expected, +): """ :param expected: the expected (find_links, index_urls) values. """ expected_find_links, expected_index_urls = expected + session = PipSession() options = pretend.stub( find_links=find_links, index_url='default_url', extra_index_urls=['url1', 'url2'], no_index=no_index, ) - search_scope = make_search_scope( - options, suppress_no_index=suppress_no_index, + link_collector = make_link_collector( + session, options=options, suppress_no_index=suppress_no_index, ) + + assert link_collector.session is session + + search_scope = link_collector.search_scope assert search_scope.find_links == expected_find_links assert search_scope.index_urls == expected_index_urls @patch('pip._internal.utils.misc.expanduser') -def test_make_search_scope__find_links_expansion(mock_expanduser, tmpdir): +def test_make_link_collector__find_links_expansion(mock_expanduser, tmpdir): """ Test "~" expansion in --find-links paths. """ @@ -63,6 +71,7 @@ def test_make_search_scope__find_links_expansion(mock_expanduser, tmpdir): mock_expanduser.side_effect = expand_path + session = PipSession() options = pretend.stub( find_links=['~/temp1', '~/temp2'], index_url='default_url', @@ -74,8 +83,9 @@ def test_make_search_scope__find_links_expansion(mock_expanduser, tmpdir): temp2_dir = os.path.join(tmpdir, 'temp2') os.mkdir(temp2_dir) - search_scope = make_search_scope(options) + link_collector = make_link_collector(session, options=options) + search_scope = link_collector.search_scope # Only ~/temp2 gets expanded. Also, the path is normalized when expanded. expected_temp2_dir = os.path.normcase(temp2_dir) assert search_scope.find_links == ['~/temp1', expected_temp2_dir]