Merge pull request #6631 from cjerdonek/finder-search-scope

Change PackageFinder.create() to accept a SearchScope

src/pip/_internal/cli/base_command.py

@@ -10,6 +10,7 @@ import sys
 import traceback
 
 from pip._internal.cli import cmdoptions
+from pip._internal.cli.cmdoptions import make_search_scope
 from pip._internal.cli.parser import (
     ConfigOptionParser, UpdatingDefaultsHelpFormatter,
 )
@@ -31,9 +32,7 @@ from pip._internal.req.constructors import (
 from pip._internal.req.req_file import parse_requirements
 from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging
-from pip._internal.utils.misc import (
-    get_prog, normalize_path, redact_password_from_url,
-)
+from pip._internal.utils.misc import get_prog, normalize_path
 from pip._internal.utils.outdated import pip_version_check
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
@@ -337,13 +336,7 @@ class RequirementCommand(Command):
         :param ignore_requires_python: Whether to ignore incompatible
             "Requires-Python" values in links. Defaults to False.
         """
-        index_urls = [options.index_url] + options.extra_index_urls
-        if options.no_index:
-            logger.debug(
-                'Ignoring indexes: %s',
-                ','.join(redact_password_from_url(url) for url in index_urls),
-            )
-            index_urls = []
+        search_scope = make_search_scope(options)
 
         target_python = TargetPython(
             platform=platform,
@@ -353,9 +346,8 @@ class RequirementCommand(Command):
         )
 
         return PackageFinder.create(
-            find_links=options.find_links,
+            search_scope=search_scope,
             format_control=options.format_control,
-            index_urls=index_urls,
             trusted_hosts=options.trusted_hosts,
             allow_all_prereleases=options.pre,
             session=session,

src/pip/_internal/cli/cmdoptions.py

@@ -9,6 +9,7 @@ pass on state. To be consistent, all options will follow this design.
 """
 from __future__ import absolute_import
 
+import logging
 import textwrap
 import warnings
 from distutils.util import strtobool
@@ -20,7 +21,9 @@ from pip._internal.exceptions import CommandError
 from pip._internal.locations import USER_CACHE_DIR, src_prefix
 from pip._internal.models.format_control import FormatControl
 from pip._internal.models.index import PyPI
+from pip._internal.models.search_scope import SearchScope
 from pip._internal.utils.hashes import STRONG_HASHES
+from pip._internal.utils.misc import redact_password_from_url
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.ui import BAR_TYPES
 
@@ -29,6 +32,8 @@ if MYPY_CHECK_RUNNING:
     from optparse import OptionParser, Values
     from pip._internal.cli.parser import ConfigOptionParser
 
+logger = logging.getLogger(__name__)
+
 
 def raise_option_error(parser, option, msg):
     """
@@ -350,6 +355,27 @@ def find_links():
     )
 
 
+def make_search_scope(options, suppress_no_index=False):
+    """
+    :param suppress_no_index: Whether to ignore the --no-index option
+        when constructing the SearchScope object.
+    """
+    index_urls = [options.index_url] + options.extra_index_urls
+    if options.no_index and not suppress_no_index:
+        logger.debug(
+            'Ignoring indexes: %s',
+            ','.join(redact_password_from_url(url) for url in index_urls),
+        )
+        index_urls = []
+
+    search_scope = SearchScope(
+        find_links=options.find_links,
+        index_urls=index_urls,
+    )
+
+    return search_scope
+
+
 def trusted_host():
     # type: () -> Option
     return Option(

src/pip/_internal/commands/list.py

@@ -8,6 +8,7 @@ from pip._vendor.six.moves import zip_longest
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.base_command import Command
+from pip._internal.cli.cmdoptions import make_search_scope
 from pip._internal.exceptions import CommandError
 from pip._internal.index import PackageFinder
 from pip._internal.utils.misc import (
@@ -109,13 +110,14 @@ class ListCommand(Command):
         self.parser.insert_option_group(0, index_opts)
         self.parser.insert_option_group(0, cmd_opts)
 
-    def _build_package_finder(self, options, index_urls, session):
+    def _build_package_finder(self, options, session):
         """
         Create a package finder appropriate to this list command.
         """
+        search_scope = make_search_scope(options)
+
         return PackageFinder.create(
-            find_links=options.find_links,
-            index_urls=index_urls,
+            search_scope=search_scope,
             allow_all_prereleases=options.pre,
             trusted_hosts=options.trusted_hosts,
             session=session,
@@ -169,13 +171,8 @@ class ListCommand(Command):
         return {pkg for pkg in packages if pkg.key not in dep_keys}
 
     def iter_packages_latest_infos(self, packages, options):
-        index_urls = [options.index_url] + options.extra_index_urls
-        if options.no_index:
-            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
-            index_urls = []
-
         with self._build_session(options) as session:
-            finder = self._build_package_finder(options, index_urls, session)
+            finder = self._build_package_finder(options, session)
 
             for dist in packages:
                 typ = 'unknown'

src/pip/_internal/commands/wheel.py

@@ -108,11 +108,6 @@ class WheelCommand(RequirementCommand):
     def run(self, options, args):
         cmdoptions.check_install_build_global(options)
 
-        index_urls = [options.index_url] + options.extra_index_urls
-        if options.no_index:
-            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
-            index_urls = []
-
         if options.build_dir:
             options.build_dir = os.path.abspath(options.build_dir)
 

src/pip/_internal/download.py

@@ -25,12 +25,13 @@ from pip._vendor.requests.utils import get_netrc_auth
 from pip._vendor.six.moves import xmlrpc_client  # type: ignore
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
-from pip._vendor.urllib3.util import IS_PYOPENSSL
 
 import pip
 from pip._internal.exceptions import HashMismatch, InstallationError
 from pip._internal.locations import write_delete_marker_file
 from pip._internal.models.index import PyPI
+# Import ssl from compat so the initial import occurs in only one place.
+from pip._internal.utils.compat import HAS_TLS, ssl
 from pip._internal.utils.encoding import auto_decode
 from pip._internal.utils.filesystem import check_path_owner
 from pip._internal.utils.glibc import libc_ver
@@ -54,14 +55,6 @@ if MYPY_CHECK_RUNNING:
     from pip._internal.utils.hashes import Hashes
     from pip._internal.vcs.versioncontrol import AuthInfo, VersionControl
 
-try:
-    import ssl  # noqa
-except ImportError:
-    ssl = None
-
-
-HAS_TLS = (ssl is not None) or IS_PYOPENSSL
-
 __all__ = ['get_file_content',
            'is_url', 'url_to_path', 'path_to_url',
            'is_archive_file', 'unpack_vcs_link',

src/pip/_internal/index.py

@@ -26,7 +26,6 @@ from pip._internal.exceptions import (
 from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.models.format_control import FormatControl
 from pip._internal.models.link import Link
-from pip._internal.models.search_scope import SearchScope
 from pip._internal.models.target_python import TargetPython
 from pip._internal.utils.compat import ipaddress
 from pip._internal.utils.logging import indent_log
@@ -46,6 +45,7 @@ if MYPY_CHECK_RUNNING:
     )
     from pip._vendor.packaging.version import _BaseVersion
     from pip._vendor.requests import Response
+    from pip._internal.models.search_scope import SearchScope
     from pip._internal.req import InstallRequirement
     from pip._internal.download import PipSession
 
@@ -592,8 +592,7 @@ class PackageFinder(object):
     @classmethod
     def create(
         cls,
-        find_links,  # type: List[str]
-        index_urls,  # type: List[str]
+        search_scope,  # type: SearchScope
         allow_all_prereleases=False,  # type: bool
         trusted_hosts=None,  # type: Optional[List[str]]
         session=None,  # type: Optional[PipSession]
@@ -623,11 +622,6 @@ class PackageFinder(object):
                 "'session'"
             )
 
-        search_scope = SearchScope.create(
-            find_links=find_links,
-            index_urls=index_urls,
-        )
-
         candidate_evaluator = CandidateEvaluator(
             target_python=target_python,
             prefer_binary=prefer_binary,

src/pip/_internal/models/search_scope.py

@@ -6,8 +6,8 @@ import posixpath
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 
-from pip._internal.download import HAS_TLS
 from pip._internal.models.index import PyPI
+from pip._internal.utils.compat import HAS_TLS
 from pip._internal.utils.misc import normalize_path, redact_password_from_url
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 

src/pip/_internal/utils/compat.py

@@ -10,12 +10,21 @@ import shutil
 import sys
 
 from pip._vendor.six import text_type
+from pip._vendor.urllib3.util import IS_PYOPENSSL
 
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
     from typing import Tuple, Text
 
+try:
+    import _ssl  # noqa
+except ImportError:
+    ssl = None
+else:
+    # This additional assignment was needed to prevent a mypy error.
+    ssl = _ssl
+
 try:
     import ipaddress
 except ImportError:
@@ -36,6 +45,8 @@ __all__ = [
 
 logger = logging.getLogger(__name__)
 
+HAS_TLS = (ssl is not None) or IS_PYOPENSSL
+
 if sys.version_info >= (3, 4):
     uses_pycache = True
     from importlib.util import cache_from_source

src/pip/_internal/utils/outdated.py

@@ -9,6 +9,7 @@ import sys
 from pip._vendor import lockfile, pkg_resources
 from pip._vendor.packaging import version as packaging_version
 
+from pip._internal.cli.cmdoptions import make_search_scope
 from pip._internal.index import PackageFinder
 from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.filesystem import check_path_owner
@@ -122,9 +123,10 @@ def pip_version_check(session, options):
         # Refresh the version if we need to or just see if we need to warn
         if pypi_version is None:
             # Lets use PackageFinder to see what the latest pip version is
+            search_scope = make_search_scope(options, suppress_no_index=True)
+
             finder = PackageFinder.create(
-                find_links=options.find_links,
-                index_urls=[options.index_url] + options.extra_index_urls,
+                search_scope=search_scope,
                 allow_all_prereleases=False,  # Explicitly set to False
                 trusted_hosts=options.trusted_hosts,
                 session=session,

tests/lib/__init__.py

@@ -13,6 +13,7 @@ import subprocess
 import pytest
 from scripttest import FoundDir, TestFileEnvironment
 
+from pip._internal.models.search_scope import SearchScope
 from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
 from pip._internal.download import PipSession
 from pip._internal.index import PackageFinder
@@ -96,9 +97,13 @@ def make_test_finder(
     if session is None:
         session = PipSession()
 
+    search_scope = SearchScope.create(
+        find_links=find_links,
+        index_urls=index_urls,
+    )
+
     return PackageFinder.create(
-        find_links,
-        index_urls,
+        search_scope=search_scope,
         allow_all_prereleases=allow_all_prereleases,
         trusted_hosts=trusted_hosts,
         session=session,

tests/unit/test_build_env.py

@@ -24,8 +24,10 @@ def run_with_build_env(script, setup_script_contents,
             from pip._internal.build_env import BuildEnvironment
             from pip._internal.download import PipSession
             from pip._internal.index import PackageFinder
+            from pip._internal.models.search_scope import SearchScope
 
-            finder = PackageFinder.create([%r], [], session=PipSession())
+            search_scope = SearchScope.create([%r], [])
+            finder = PackageFinder.create(search_scope, session=PipSession())
             build_env = BuildEnvironment()
 
             try:

tests/unit/test_cmdoptions.py

@@ -1,6 +1,35 @@
+import pretend
 import pytest
 
-from pip._internal.cli.cmdoptions import _convert_python_version
+from pip._internal.cli.cmdoptions import (
+    _convert_python_version, make_search_scope,
+)
+
+
+@pytest.mark.parametrize(
+    'no_index, suppress_no_index, expected_index_urls', [
+        (False, False, ['default_url', 'url1', 'url2']),
+        (False, True, ['default_url', 'url1', 'url2']),
+        (True, False, []),
+        # Passing suppress_no_index=True suppresses no_index=True.
+        (True, True, ['default_url', 'url1', 'url2']),
+    ],
+)
+def test_make_search_scope(no_index, suppress_no_index, expected_index_urls):
+    """
+    :param expected: the expected index_urls value.
+    """
+    options = pretend.stub(
+        find_links=['link1'],
+        index_url='default_url',
+        extra_index_urls=['url1', 'url2'],
+        no_index=no_index,
+    )
+    search_scope = make_search_scope(
+        options, suppress_no_index=suppress_no_index,
+    )
+    assert search_scope.find_links == ['link1']
+    assert search_scope.index_urls == expected_index_urls
 
 
 @pytest.mark.parametrize('value, expected', [

tests/unit/test_index.py

@@ -11,6 +11,7 @@ from pip._internal.index import (
     _check_link_requires_python, _clean_link, _determine_base_url,
     _egg_info_matches, _find_name_version_sep, _get_html_page,
 )
+from pip._internal.models.search_scope import SearchScope
 from pip._internal.models.target_python import TargetPython
 from tests.lib import CURRENT_PY_VERSION_INFO, make_test_finder
 
@@ -154,9 +155,12 @@ class TestPackageFinder:
         """
         Test that target_python is passed to CandidateEvaluator as is.
         """
+        search_scope = SearchScope([], [])
         target_python = TargetPython(py_version_info=(3, 7, 3))
         finder = PackageFinder.create(
-            [], [], session=object(), target_python=target_python,
+            search_scope=search_scope,
+            session=object(),
+            target_python=target_python,
         )
         evaluator = finder.candidate_evaluator
         actual_target_python = evaluator._target_python
@@ -237,8 +241,11 @@ class TestPackageFinder:
         """
         # Use PackageFinder.create() rather than make_test_finder()
         # to make sure we're really passing trusted_hosts=None.
+        search_scope = SearchScope([], [])
         finder = PackageFinder.create(
-            [], [], trusted_hosts=None, session=object(),
+            search_scope=search_scope,
+            trusted_hosts=None,
+            session=object(),
         )
 
         actual = list(finder.iter_secure_origins())

tests/unit/test_unit_outdated.py

@@ -58,8 +58,8 @@ class MockDistribution(object):
 def _options():
     ''' Some default options that we pass to outdated.pip_version_check '''
     return pretend.stub(
-        find_links=False, extra_index_urls=[], index_url='default_url',
-        pre=False, trusted_hosts=False, cache_dir='',
+        find_links=False, index_url='default_url', extra_index_urls=[],
+        no_index=False, pre=False, trusted_hosts=False, cache_dir='',
     )