Merge pull request #7144 from pradyunsg/index-sub-package

Introduce an `index` sub-package

docs/html/development/architecture/package-finding.rst

@@ -1,13 +1,13 @@
-Finding and choosing files (``index.py`` and ``PackageFinder``)
+Finding and choosing files (``index`` and ``PackageFinder``)
 ---------------------------------------------------------------
 
-The ``index.py`` module is a top-level module in pip responsible for deciding
+The ``pip._internal.index`` sub-package in pip is responsible for deciding
 what file to download and from where, given a requirement for a project. The
-module's functionality is largely exposed through and coordinated by the
-module's ``PackageFinder`` class.
+package's functionality is largely exposed through and coordinated by the
+package's ``PackageFinder`` class.
 
 
-.. _index-py-overview:
+.. _index-overview:
 
 Overview
 ********
@@ -38,7 +38,7 @@ file to download for a package, given a requirement:
    <candidate-evaluator-class>` class).
 
 The remainder of this section is organized by documenting some of the
-classes inside ``index.py``, in the following order:
+classes inside the ``index`` package, in the following order:
 
 * the main :ref:`PackageFinder <package-finder-class>` class,
 * the :ref:`LinkCollector <link-collector-class>` class,
@@ -54,7 +54,7 @@ The ``PackageFinder`` class
 ***************************
 
 The ``PackageFinder`` class is the primary way through which code in pip
-interacts with ``index.py``. It is an umbrella class that encapsulates and
+interacts with ``index`` package. It is an umbrella class that encapsulates and
 groups together various package-finding functionality.
 
 The ``PackageFinder`` class is responsible for searching the network and file
@@ -89,7 +89,7 @@ case, the ``PackageFinder`` instance is created by the
 ``self_outdated_check.py`` module's ``pip_self_version_check()`` function.
 
 The ``PackageFinder`` class is responsible for doing all of the things listed
-in the :ref:`Overview <index-py-overview>` section like fetching and parsing
+in the :ref:`Overview <index-overview>` section like fetching and parsing
 `PEP 503`_ simple repository HTML pages, evaluating which links in the simple
 repository pages are relevant for each requirement, and further filtering and
 sorting by preference the candidates for install coming from the relevant
@@ -105,7 +105,7 @@ One of ``PackageFinder``'s main top-level methods is
    :ref:`LinkEvaluator <link-evaluator-class>` object to filter out some of
    those links, and then returns a list of ``InstallationCandidates`` (aka
    candidates for install). This corresponds to steps 1-3 of the
-   :ref:`Overview <index-py-overview>` above.
+   :ref:`Overview <index-overview>` above.
 2. Constructs a ``CandidateEvaluator`` object and uses that to determine
    the best candidate. It does this by calling the ``CandidateEvaluator``
    class's ``compute_best_candidate()`` method on the return value of
@@ -133,8 +133,8 @@ method is the ``collect_links()`` method. The :ref:`PackageFinder
 <package-finder-class>` class invokes this method as the first step of its
 ``find_all_candidates()`` method.
 
-The ``LinkCollector`` class is the only class in the ``index.py`` module that
-makes network requests and is the only class in the module that depends
+The ``LinkCollector`` class is the only class in the ``index`` sub-package that
+makes network requests and is the only class in the sub-package that depends
 directly on ``PipSession``, which stores pip's configuration options and
 state for making requests.
 

src/pip/_internal/build_env.py

@@ -23,7 +23,7 @@ from pip._internal.utils.ui import open_spinner
 
 if MYPY_CHECK_RUNNING:
     from typing import Tuple, Set, Iterable, Optional, List
-    from pip._internal.index import PackageFinder
+    from pip._internal.index.package_finder import PackageFinder
 
 logger = logging.getLogger(__name__)
 

src/pip/_internal/cache.py

@@ -20,7 +20,7 @@ from pip._internal.wheel import InvalidWheelFilename, Wheel
 
 if MYPY_CHECK_RUNNING:
     from typing import Optional, Set, List, Any
-    from pip._internal.index import FormatControl
+    from pip._internal.models.format_control import FormatControl
     from pip._internal.pep425tags import Pep425Tag
 
 logger = logging.getLogger(__name__)

src/pip/_internal/cli/req_command.py

@@ -14,7 +14,7 @@ from functools import partial
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.command_context import CommandContextMixIn
 from pip._internal.exceptions import CommandError
-from pip._internal.index import PackageFinder
+from pip._internal.index.package_finder import PackageFinder
 from pip._internal.legacy_resolve import Resolver
 from pip._internal.models.selection_prefs import SelectionPreferences
 from pip._internal.network.session import PipSession

src/pip/_internal/commands/list.py

@@ -12,7 +12,7 @@ from pip._vendor.six.moves import zip_longest
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import IndexGroupCommand
 from pip._internal.exceptions import CommandError
-from pip._internal.index import PackageFinder
+from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.selection_prefs import SelectionPreferences
 from pip._internal.self_outdated_check import make_link_collector
 from pip._internal.utils.misc import (

src/pip/_internal/index/__init__.py

@@ -0,0 +1,2 @@
+"""Index interaction code
+"""

src/pip/_internal/index/package_finder.py

@@ -38,7 +38,7 @@ if MYPY_CHECK_RUNNING:
         FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union,
     )
     from pip._vendor.packaging.version import _BaseVersion
-    from pip._internal.collector import LinkCollector
+    from pip._internal.index.collector import LinkCollector
     from pip._internal.models.search_scope import SearchScope
     from pip._internal.req import InstallRequirement
     from pip._internal.pep425tags import Pep425Tag

src/pip/_internal/legacy_resolve.py

@@ -46,7 +46,7 @@ if MYPY_CHECK_RUNNING:
 
     from pip._internal.distributions import AbstractDistribution
     from pip._internal.network.session import PipSession
-    from pip._internal.index import PackageFinder
+    from pip._internal.index.package_finder import PackageFinder
     from pip._internal.operations.prepare import RequirementPreparer
     from pip._internal.req.req_install import InstallRequirement
     from pip._internal.req.req_set import RequirementSet

src/pip/_internal/models/link.py

@@ -19,7 +19,7 @@ from pip._internal.utils.urls import path_to_url, url_to_path
 
 if MYPY_CHECK_RUNNING:
     from typing import Optional, Text, Tuple, Union
-    from pip._internal.collector import HTMLPage
+    from pip._internal.index.collector import HTMLPage
     from pip._internal.utils.hashes import Hashes
 
 

src/pip/_internal/operations/prepare.py

@@ -61,7 +61,7 @@ if MYPY_CHECK_RUNNING:
     from mypy_extensions import TypedDict
 
     from pip._internal.distributions import AbstractDistribution
-    from pip._internal.index import PackageFinder
+    from pip._internal.index.package_finder import PackageFinder
     from pip._internal.models.link import Link
     from pip._internal.req.req_install import InstallRequirement
     from pip._internal.req.req_tracker import RequirementTracker

src/pip/_internal/req/req_file.py

@@ -36,7 +36,7 @@ if MYPY_CHECK_RUNNING:
     )
     from pip._internal.req import InstallRequirement
     from pip._internal.cache import WheelCache
-    from pip._internal.index import PackageFinder
+    from pip._internal.index.package_finder import PackageFinder
     from pip._internal.network.session import PipSession
 
     ReqFileLines = Iterator[Tuple[int, Text]]

src/pip/_internal/req/req_install.py

@@ -67,7 +67,7 @@ if MYPY_CHECK_RUNNING:
     )
     from pip._internal.build_env import BuildEnvironment
     from pip._internal.cache import WheelCache
-    from pip._internal.index import PackageFinder
+    from pip._internal.index.package_finder import PackageFinder
     from pip._vendor.pkg_resources import Distribution
     from pip._vendor.packaging.specifiers import SpecifierSet
     from pip._vendor.packaging.markers import Marker

src/pip/_internal/self_outdated_check.py

@@ -14,8 +14,8 @@ from pip._vendor import pkg_resources
 from pip._vendor.packaging import version as packaging_version
 from pip._vendor.six import ensure_binary
 
-from pip._internal.collector import LinkCollector
-from pip._internal.index import PackageFinder
+from pip._internal.index.collector import LinkCollector
+from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.search_scope import SearchScope
 from pip._internal.models.selection_prefs import SelectionPreferences
 from pip._internal.utils.compat import WINDOWS

tests/lib/__init__.py

@@ -13,8 +13,8 @@ from textwrap import dedent
 import pytest
 from scripttest import FoundDir, TestFileEnvironment
 
-from pip._internal.collector import LinkCollector
-from pip._internal.index import PackageFinder
+from pip._internal.index.collector import LinkCollector
+from pip._internal.index.package_finder import PackageFinder
 from pip._internal.locations import get_major_minor_version
 from pip._internal.models.search_scope import SearchScope
 from pip._internal.models.selection_prefs import SelectionPreferences

tests/unit/test_build_env.py

@@ -22,8 +22,8 @@ def run_with_build_env(script, setup_script_contents,
             import sys
 
             from pip._internal.build_env import BuildEnvironment
-            from pip._internal.collector import LinkCollector
-            from pip._internal.index import PackageFinder
+            from pip._internal.index.collector import LinkCollector
+            from pip._internal.index.package_finder import PackageFinder
             from pip._internal.models.search_scope import SearchScope
             from pip._internal.models.selection_prefs import (
                 SelectionPreferences

tests/unit/test_collector.py

@@ -9,7 +9,7 @@ from mock import Mock, patch
 from pip._vendor import html5lib, requests
 from pip._vendor.six.moves.urllib import request as urllib_request
 
-from pip._internal.collector import (
+from pip._internal.index.collector import (
     HTMLPage,
     _clean_link,
     _determine_base_url,
@@ -334,7 +334,7 @@ def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme):
     assert page is None
     assert caplog.record_tuples == [
         (
-            "pip._internal.collector",
+            "pip._internal.index.collector",
             logging.DEBUG,
             "Cannot look at {} URL {}".format(vcs_scheme, url),
         ),
@@ -367,7 +367,8 @@ def test_get_html_page_directory_append_index(tmpdir):
 
     session = mock.Mock(PipSession)
     fake_response = make_fake_html_response(expected_url)
-    with mock.patch("pip._internal.collector._get_html_response") as mock_func:
+    mock_func = mock.patch("pip._internal.index.collector._get_html_response")
+    with mock_func as mock_func:
         mock_func.return_value = fake_response
         actual = _get_html_page(Link(dir_url), session=session)
         assert mock_func.mock_calls == [
@@ -434,7 +435,7 @@ def check_links_include(links, names):
 
 class TestLinkCollector(object):
 
-    @patch('pip._internal.collector._get_html_response')
+    @patch('pip._internal.index.collector._get_html_response')
     def test_collect_links(self, mock_get_html_response, caplog, data):
         caplog.set_level(logging.DEBUG)
 
@@ -474,5 +475,5 @@ class TestLinkCollector(object):
         1 location(s) to search for versions of twine:
         * https://pypi.org/simple/twine/""")
         assert caplog.record_tuples == [
-            ('pip._internal.collector', logging.DEBUG, expected_message),
+            ('pip._internal.index.collector', logging.DEBUG, expected_message),
         ]

tests/unit/test_finder.py

@@ -12,7 +12,7 @@ from pip._internal.exceptions import (
     BestVersionAlreadyInstalled,
     DistributionNotFound,
 )
-from pip._internal.index import (
+from pip._internal.index.package_finder import (
     CandidateEvaluator,
     InstallationCandidate,
     Link,
@@ -62,7 +62,10 @@ def test_no_partial_name_match(data):
 
 def test_tilde():
     """Finder can accept a path with ~ in it and will normalize it."""
-    with patch('pip._internal.collector.os.path.exists', return_value=True):
+    patched_exists = patch(
+        'pip._internal.index.collector.os.path.exists', return_value=True
+    )
+    with patched_exists:
         finder = make_test_finder(find_links=['~/python-pkgs'])
     req = install_req_from_line("gmpy")
     with pytest.raises(DistributionNotFound):

tests/unit/test_index.py

@@ -3,8 +3,8 @@ import logging
 import pytest
 from pip._vendor.packaging.specifiers import SpecifierSet
 
-from pip._internal.collector import LinkCollector
-from pip._internal.index import (
+from pip._internal.index.collector import LinkCollector
+from pip._internal.index.package_finder import (
     CandidateEvaluator,
     CandidatePreferences,
     FormatControl,

tests/unit/test_req_file.py

@@ -7,7 +7,7 @@ import pytest
 from mock import Mock, patch
 from pretend import stub
 
-import pip._internal.index
+import pip._internal.req.req_file  # this will be monkeypatched
 from pip._internal.exceptions import (
     InstallationError,
     RequirementsFileParseError,
@@ -272,7 +272,6 @@ class TestProcessLine(object):
     def test_nested_requirements_file(self, monkeypatch):
         line = '-r another_file'
         req = install_req_from_line('SomeProject')
-        import pip._internal.req.req_file
 
         def stub_parse_requirements(req_url, finder, comes_from, options,
                                     session, wheel_cache, constraint):
@@ -285,7 +284,6 @@ class TestProcessLine(object):
     def test_nested_constraints_file(self, monkeypatch):
         line = '-c another_file'
         req = install_req_from_line('SomeProject')
-        import pip._internal.req.req_file
 
         def stub_parse_requirements(req_url, finder, comes_from, options,
                                     session, wheel_cache, constraint):

tests/unit/test_self_check_outdated.py

@@ -10,7 +10,7 @@ from mock import patch
 from pip._vendor import pkg_resources
 
 from pip._internal import self_outdated_check
-from pip._internal.index import InstallationCandidate
+from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.network.session import PipSession
 from pip._internal.self_outdated_check import (
     SelfCheckState,