Avoid downloading candidates of a same version

Since 41a30089de, Candidate objects prepare their underlying
distribution eagerly on creation, so the error can be caught
deterministically to trigger backtracking.

This however has a negative side-effect. Since dist preparation involves
metadata validation, a remote distribution must be downloaded on
Candidate creation. This means that an sdist will be downloaded,
validated, and discarded (since its version is known to be incompatible)
during backtracking.

This commit moves version deduplication of candidates from indexes to
before the Candidate object is created, to avoid unneeded preparation.

Note that we still need another round of deduplication in FoundCandidates
to remove duplicated candidates when a distribution is already installed.

src/pip/_internal/resolution/resolvelib/factory.py

@@ -228,9 +228,12 @@ class Factory:
             all_yanked = all(ican.link.is_yanked for ican in icans)
 
             # PackageFinder returns earlier versions first, so we reverse.
+            versions_found = set()  # type: Set[_BaseVersion]
             for ican in reversed(icans):
                 if not all_yanked and ican.link.is_yanked:
                     continue
+                if ican.version in versions_found:
+                    continue
                 candidate = self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,
@@ -241,6 +244,7 @@ class Factory:
                 if candidate is None:
                     continue
                 yield candidate
+                versions_found.add(ican.version)
 
         return FoundCandidates(
             iter_index_candidates,

src/pip/_internal/resolution/resolvelib/found_candidates.py

@@ -16,23 +16,11 @@ from pip._vendor.six.moves import collections_abc  # type: ignore
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
-    from typing import Callable, Iterator, Optional, Set
-
-    from pip._vendor.packaging.version import _BaseVersion
+    from typing import Callable, Iterator, Optional
 
     from .base import Candidate
 
 
-def _deduplicated_by_version(candidates):
-    # type: (Iterator[Candidate]) -> Iterator[Candidate]
-    returned = set()  # type: Set[_BaseVersion]
-    for candidate in candidates:
-        if candidate.version in returned:
-            continue
-        returned.add(candidate.version)
-        yield candidate
-
-
 def _insert_installed(installed, others):
     # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
     """Iterator for ``FoundCandidates``.
@@ -86,12 +74,15 @@ class FoundCandidates(collections_abc.Sequence):
     def __iter__(self):
         # type: () -> Iterator[Candidate]
         if not self._installed:
-            candidates = self._get_others()
-        elif self._prefers_installed:
-            candidates = itertools.chain([self._installed], self._get_others())
-        else:
-            candidates = _insert_installed(self._installed, self._get_others())
-        return _deduplicated_by_version(candidates)
+            return self._get_others()
+        others = (
+            candidate
+            for candidate in self._get_others()
+            if candidate.version != self._installed.version
+        )
+        if self._prefers_installed:
+            return itertools.chain([self._installed], others)
+        return _insert_installed(self._installed, others)
 
     def __len__(self):
         # type: () -> int