Merge pull request #9771 from uranusjr/resolvelib-060

news/resolvelib.vendor.rst

@@ -1 +1 @@
-Upgrade vendored resolvelib to 0.5.5.
+Upgrade vendored resolvelib to 0.6.0.

src/pip/_internal/resolution/resolvelib/factory.py

@@ -7,6 +7,7 @@ from typing import (
     Iterable,
     Iterator,
     List,
+    Mapping,
     Optional,
     Sequence,
     Set,
@@ -104,6 +105,9 @@ class Factory:
         self._installed_candidate_cache = (
             {}
         )  # type: Dict[str, AlreadyInstalledCandidate]
+        self._extras_candidate_cache = (
+            {}
+        )  # type: Dict[Tuple[int, FrozenSet[str]], ExtrasCandidate]
 
         if not ignore_installed:
             self._installed_dists = {
@@ -118,6 +122,16 @@ class Factory:
         # type: () -> bool
         return self._force_reinstall
 
+    def _make_extras_candidate(self, base, extras):
+        # type: (BaseCandidate, FrozenSet[str]) -> ExtrasCandidate
+        cache_key = (id(base), extras)
+        try:
+            candidate = self._extras_candidate_cache[cache_key]
+        except KeyError:
+            candidate = ExtrasCandidate(base, extras)
+            self._extras_candidate_cache[cache_key] = candidate
+        return candidate
+
     def _make_candidate_from_dist(
         self,
         dist,  # type: Distribution
@@ -130,9 +144,9 @@ class Factory:
         except KeyError:
             base = AlreadyInstalledCandidate(dist, template, factory=self)
             self._installed_candidate_cache[dist.key] = base
-        if extras:
-            return ExtrasCandidate(base, extras)
-        return base
+        if not extras:
+            return base
+        return self._make_extras_candidate(base, extras)
 
     def _make_candidate_from_link(
         self,
@@ -182,18 +196,18 @@ class Factory:
                     return None
             base = self._link_candidate_cache[link]
 
-        if extras:
-            return ExtrasCandidate(base, extras)
-        return base
+        if not extras:
+            return base
+        return self._make_extras_candidate(base, extras)
 
     def _iter_found_candidates(
         self,
-        ireqs,  # type: Sequence[InstallRequirement]
-        specifier,  # type: SpecifierSet
-        hashes,  # type: Hashes
-        prefers_installed,  # type: bool
-    ):
-        # type: (...) -> Iterable[Candidate]
+        ireqs: Sequence[InstallRequirement],
+        specifier: SpecifierSet,
+        hashes: Hashes,
+        prefers_installed: bool,
+        incompatible_ids: Set[int],
+    ) -> Iterable[Candidate]:
         if not ireqs:
             return ()
 
@@ -257,20 +271,27 @@ class Factory:
             iter_index_candidate_infos,
             installed_candidate,
             prefers_installed,
+            incompatible_ids,
         )
 
     def find_candidates(
         self,
-        requirements,  # type: Sequence[Requirement]
-        constraint,  # type: Constraint
-        prefers_installed,  # type: bool
-    ):
-        # type: (...) -> Iterable[Candidate]
+        identifier: str,
+        requirements: Mapping[str, Iterator[Requirement]],
+        incompatibilities: Mapping[str, Iterator[Candidate]],
+        constraint: Constraint,
+        prefers_installed: bool,
+    ) -> Iterable[Candidate]:
+
+        # Since we cache all the candidates, incompatibility identification
+        # can be made quicker by comparing only the id() values.
+        incompat_ids = {id(c) for c in incompatibilities.get(identifier, ())}
+
         explicit_candidates = set()  # type: Set[Candidate]
         ireqs = []  # type: List[InstallRequirement]
-        for req in requirements:
+        for req in requirements[identifier]:
             cand, ireq = req.get_candidate_lookup()
-            if cand is not None:
+            if cand is not None and id(cand) not in incompat_ids:
                 explicit_candidates.add(cand)
             if ireq is not None:
                 ireqs.append(ireq)
@@ -283,13 +304,14 @@ class Factory:
                 constraint.specifier,
                 constraint.hashes,
                 prefers_installed,
+                incompat_ids,
             )
 
         return (
             c
             for c in explicit_candidates
             if constraint.is_satisfied_by(c)
-            and all(req.is_satisfied_by(c) for req in requirements)
+            and all(req.is_satisfied_by(c) for req in requirements[identifier])
         )
 
     def make_requirement_from_install_req(self, ireq, requested_extras):

src/pip/_internal/resolution/resolvelib/found_candidates.py

@@ -100,13 +100,15 @@ class FoundCandidates(collections_abc.Sequence):
 
     def __init__(
         self,
-        get_infos,  # type: Callable[[], Iterator[IndexCandidateInfo]]
-        installed,  # type: Optional[Candidate]
-        prefers_installed,  # type: bool
+        get_infos: Callable[[], Iterator[IndexCandidateInfo]],
+        installed: Optional[Candidate],
+        prefers_installed: bool,
+        incompatible_ids: Set[int],
     ):
         self._get_infos = get_infos
         self._installed = installed
         self._prefers_installed = prefers_installed
+        self._incompatible_ids = incompatible_ids
 
     def __getitem__(self, index):
         # type: (int) -> Candidate
@@ -119,10 +121,12 @@ class FoundCandidates(collections_abc.Sequence):
         # type: () -> Iterator[Candidate]
         infos = self._get_infos()
         if not self._installed:
-            return _iter_built(infos)
-        if self._prefers_installed:
-            return _iter_built_with_prepended(self._installed, infos)
-        return _iter_built_with_inserted(self._installed, infos)
+            iterator = _iter_built(infos)
+        elif self._prefers_installed:
+            iterator = _iter_built_with_prepended(self._installed, infos)
+        else:
+            iterator = _iter_built_with_inserted(self._installed, infos)
+        return (c for c in iterator if id(c) not in self._incompatible_ids)
 
     def __len__(self):
         # type: () -> int

src/pip/_internal/resolution/resolvelib/provider.py

@@ -1,4 +1,13 @@
-from typing import TYPE_CHECKING, Dict, Iterable, Optional, Sequence, Union
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    Iterator,
+    Mapping,
+    Optional,
+    Sequence,
+    Union,
+)
 
 from pip._vendor.resolvelib.providers import AbstractProvider
 
@@ -134,12 +143,12 @@ class PipProvider(_ProviderBase):
 
         return (delay_this, restrictive, order, key)
 
-    def find_matches(self, requirements):
-        # type: (Sequence[Requirement]) -> Iterable[Candidate]
-        if not requirements:
-            return []
-        name = requirements[0].project_name
-
+    def find_matches(
+        self,
+        identifier: str,
+        requirements: Mapping[str, Iterator[Requirement]],
+        incompatibilities: Mapping[str, Iterator[Candidate]],
+    ) -> Iterable[Candidate]:
         def _eligible_for_upgrade(name):
             # type: (str) -> bool
             """Are upgrades allowed for this project?
@@ -159,9 +168,11 @@ class PipProvider(_ProviderBase):
             return False
 
         return self._factory.find_candidates(
-            requirements,
-            constraint=self._constraints.get(name, Constraint.empty()),
-            prefers_installed=(not _eligible_for_upgrade(name)),
+            identifier=identifier,
+            requirements=requirements,
+            constraint=self._constraints.get(identifier, Constraint.empty()),
+            prefers_installed=(not _eligible_for_upgrade(identifier)),
+            incompatibilities=incompatibilities,
         )
 
     def is_satisfied_by(self, requirement, candidate):

src/pip/_vendor/resolvelib/__init__.py

@@ -11,7 +11,7 @@ __all__ = [
     "ResolutionTooDeep",
 ]
 
-__version__ = "0.5.5"
+__version__ = "0.6.0"
 
 
 from .providers import AbstractProvider, AbstractResolver

src/pip/_vendor/resolvelib/compat/collections_abc.py

@@ -1,6 +1,6 @@
-__all__ = ["Sequence"]
+__all__ = ["Mapping", "Sequence"]
 
 try:
-    from collections.abc import Sequence
+    from collections.abc import Mapping, Sequence
 except ImportError:
-    from collections import Sequence
+    from collections import Mapping, Sequence

src/pip/_vendor/resolvelib/providers.py

@@ -50,8 +50,18 @@ class AbstractProvider(object):
         """
         raise NotImplementedError
 
-    def find_matches(self, requirements):
-        """Find all possible candidates that satisfy the given requirements.
+    def find_matches(self, identifier, requirements, incompatibilities):
+        """Find all possible candidates that satisfy given constraints.
+
+        :param identifier: An identifier as returned by ``identify()``. This
+            identifies the dependency matches of which should be returned.
+        :param requirements: A mapping of requirements that all returned
+            candidates must satisfy. Each key is an identifier, and the value
+            an iterator of requirements for that dependency.
+        :param incompatibilities: A mapping of known incompatibilities of
+            each dependency. Each key is an identifier, and the value an
+            iterator of incompatibilities known to the resolver. All
+            incompatibilities *must* be excluded from the return value.
 
         This should try to get candidates based on the requirements' types.
         For VCS, local, and archive requirements, the one-and-only match is
@@ -66,10 +76,6 @@ class AbstractProvider(object):
         * An collection of candidates.
         * An iterable of candidates. This will be consumed immediately into a
           list of candidates.
-
-        :param requirements: A collection of requirements which all of the
-            returned candidates must match. All requirements are guaranteed to
-            have the same identifier. The collection is never empty.
         """
         raise NotImplementedError
 

src/pip/_vendor/resolvelib/providers.pyi

@@ -3,10 +3,10 @@ from typing import (
     Collection,
     Generic,
     Iterable,
+    Iterator,
     Mapping,
     Optional,
     Protocol,
-    Sequence,
     Union,
 )
 
@@ -31,7 +31,12 @@ class AbstractProvider(Generic[RT, CT, KT]):
         candidates: IterableView[CT],
         information: Collection[RequirementInformation[RT, CT]],
     ) -> Preference: ...
-    def find_matches(self, requirements: Sequence[RT]) -> Matches: ...
+    def find_matches(
+        self,
+        identifier: KT,
+        requirements: Mapping[KT, Iterator[RT]],
+        incompatibilities: Mapping[KT, Iterator[CT]],
+    ) -> Matches: ...
     def is_satisfied_by(self, requirement: RT, candidate: CT) -> bool: ...
     def get_dependencies(self, candidate: CT) -> Iterable[RT]: ...
 

src/pip/_vendor/resolvelib/resolvers.py

@@ -1,7 +1,8 @@
 import collections
+import operator
 
 from .providers import AbstractResolver
-from .structs import DirectedGraph, build_iter_view
+from .structs import DirectedGraph, IteratorMapping, build_iter_view
 
 
 RequirementInformation = collections.namedtuple(
@@ -73,45 +74,12 @@ class Criterion(object):
         )
         return "Criterion({})".format(requirements)
 
-    @classmethod
-    def from_requirement(cls, provider, requirement, parent):
-        """Build an instance from a requirement."""
-        matches = provider.find_matches(requirements=[requirement])
-        cands = build_iter_view(matches)
-        infos = [RequirementInformation(requirement, parent)]
-        criterion = cls(cands, infos, incompatibilities=[])
-        if not cands:
-            raise RequirementsConflicted(criterion)
-        return criterion
-
     def iter_requirement(self):
         return (i.requirement for i in self.information)
 
     def iter_parent(self):
         return (i.parent for i in self.information)
 
-    def merged_with(self, provider, requirement, parent):
-        """Build a new instance from this and a new requirement."""
-        infos = list(self.information)
-        infos.append(RequirementInformation(requirement, parent))
-        matches = provider.find_matches([r for r, _ in infos])
-        cands = build_iter_view(matches)
-        criterion = type(self)(cands, infos, list(self.incompatibilities))
-        if not cands:
-            raise RequirementsConflicted(criterion)
-        return criterion
-
-    def excluded_of(self, candidates):
-        """Build a new instance from this, but excluding specified candidates.
-
-        Returns the new instance, or None if we still have no valid candidates.
-        """
-        cands = self.candidates.excluding(candidates)
-        if not cands:
-            return None
-        incompats = self.incompatibilities + candidates
-        return type(self)(cands, list(self.information), incompats)
-
 
 class ResolutionError(ResolverException):
     pass
@@ -168,13 +136,42 @@ class Resolution(object):
 
     def _merge_into_criterion(self, requirement, parent):
         self._r.adding_requirement(requirement=requirement, parent=parent)
-        name = self._p.identify(requirement_or_candidate=requirement)
-        if name in self.state.criteria:
-            crit = self.state.criteria[name]
-            crit = crit.merged_with(self._p, requirement, parent)
+
+        identifier = self._p.identify(requirement_or_candidate=requirement)
+        criterion = self.state.criteria.get(identifier)
+        if criterion:
+            incompatibilities = list(criterion.incompatibilities)
         else:
-            crit = Criterion.from_requirement(self._p, requirement, parent)
-        return name, crit
+            incompatibilities = []
+
+        matches = self._p.find_matches(
+            identifier=identifier,
+            requirements=IteratorMapping(
+                self.state.criteria,
+                operator.methodcaller("iter_requirement"),
+                {identifier: [requirement]},
+            ),
+            incompatibilities=IteratorMapping(
+                self.state.criteria,
+                operator.attrgetter("incompatibilities"),
+                {identifier: incompatibilities},
+            ),
+        )
+
+        if criterion:
+            information = list(criterion.information)
+            information.append(RequirementInformation(requirement, parent))
+        else:
+            information = [RequirementInformation(requirement, parent)]
+
+        criterion = Criterion(
+            candidates=build_iter_view(matches),
+            information=information,
+            incompatibilities=incompatibilities,
+        )
+        if not criterion.candidates:
+            raise RequirementsConflicted(criterion)
+        return identifier, criterion
 
     def _get_criterion_item_preference(self, item):
         name, criterion = item
@@ -268,7 +265,7 @@ class Resolution(object):
             broken_state = self._states.pop()
             name, candidate = broken_state.mapping.popitem()
             incompatibilities_from_broken = [
-                (k, v.incompatibilities)
+                (k, list(v.incompatibilities))
                 for k, v in broken_state.criteria.items()
             ]
 
@@ -287,10 +284,27 @@ class Resolution(object):
                         criterion = self.state.criteria[k]
                     except KeyError:
                         continue
-                    criterion = criterion.excluded_of(incompatibilities)
-                    if criterion is None:
+                    matches = self._p.find_matches(
+                        identifier=k,
+                        requirements=IteratorMapping(
+                            self.state.criteria,
+                            operator.methodcaller("iter_requirement"),
+                        ),
+                        incompatibilities=IteratorMapping(
+                            self.state.criteria,
+                            operator.attrgetter("incompatibilities"),
+                            {k: incompatibilities},
+                        ),
+                    )
+                    candidates = build_iter_view(matches)
+                    if not candidates:
                         return False
-                    self.state.criteria[k] = criterion
+                    incompatibilities.extend(criterion.incompatibilities)
+                    self.state.criteria[k] = Criterion(
+                        candidates=candidates,
+                        information=list(criterion.information),
+                        incompatibilities=incompatibilities,
+                    )
                 return True
 
             self._push_new_state()

src/pip/_vendor/resolvelib/structs.py

@@ -1,3 +1,4 @@
+import itertools
 from .compat import collections_abc
 
 
@@ -67,6 +68,31 @@ class DirectedGraph(object):
         return iter(self._backwards[key])
 
 
+class IteratorMapping(collections_abc.Mapping):
+    def __init__(self, mapping, accessor, appends=None):
+        self._mapping = mapping
+        self._accessor = accessor
+        self._appends = appends or {}
+
+    def __contains__(self, key):
+        return key in self._mapping or key in self._appends
+
+    def __getitem__(self, k):
+        try:
+            v = self._mapping[k]
+        except KeyError:
+            return iter(self._appends[k])
+        return itertools.chain(self._accessor(v), self._appends.get(k, ()))
+
+    def __iter__(self):
+        more = (k for k in self._appends if k not in self._mapping)
+        return itertools.chain(self._mapping, more)
+
+    def __len__(self):
+        more = len(k for k in self._appends if k not in self._mapping)
+        return len(self._mapping) + more
+
+
 class _FactoryIterableView(object):
     """Wrap an iterator factory returned by `find_matches()`.
 

src/pip/_vendor/vendor.txt

@@ -14,7 +14,7 @@ requests==2.25.1
     chardet==4.0.0
     idna==2.10
     urllib3==1.26.4
-resolvelib==0.5.5
+resolvelib==0.6.0
 setuptools==44.0.0
 six==1.15.0
 tenacity==6.3.1

tests/unit/resolution_resolvelib/test_requirement.py

@@ -59,7 +59,11 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory):
     for spec, _, match_count in test_cases:
         req = factory.make_requirement_from_spec(spec, comes_from=None)
         matches = factory.find_candidates(
-            [req], Constraint.empty(), prefers_installed=False,
+            req.name,
+            {req.name: [req]},
+            {},
+            Constraint.empty(),
+            prefers_installed=False,
         )
         assert sum(1 for _ in matches) == match_count
 
@@ -70,7 +74,11 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory):
     for spec, _, _ in test_cases:
         req = factory.make_requirement_from_spec(spec, comes_from=None)
         candidates = factory.find_candidates(
-            [req], Constraint.empty(), prefers_installed=False,
+            req.name,
+            {req.name: [req]},
+            {},
+            Constraint.empty(),
+            prefers_installed=False,
         )
         for c in candidates:
             assert isinstance(c, Candidate)