McSinyx
/
pip
mirror of https://github.com/pypa/pip
1
1
Fork 0
Code Issues Releases Activity

Merge pull request #8267 from uranusjr/new-resolver-candidate-order 

Upgrade ResolveLib to 0.4.0 and implement the new Provider.find_matches() interface
This commit is contained in:
Paul Moore 2020-05-28 10:46:02 +01:00 committed by GitHub
parent 7c10d3ed3b 9ee19a1190
commit 03b11eed8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 229 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/pip/_internal/resolution/resolvelib/base.py
View File

@ -3,15 +3,20 @@ from pip._vendor.packaging.utils import canonicalize_name
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import Iterable, Optional, Sequence, Set
from typing import FrozenSet, Iterable, Optional, Tuple
from pip._vendor.packaging.version import _BaseVersion
from pip._internal.req.req_install import InstallRequirement
from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.packaging.version import _BaseVersion
CandidateLookup = Tuple[
Optional["Candidate"],
Optional[InstallRequirement],
]
def format_name(project, extras):
# type: (str, Set[str]) -> str
# type: (str, FrozenSet[str]) -> str
if not extras:
return project
canonical_extras = sorted(canonicalize_name(e) for e in extras)
@ -24,14 +29,14 @@ class Requirement(object):
# type: () -> str
raise NotImplementedError("Subclass should override")
def find_matches(self, constraint):
# type: (SpecifierSet) -> Sequence[Candidate]
raise NotImplementedError("Subclass should override")
def is_satisfied_by(self, candidate):
# type: (Candidate) -> bool
return False
def get_candidate_lookup(self):
# type: () -> CandidateLookup
raise NotImplementedError("Subclass should override")
class Candidate(object):
@property

16
src/pip/_internal/resolution/resolvelib/candidates.py
View File

@ -17,7 +17,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from .base import Candidate, format_name
if MYPY_CHECK_RUNNING:
from typing import Any, Iterable, Optional, Set, Tuple, Union
from typing import Any, FrozenSet, Iterable, Optional, Tuple, Union
from pip._vendor.packaging.version import _BaseVersion
from pip._vendor.pkg_resources import Distribution
@ -132,6 +132,10 @@ class _InstallRequirementBackedCandidate(Candidate):
link=str(self.link),
)
def __hash__(self):
# type: () -> int
return hash((self.__class__, self.link))
def __eq__(self, other):
# type: (Any) -> bool
if isinstance(other, self.__class__):
@ -313,6 +317,10 @@ class AlreadyInstalledCandidate(Candidate):
distribution=self.dist,
)
def __hash__(self):
# type: () -> int
return hash((self.__class__, self.name, self.version))
def __eq__(self, other):
# type: (Any) -> bool
if isinstance(other, self.__class__):
@ -371,7 +379,7 @@ class ExtrasCandidate(Candidate):
def __init__(
self,
base, # type: BaseCandidate
extras, # type: Set[str]
extras, # type: FrozenSet[str]
):
# type: (...) -> None
self.base = base
@ -385,6 +393,10 @@ class ExtrasCandidate(Candidate):
extras=self.extras,
)
def __hash__(self):
# type: () -> int
return hash((self.base, self.extras))
def __eq__(self, other):
# type: (Any) -> bool
if isinstance(other, self.__class__):

93
src/pip/_internal/resolution/resolvelib/factory.py
View File

@ -9,6 +9,7 @@ from pip._internal.exceptions import (
UnsupportedPythonVersion,
)
from pip._internal.utils.compatibility_tags import get_supported
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.misc import (
dist_in_site_packages,
dist_in_usersite,
@ -31,7 +32,17 @@ from .requirements import (
)
if MYPY_CHECK_RUNNING:
from typing import Dict, Iterable, Iterator, Optional, Set, Tuple, TypeVar
from typing import (
FrozenSet,
Dict,
Iterable,
List,
Optional,
Sequence,
Set,
Tuple,
TypeVar,
)
from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.packaging.version import _BaseVersion
@ -71,7 +82,7 @@ class Factory(object):
):
# type: (...) -> None
self.finder = finder
self._finder = finder
self.preparer = preparer
self._wheel_cache = wheel_cache
self._python_candidate = RequiresPythonCandidate(py_version_info)
@ -94,7 +105,7 @@ class Factory(object):
def _make_candidate_from_dist(
self,
dist, # type: Distribution
extras, # type: Set[str]
extras, # type: FrozenSet[str]
parent, # type: InstallRequirement
):
# type: (...) -> Candidate
@ -106,7 +117,7 @@ class Factory(object):
def _make_candidate_from_link(
self,
link, # type: Link
extras, # type: Set[str]
extras, # type: FrozenSet[str]
parent, # type: InstallRequirement
name, # type: Optional[str]
version, # type: Optional[_BaseVersion]
@ -130,9 +141,28 @@ class Factory(object):
return ExtrasCandidate(base, extras)
return base
def iter_found_candidates(self, ireq, extras):
# type: (InstallRequirement, Set[str]) -> Iterator[Candidate]
name = canonicalize_name(ireq.req.name)
def _iter_found_candidates(
self,
ireqs, # type: Sequence[InstallRequirement]
specifier, # type: SpecifierSet
):
# type: (...) -> Iterable[Candidate]
if not ireqs:
return ()
# The InstallRequirement implementation requires us to give it a
# "parent", which doesn't really fit with graph-based resolution.
# Here we just choose the first requirement to represent all of them.
# Hopefully the Project model can correct this mismatch in the future.
parent = ireqs[0]
name = canonicalize_name(parent.req.name)
hashes = Hashes()
extras = frozenset() # type: FrozenSet[str]
for ireq in ireqs:
specifier &= ireq.req.specifier
hashes |= ireq.hashes(trust_internet=False)
extras |= frozenset(ireq.extras)
# We use this to ensure that we only yield a single candidate for
# each version (the finder's preferred one for that version). The
@ -148,21 +178,18 @@ class Factory(object):
if not self._force_reinstall and name in self._installed_dists:
installed_dist = self._installed_dists[name]
installed_version = installed_dist.parsed_version
if ireq.req.specifier.contains(
installed_version,
prereleases=True
):
if specifier.contains(installed_version, prereleases=True):
candidate = self._make_candidate_from_dist(
dist=installed_dist,
extras=extras,
parent=ireq,
parent=parent,
)
candidates[installed_version] = candidate
found = self.finder.find_best_candidate(
project_name=ireq.req.name,
specifier=ireq.req.specifier,
hashes=ireq.hashes(trust_internet=False),
found = self._finder.find_best_candidate(
project_name=name,
specifier=specifier,
hashes=hashes,
)
for ican in found.iter_applicable():
if ican.version == installed_version:
@ -170,7 +197,7 @@ class Factory(object):
candidate = self._make_candidate_from_link(
link=ican.link,
extras=extras,
parent=ireq,
parent=parent,
name=name,
version=ican.version,
)
@ -178,13 +205,41 @@ class Factory(object):
return six.itervalues(candidates)
def find_candidates(self, requirements, constraint):
# type: (Sequence[Requirement], SpecifierSet) -> Iterable[Candidate]
explicit_candidates = set() # type: Set[Candidate]
ireqs = [] # type: List[InstallRequirement]
for req in requirements:
cand, ireq = req.get_candidate_lookup()
if cand is not None:
explicit_candidates.add(cand)
if ireq is not None:
ireqs.append(ireq)
# If none of the requirements want an explicit candidate, we can ask
# the finder for candidates.
if not explicit_candidates:
return self._iter_found_candidates(ireqs, constraint)
if constraint:
name = explicit_candidates.pop().name
raise InstallationError(
"Could not satisfy constraints for {!r}: installation from "
"path or url cannot be constrained to a version".format(name)
)
return (
c for c in explicit_candidates
if all(req.is_satisfied_by(c) for req in requirements)
)
def make_requirement_from_install_req(self, ireq):
# type: (InstallRequirement) -> Requirement
if not ireq.link:
return SpecifierRequirement(ireq, factory=self)
return SpecifierRequirement(ireq)
cand = self._make_candidate_from_link(
ireq.link,
extras=set(ireq.extras),
extras=frozenset(ireq.extras),
parent=ireq,
name=canonicalize_name(ireq.name) if ireq.name else None,
version=None,

28
src/pip/_internal/resolution/resolvelib/provider.py
View File

@ -4,7 +4,16 @@ from pip._vendor.resolvelib.providers import AbstractProvider
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import Any, Dict, Optional, Sequence, Set, Tuple, Union
from typing import (
Any,
Dict,
Iterable,
Optional,
Sequence,
Set,
Tuple,
Union,
)
from .base import Requirement, Candidate
from .factory import Factory
@ -45,7 +54,7 @@ class PipProvider(AbstractProvider):
self.user_requested = user_requested
def _sort_matches(self, matches):
# type: (Sequence[Candidate]) -> Sequence[Candidate]
# type: (Iterable[Candidate]) -> Sequence[Candidate]
# The requirement is responsible for returning a sequence of potential
# candidates, one per version. The provider handles the logic of
@ -68,7 +77,6 @@ class PipProvider(AbstractProvider):
# - The project was specified on the command line, or
# - The project is a dependency and the "eager" upgrade strategy
# was requested.
def _eligible_for_upgrade(name):
# type: (str) -> bool
"""Are upgrades allowed for this project?
@ -121,11 +129,15 @@ class PipProvider(AbstractProvider):
# Use the "usual" value for now
return len(candidates)
def find_matches(self, requirement):
# type: (Requirement) -> Sequence[Candidate]
constraint = self._constraints.get(requirement.name, SpecifierSet())
matches = requirement.find_matches(constraint)
return self._sort_matches(matches)
def find_matches(self, requirements):
# type: (Sequence[Requirement]) -> Iterable[Candidate]
if not requirements:
return []
constraint = self._constraints.get(
requirements[0].name, SpecifierSet(),
)
candidates = self._factory.find_candidates(requirements, constraint)
return reversed(self._sort_matches(candidates))
def is_satisfied_by(self, requirement, candidate):
# type: (Requirement, Candidate) -> bool

53
src/pip/_internal/resolution/resolvelib/requirements.py
View File

@ -1,19 +1,15 @@
from pip._vendor.packaging.utils import canonicalize_name
from pip._internal.exceptions import InstallationError
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from .base import Requirement, format_name
if MYPY_CHECK_RUNNING:
from typing import Sequence
from pip._vendor.packaging.specifiers import SpecifierSet
from pip._internal.req.req_install import InstallRequirement
from .base import Candidate
from .factory import Factory
from .base import Candidate, CandidateLookup
class ExplicitRequirement(Requirement):
@ -34,15 +30,9 @@ class ExplicitRequirement(Requirement):
# No need to canonicalise - the candidate did this
return self.candidate.name
def find_matches(self, constraint):
# type: (SpecifierSet) -> Sequence[Candidate]
if len(constraint) > 0:
raise InstallationError(
"Could not satisfy constraints for '{}': "
"installation from path or url cannot be "
"constrained to a version".format(self.name)
)
return [self.candidate]
def get_candidate_lookup(self):
# type: () -> CandidateLookup
return self.candidate, None
def is_satisfied_by(self, candidate):
# type: (Candidate) -> bool
@ -50,12 +40,11 @@ class ExplicitRequirement(Requirement):
class SpecifierRequirement(Requirement):
def __init__(self, ireq, factory):
# type: (InstallRequirement, Factory) -> None
def __init__(self, ireq):
# type: (InstallRequirement) -> None
assert ireq.link is None, "This is a link, not a specifier"
self._ireq = ireq
self._factory = factory
self.extras = set(ireq.extras)
self._extras = frozenset(ireq.extras)
def __str__(self):
# type: () -> str
@ -72,21 +61,11 @@ class SpecifierRequirement(Requirement):
def name(self):
# type: () -> str
canonical_name = canonicalize_name(self._ireq.req.name)
return format_name(canonical_name, self.extras)
return format_name(canonical_name, self._extras)
def find_matches(self, constraint):
# type: (SpecifierSet) -> Sequence[Candidate]
# We should only return one candidate per version, but
# iter_found_candidates does that for us, so we don't need
# to do anything special here.
return [
c
for c in self._factory.iter_found_candidates(
self._ireq, self.extras
)
if constraint.contains(c.version, prereleases=True)
]
def get_candidate_lookup(self):
# type: () -> CandidateLookup
return None, self._ireq
def is_satisfied_by(self, candidate):
# type: (Candidate) -> bool
@ -120,13 +99,11 @@ class RequiresPythonRequirement(Requirement):
# type: () -> str
return self._candidate.name
def find_matches(self, constraint):
# type: (SpecifierSet) -> Sequence[Candidate]
assert len(constraint) == 0, \
"RequiresPythonRequirement cannot have constraints"
def get_candidate_lookup(self):
# type: () -> CandidateLookup
if self.specifier.contains(self._candidate.version, prereleases=True):
return [self._candidate]
return []
return self._candidate, None
return None, None
def is_satisfied_by(self, candidate):
# type: (Candidate) -> bool

12
src/pip/_internal/utils/hashes.py
View File

@ -46,6 +46,18 @@ class Hashes(object):
"""
self._allowed = {} if hashes is None else hashes
def __or__(self, other):
# type: (Hashes) -> Hashes
if not isinstance(other, Hashes):
return NotImplemented
new = self._allowed.copy()
for alg, values in iteritems(other._allowed):
try:
new[alg] += values
except KeyError:
new[alg] = values
return Hashes(new)
@property
def digest_count(self):
# type: () -> int

2
src/pip/_vendor/resolvelib/__init__.py
View File

@ -11,7 +11,7 @@ __all__ = [
"ResolutionTooDeep",
]
__version__ = "0.3.0"
__version__ = "0.4.0"
from .providers import AbstractProvider, AbstractResolver

0
src/pip/_vendor/resolvelib/compat/__init__.py Normal file
View File

6
src/pip/_vendor/resolvelib/compat/collections_abc.py Normal file
View File

@ -0,0 +1,6 @@
__all__ = ["Sequence"]
try:
from collections.abc import Sequence
except ImportError:
from collections import Sequence

48
src/pip/_vendor/resolvelib/providers.py
View File

@ -27,7 +27,7 @@ class AbstractProvider(object):
* `requirement` specifies a requirement contributing to the current
candidate list
* `parent` specifies the candidate that provids (dependend on) the
* `parent` specifies the candidate that provides (dependend on) the
requirement, or `None` to indicate a root requirement.
The preference could depend on a various of issues, including (not
@ -48,23 +48,28 @@ class AbstractProvider(object):
"""
raise NotImplementedError
def find_matches(self, requirement):
"""Find all possible candidates that satisfy a requirement.
def find_matches(self, requirements):
"""Find all possible candidates that satisfy the given requirements.
This should try to get candidates based on the requirement's type.
This should try to get candidates based on the requirements' types.
For VCS, local, and archive requirements, the one-and-only match is
returned, and for a "named" requirement, the index(es) should be
consulted to find concrete candidates for this requirement.
The returned candidates should be sorted by reversed preference, e.g.
the most preferred should be LAST. This is done so list-popping can be
as efficient as possible.
:param requirements: A collection of requirements which all of the the
returned candidates must match. All requirements are guaranteed to
have the same identifier. The collection is never empty.
:returns: An iterable that orders candidates by preference, e.g. the
most preferred candidate should come first.
"""
raise NotImplementedError
def is_satisfied_by(self, requirement, candidate):
"""Whether the given requirement can be satisfied by a candidate.
The candidate is guarenteed to have been generated from the
requirement.
A boolean should be returned to indicate whether `candidate` is a
viable solution to the requirement.
"""
@ -92,30 +97,13 @@ class AbstractResolver(object):
def resolve(self, requirements, **kwargs):
"""Take a collection of constraints, spit out the resolution result.
Parameters
----------
requirements : Collection
A collection of constraints
kwargs : optional
Additional keyword arguments that subclasses may accept.
This returns a representation of the final resolution state, with one
guarenteed attribute ``mapping`` that contains resolved candidates as
values. The keys are their respective identifiers.
Raises
------
self.base_exception
Any raised exception is guaranteed to be a subclass of
self.base_exception. The string representation of an exception
should be human readable and provide context for why it occurred.
:param requirements: A collection of constraints.
:param kwargs: Additional keyword arguments that subclasses may accept.
Returns
-------
retval : object
A representation of the final resolution state. It can be any object
with a `mapping` attribute that is a Mapping. Other attributes can
be used to provide resolver-specific information.
The `mapping` attribute MUST be key-value pair is an identifier of a
requirement (as returned by the provider's `identify` method) mapped
to the resolved candidate (chosen from the return value of the
provider's `find_matches` method).
:raises: ``self.base_exception`` or its subclass.
"""
raise NotImplementedError

12
src/pip/_vendor/resolvelib/reporters.py
View File

@ -23,12 +23,18 @@ class BaseReporter(object):
"""Called before the resolution ends successfully.
"""
def adding_requirement(self, requirement):
"""Called when the resolver adds a new requirement into the resolve criteria.
def adding_requirement(self, requirement, parent):
"""Called when adding a new requirement into the resolve criteria.
:param requirement: The additional requirement to be applied to filter
the available candidaites.
:param parent: The candidate that requires ``requirement`` as a
dependency, or None if ``requirement`` is one of the root
requirements passed in from ``Resolver.resolve()``.
"""
def backtracking(self, candidate):
"""Called when the resolver rejects a candidate during backtracking.
"""Called when rejecting a candidate during backtracking.
"""
def pinning(self, candidate):

66
src/pip/_vendor/resolvelib/resolvers.py
View File

@ -1,5 +1,6 @@
import collections
from .compat import collections_abc
from .providers import AbstractResolver
from .structs import DirectedGraph
@ -68,16 +69,18 @@ class Criterion(object):
def __repr__(self):
requirements = ", ".join(
"{!r} from {!r}".format(req, parent)
"({!r}, via={!r})".format(req, parent)
for req, parent in self.information
)
return "<Criterion {}>".format(requirements)
return "Criterion({})".format(requirements)
@classmethod
def from_requirement(cls, provider, requirement, parent):
"""Build an instance from a requirement.
"""
candidates = provider.find_matches(requirement)
candidates = provider.find_matches([requirement])
if not isinstance(candidates, collections_abc.Sequence):
candidates = list(candidates)
criterion = cls(
candidates=candidates,
information=[RequirementInformation(requirement, parent)],
@ -98,11 +101,9 @@ class Criterion(object):
"""
infos = list(self.information)
infos.append(RequirementInformation(requirement, parent))
candidates = [
c
for c in self.candidates
if provider.is_satisfied_by(requirement, c)
]
candidates = provider.find_matches([r for r, _ in infos])
if not isinstance(candidates, collections_abc.Sequence):
candidates = list(candidates)
criterion = type(self)(candidates, infos, list(self.incompatibilities))
if not candidates:
raise RequirementsConflicted(criterion)
@ -179,7 +180,7 @@ class Resolution(object):
self._states.append(state)
def _merge_into_criterion(self, requirement, parent):
self._r.adding_requirement(requirement)
self._r.adding_requirement(requirement, parent)
name = self._p.identify(requirement)
try:
crit = self.state.criteria[name]
@ -218,13 +219,24 @@ class Resolution(object):
def _attempt_to_pin_criterion(self, name, criterion):
causes = []
for candidate in reversed(criterion.candidates):
for candidate in criterion.candidates:
try:
criteria = self._get_criteria_to_update(candidate)
except RequirementsConflicted as e:
causes.append(e.criterion)
continue
# Check the newly-pinned candidate actually works. This should
# always pass under normal circumstances, but in the case of a
# faulty provider, we will raise an error to notify the implementer
# to fix find_matches() and/or is_satisfied_by().
satisfied = all(
self._p.is_satisfied_by(r, candidate)
for r in criterion.iter_requirement()
)
if not satisfied:
raise InconsistentCandidate(candidate, criterion)
# Put newly-pinned candidate at the end. This is essential because
# backtracking looks at this mapping to get the last pin.
self._r.pinning(candidate)
@ -232,13 +244,6 @@ class Resolution(object):
self.state.mapping[name] = candidate
self.state.criteria.update(criteria)
# Check the newly-pinned candidate actually works. This should
# always pass under normal circumstances, but in the case of a
# faulty provider, we will raise an error to notify the implementer
# to fix find_matches() and/or is_satisfied_by().
if not self._is_current_pin_satisfying(name, criterion):
raise InconsistentCandidate(candidate, criterion)
return []
# All candidates tried, nothing works. This criterion is a dead
@ -246,23 +251,32 @@ class Resolution(object):
return causes
def _backtrack(self):
# We need at least 3 states here:
# (a) One known not working, to drop.
# (b) One to backtrack to.
# (c) One to restore state (b) to its state prior to candidate-pinning,
# so we can pin another one instead.
while len(self._states) >= 3:
del self._states[-1]
# Drop the current state, it's known not to work.
del self._states[-1]
# Retract the last candidate pin, and create a new (b).
name, candidate = self._states.pop().mapping.popitem()
# We need at least 2 states here:
# (a) One to backtrack to.
# (b) One to restore state (a) to its state prior to candidate-pinning,
# so we can pin another one instead.
while len(self._states) >= 2:
# Retract the last candidate pin.
prev_state = self._states.pop()
try:
name, candidate = prev_state.mapping.popitem()
except KeyError:
continue
self._r.backtracking(candidate)
# Create a new state to work on, with the newly known not-working
# candidate excluded.
self._push_new_state()
# Mark the retracted candidate as incompatible.
criterion = self.state.criteria[name].excluded_of(candidate)
if criterion is None:
# This state still does not work. Try the still previous state.
del self._states[-1]
continue
self.state.criteria[name] = criterion

2
src/pip/_vendor/vendor.txt
View File

@ -16,7 +16,7 @@ requests==2.23.0
chardet==3.0.4
idna==2.9
urllib3==1.25.8
resolvelib==0.3.0
resolvelib==0.4.0
retrying==1.3.3
setuptools==44.0.0
six==1.14.0

10
tests/unit/resolution_resolvelib/test_requirement.py
View File

@ -57,16 +57,18 @@ def test_new_resolver_requirement_has_name(test_cases, factory):
def test_new_resolver_correct_number_of_matches(test_cases, factory):
"""Requirements should return the correct number of candidates"""
for spec, name, matches in test_cases:
for spec, name, match_count in test_cases:
req = factory.make_requirement_from_spec(spec, comes_from=None)
assert len(req.find_matches(SpecifierSet())) == matches
matches = factory.find_candidates([req], SpecifierSet())
assert len(list(matches)) == match_count
def test_new_resolver_candidates_match_requirement(test_cases, factory):
"""Candidates returned from find_matches should satisfy the requirement"""
"""Candidates returned from find_candidates should satisfy the requirement
"""
for spec, name, matches in test_cases:
req = factory.make_requirement_from_spec(spec, comes_from=None)
for c in req.find_matches(SpecifierSet()):
for c in factory.find_candidates([req], SpecifierSet()):
assert isinstance(c, Candidate)
assert req.is_satisfied_by(c)