mirror of
https://github.com/pypa/pip
synced 2023-12-13 21:30:23 +01:00
Merge pull request #8932 from uranusjr/new-resolver-lazy-sequence
This commit is contained in:
parent
4aec7e81c9
commit
6a8956d7a8
6 changed files with 183 additions and 104 deletions
2
news/8023.feature.rst
Normal file
2
news/8023.feature.rst
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
New resolver: Avoid accessing indexes when the installed candidate is preferred
|
||||||
|
and considered good enough.
|
|
@ -1,7 +1,5 @@
|
||||||
import collections
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from pip._vendor import six
|
|
||||||
from pip._vendor.packaging.utils import canonicalize_name
|
from pip._vendor.packaging.utils import canonicalize_name
|
||||||
|
|
||||||
from pip._internal.exceptions import (
|
from pip._internal.exceptions import (
|
||||||
|
@ -30,6 +28,7 @@ from .candidates import (
|
||||||
LinkCandidate,
|
LinkCandidate,
|
||||||
RequiresPythonCandidate,
|
RequiresPythonCandidate,
|
||||||
)
|
)
|
||||||
|
from .found_candidates import FoundCandidates
|
||||||
from .requirements import (
|
from .requirements import (
|
||||||
ExplicitRequirement,
|
ExplicitRequirement,
|
||||||
RequiresPythonRequirement,
|
RequiresPythonRequirement,
|
||||||
|
@ -41,6 +40,7 @@ if MYPY_CHECK_RUNNING:
|
||||||
FrozenSet,
|
FrozenSet,
|
||||||
Dict,
|
Dict,
|
||||||
Iterable,
|
Iterable,
|
||||||
|
Iterator,
|
||||||
List,
|
List,
|
||||||
Optional,
|
Optional,
|
||||||
Sequence,
|
Sequence,
|
||||||
|
@ -102,7 +102,7 @@ class Factory(object):
|
||||||
if not ignore_installed:
|
if not ignore_installed:
|
||||||
self._installed_dists = {
|
self._installed_dists = {
|
||||||
canonicalize_name(dist.project_name): dist
|
canonicalize_name(dist.project_name): dist
|
||||||
for dist in get_installed_distributions()
|
for dist in get_installed_distributions(local_only=False)
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
self._installed_dists = {}
|
self._installed_dists = {}
|
||||||
|
@ -156,6 +156,7 @@ class Factory(object):
|
||||||
ireqs, # type: Sequence[InstallRequirement]
|
ireqs, # type: Sequence[InstallRequirement]
|
||||||
specifier, # type: SpecifierSet
|
specifier, # type: SpecifierSet
|
||||||
hashes, # type: Hashes
|
hashes, # type: Hashes
|
||||||
|
prefers_installed, # type: bool
|
||||||
):
|
):
|
||||||
# type: (...) -> Iterable[Candidate]
|
# type: (...) -> Iterable[Candidate]
|
||||||
if not ireqs:
|
if not ireqs:
|
||||||
|
@ -174,54 +175,49 @@ class Factory(object):
|
||||||
hashes &= ireq.hashes(trust_internet=False)
|
hashes &= ireq.hashes(trust_internet=False)
|
||||||
extras |= frozenset(ireq.extras)
|
extras |= frozenset(ireq.extras)
|
||||||
|
|
||||||
# We use this to ensure that we only yield a single candidate for
|
|
||||||
# each version (the finder's preferred one for that version). The
|
|
||||||
# requirement needs to return only one candidate per version, so we
|
|
||||||
# implement that logic here so that requirements using this helper
|
|
||||||
# don't all have to do the same thing later.
|
|
||||||
candidates = collections.OrderedDict() # type: VersionCandidates
|
|
||||||
|
|
||||||
# Get the installed version, if it matches, unless the user
|
# Get the installed version, if it matches, unless the user
|
||||||
# specified `--force-reinstall`, when we want the version from
|
# specified `--force-reinstall`, when we want the version from
|
||||||
# the index instead.
|
# the index instead.
|
||||||
installed_version = None
|
|
||||||
installed_candidate = None
|
installed_candidate = None
|
||||||
if not self._force_reinstall and name in self._installed_dists:
|
if not self._force_reinstall and name in self._installed_dists:
|
||||||
installed_dist = self._installed_dists[name]
|
installed_dist = self._installed_dists[name]
|
||||||
installed_version = installed_dist.parsed_version
|
if specifier.contains(installed_dist.version, prereleases=True):
|
||||||
if specifier.contains(installed_version, prereleases=True):
|
|
||||||
installed_candidate = self._make_candidate_from_dist(
|
installed_candidate = self._make_candidate_from_dist(
|
||||||
dist=installed_dist,
|
dist=installed_dist,
|
||||||
extras=extras,
|
extras=extras,
|
||||||
template=template,
|
template=template,
|
||||||
)
|
)
|
||||||
|
|
||||||
found = self._finder.find_best_candidate(
|
def iter_index_candidates():
|
||||||
project_name=name,
|
# type: () -> Iterator[Candidate]
|
||||||
specifier=specifier,
|
result = self._finder.find_best_candidate(
|
||||||
hashes=hashes,
|
project_name=name,
|
||||||
)
|
specifier=specifier,
|
||||||
for ican in found.iter_applicable():
|
hashes=hashes,
|
||||||
if ican.version == installed_version and installed_candidate:
|
)
|
||||||
candidate = installed_candidate
|
# PackageFinder returns earlier versions first, so we reverse.
|
||||||
else:
|
for ican in reversed(list(result.iter_applicable())):
|
||||||
candidate = self._make_candidate_from_link(
|
yield self._make_candidate_from_link(
|
||||||
link=ican.link,
|
link=ican.link,
|
||||||
extras=extras,
|
extras=extras,
|
||||||
template=template,
|
template=template,
|
||||||
name=name,
|
name=name,
|
||||||
version=ican.version,
|
version=ican.version,
|
||||||
)
|
)
|
||||||
candidates[ican.version] = candidate
|
|
||||||
|
|
||||||
# Yield the installed version even if it is not found on the index.
|
return FoundCandidates(
|
||||||
if installed_version and installed_candidate:
|
iter_index_candidates,
|
||||||
candidates[installed_version] = installed_candidate
|
installed_candidate,
|
||||||
|
prefers_installed,
|
||||||
|
)
|
||||||
|
|
||||||
return six.itervalues(candidates)
|
def find_candidates(
|
||||||
|
self,
|
||||||
def find_candidates(self, requirements, constraint):
|
requirements, # type: Sequence[Requirement]
|
||||||
# type: (Sequence[Requirement], Constraint) -> Iterable[Candidate]
|
constraint, # type: Constraint
|
||||||
|
prefers_installed, # type: bool
|
||||||
|
):
|
||||||
|
# type: (...) -> Iterable[Candidate]
|
||||||
explicit_candidates = set() # type: Set[Candidate]
|
explicit_candidates = set() # type: Set[Candidate]
|
||||||
ireqs = [] # type: List[InstallRequirement]
|
ireqs = [] # type: List[InstallRequirement]
|
||||||
for req in requirements:
|
for req in requirements:
|
||||||
|
@ -238,6 +234,7 @@ class Factory(object):
|
||||||
ireqs,
|
ireqs,
|
||||||
constraint.specifier,
|
constraint.specifier,
|
||||||
constraint.hashes,
|
constraint.hashes,
|
||||||
|
prefers_installed,
|
||||||
)
|
)
|
||||||
|
|
||||||
if constraint:
|
if constraint:
|
||||||
|
|
98
src/pip/_internal/resolution/resolvelib/found_candidates.py
Normal file
98
src/pip/_internal/resolution/resolvelib/found_candidates.py
Normal file
|
@ -0,0 +1,98 @@
|
||||||
|
import itertools
|
||||||
|
import operator
|
||||||
|
|
||||||
|
from pip._vendor.six.moves import collections_abc # type: ignore
|
||||||
|
|
||||||
|
from pip._internal.utils.compat import lru_cache
|
||||||
|
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||||
|
|
||||||
|
if MYPY_CHECK_RUNNING:
|
||||||
|
from typing import Callable, Iterator, Optional, Set
|
||||||
|
|
||||||
|
from pip._vendor.packaging.version import _BaseVersion
|
||||||
|
|
||||||
|
from .base import Candidate
|
||||||
|
|
||||||
|
|
||||||
|
def _deduplicated_by_version(candidates):
|
||||||
|
# type: (Iterator[Candidate]) -> Iterator[Candidate]
|
||||||
|
returned = set() # type: Set[_BaseVersion]
|
||||||
|
for candidate in candidates:
|
||||||
|
if candidate.version in returned:
|
||||||
|
continue
|
||||||
|
returned.add(candidate.version)
|
||||||
|
yield candidate
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_installed(installed, others):
|
||||||
|
# type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
|
||||||
|
"""Iterator for ``FoundCandidates``.
|
||||||
|
|
||||||
|
This iterator is used when the resolver prefers to upgrade an
|
||||||
|
already-installed package. Candidates from index are returned in their
|
||||||
|
normal ordering, except replaced when the version is already installed.
|
||||||
|
|
||||||
|
Since candidates from index are already sorted by reverse version order,
|
||||||
|
`sorted()` here would keep the ordering mostly intact, only shuffling the
|
||||||
|
already-installed candidate into the correct position. We put the already-
|
||||||
|
installed candidate in front of those from the index, so it's put in front
|
||||||
|
after sorting due to Python sorting's stableness guarentee.
|
||||||
|
"""
|
||||||
|
candidates = sorted(
|
||||||
|
itertools.chain([installed], others),
|
||||||
|
key=operator.attrgetter("version"),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
return iter(candidates)
|
||||||
|
|
||||||
|
|
||||||
|
class FoundCandidates(collections_abc.Sequence):
|
||||||
|
"""A lazy sequence to provide candidates to the resolver.
|
||||||
|
|
||||||
|
The intended usage is to return this from `find_matches()` so the resolver
|
||||||
|
can iterate through the sequence multiple times, but only access the index
|
||||||
|
page when remote packages are actually needed. This improve performances
|
||||||
|
when suitable candidates are already installed on disk.
|
||||||
|
"""
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
get_others, # type: Callable[[], Iterator[Candidate]]
|
||||||
|
installed, # type: Optional[Candidate]
|
||||||
|
prefers_installed, # type: bool
|
||||||
|
):
|
||||||
|
self._get_others = get_others
|
||||||
|
self._installed = installed
|
||||||
|
self._prefers_installed = prefers_installed
|
||||||
|
|
||||||
|
def __getitem__(self, index):
|
||||||
|
# type: (int) -> Candidate
|
||||||
|
# Implemented to satisfy the ABC check. This is not needed by the
|
||||||
|
# resolver, and should not be used by the provider either (for
|
||||||
|
# performance reasons).
|
||||||
|
raise NotImplementedError("don't do this")
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
# type: () -> Iterator[Candidate]
|
||||||
|
if not self._installed:
|
||||||
|
candidates = self._get_others()
|
||||||
|
elif self._prefers_installed:
|
||||||
|
candidates = itertools.chain([self._installed], self._get_others())
|
||||||
|
else:
|
||||||
|
candidates = _insert_installed(self._installed, self._get_others())
|
||||||
|
return _deduplicated_by_version(candidates)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
# type: () -> int
|
||||||
|
# Implemented to satisfy the ABC check. This is not needed by the
|
||||||
|
# resolver, and should not be used by the provider either (for
|
||||||
|
# performance reasons).
|
||||||
|
raise NotImplementedError("don't do this")
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def __bool__(self):
|
||||||
|
# type: () -> bool
|
||||||
|
if self._prefers_installed and self._installed:
|
||||||
|
return True
|
||||||
|
return any(self)
|
||||||
|
|
||||||
|
__nonzero__ = __bool__ # XXX: Python 2.
|
|
@ -54,30 +54,26 @@ class PipProvider(AbstractProvider):
|
||||||
self._upgrade_strategy = upgrade_strategy
|
self._upgrade_strategy = upgrade_strategy
|
||||||
self._user_requested = user_requested
|
self._user_requested = user_requested
|
||||||
|
|
||||||
def _sort_matches(self, matches):
|
def identify(self, dependency):
|
||||||
# type: (Iterable[Candidate]) -> Sequence[Candidate]
|
# type: (Union[Requirement, Candidate]) -> str
|
||||||
|
return dependency.name
|
||||||
|
|
||||||
# The requirement is responsible for returning a sequence of potential
|
def get_preference(
|
||||||
# candidates, one per version. The provider handles the logic of
|
self,
|
||||||
# deciding the order in which these candidates should be passed to
|
resolution, # type: Optional[Candidate]
|
||||||
# the resolver.
|
candidates, # type: Sequence[Candidate]
|
||||||
|
information # type: Sequence[Tuple[Requirement, Candidate]]
|
||||||
|
):
|
||||||
|
# type: (...) -> Any
|
||||||
|
transitive = all(parent is not None for _, parent in information)
|
||||||
|
return (transitive, bool(candidates))
|
||||||
|
|
||||||
# The `matches` argument is a sequence of candidates, one per version,
|
def find_matches(self, requirements):
|
||||||
# which are potential options to be installed. The requirement will
|
# type: (Sequence[Requirement]) -> Iterable[Candidate]
|
||||||
# have already sorted out whether to give us an already-installed
|
if not requirements:
|
||||||
# candidate or a version from PyPI (i.e., it will deal with options
|
return []
|
||||||
# like --force-reinstall and --ignore-installed).
|
name = requirements[0].name
|
||||||
|
|
||||||
# We now work out the correct order.
|
|
||||||
#
|
|
||||||
# 1. If no other considerations apply, later versions take priority.
|
|
||||||
# 2. An already installed distribution is preferred over any other,
|
|
||||||
# unless the user has requested an upgrade.
|
|
||||||
# Upgrades are allowed when:
|
|
||||||
# * The --upgrade flag is set, and
|
|
||||||
# - The project was specified on the command line, or
|
|
||||||
# - The project is a dependency and the "eager" upgrade strategy
|
|
||||||
# was requested.
|
|
||||||
def _eligible_for_upgrade(name):
|
def _eligible_for_upgrade(name):
|
||||||
# type: (str) -> bool
|
# type: (str) -> bool
|
||||||
"""Are upgrades allowed for this project?
|
"""Are upgrades allowed for this project?
|
||||||
|
@ -96,56 +92,11 @@ class PipProvider(AbstractProvider):
|
||||||
return (name in self._user_requested)
|
return (name in self._user_requested)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def sort_key(c):
|
return self._factory.find_candidates(
|
||||||
# type: (Candidate) -> int
|
requirements,
|
||||||
"""Return a sort key for the matches.
|
constraint=self._constraints.get(name, Constraint.empty()),
|
||||||
|
prefers_installed=(not _eligible_for_upgrade(name)),
|
||||||
The highest priority should be given to installed candidates that
|
|
||||||
are not eligible for upgrade. We use the integer value in the first
|
|
||||||
part of the key to sort these before other candidates.
|
|
||||||
|
|
||||||
We only pull the installed candidate to the bottom (i.e. most
|
|
||||||
preferred), but otherwise keep the ordering returned by the
|
|
||||||
requirement. The requirement is responsible for returning a list
|
|
||||||
otherwise sorted for the resolver, taking account for versions
|
|
||||||
and binary preferences as specified by the user.
|
|
||||||
"""
|
|
||||||
if c.is_installed and not _eligible_for_upgrade(c.name):
|
|
||||||
return 1
|
|
||||||
return 0
|
|
||||||
|
|
||||||
return sorted(matches, key=sort_key)
|
|
||||||
|
|
||||||
def identify(self, dependency):
|
|
||||||
# type: (Union[Requirement, Candidate]) -> str
|
|
||||||
return dependency.name
|
|
||||||
|
|
||||||
def get_preference(
|
|
||||||
self,
|
|
||||||
resolution, # type: Optional[Candidate]
|
|
||||||
candidates, # type: Sequence[Candidate]
|
|
||||||
information # type: Sequence[Tuple[Requirement, Optional[Candidate]]]
|
|
||||||
):
|
|
||||||
# type: (...) -> Any
|
|
||||||
"""Return a sort key to determine what dependency to look next.
|
|
||||||
|
|
||||||
A smaller value makes a dependency higher priority. We put direct
|
|
||||||
(user-requested) dependencies first since they may contain useful
|
|
||||||
user-specified version ranges. Users tend to expect us to catch
|
|
||||||
problems in them early as well.
|
|
||||||
"""
|
|
||||||
transitive = all(parent is not None for _, parent in information)
|
|
||||||
return (transitive, len(candidates))
|
|
||||||
|
|
||||||
def find_matches(self, requirements):
|
|
||||||
# type: (Sequence[Requirement]) -> Iterable[Candidate]
|
|
||||||
if not requirements:
|
|
||||||
return []
|
|
||||||
constraint = self._constraints.get(
|
|
||||||
requirements[0].name, Constraint.empty(),
|
|
||||||
)
|
)
|
||||||
candidates = self._factory.find_candidates(requirements, constraint)
|
|
||||||
return reversed(self._sort_matches(candidates))
|
|
||||||
|
|
||||||
def is_satisfied_by(self, requirement, candidate):
|
def is_satisfied_by(self, requirement, candidate):
|
||||||
# type: (Requirement, Candidate) -> bool
|
# type: (Requirement, Candidate) -> bool
|
||||||
|
|
|
@ -1020,3 +1020,29 @@ def test_new_resolver_no_deps_checks_requires_python(script):
|
||||||
"{}.{}.{} not in '<2'".format(*sys.version_info[:3])
|
"{}.{}.{} not in '<2'".format(*sys.version_info[:3])
|
||||||
)
|
)
|
||||||
assert message in result.stderr
|
assert message in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_new_resolver_prefers_installed_in_upgrade_if_latest(script):
|
||||||
|
create_basic_wheel_for_package(script, "pkg", "1")
|
||||||
|
local_pkg = create_test_package_with_setup(script, name="pkg", version="2")
|
||||||
|
|
||||||
|
# Install the version that's not on the index.
|
||||||
|
script.pip(
|
||||||
|
"install",
|
||||||
|
"--use-feature=2020-resolver",
|
||||||
|
"--no-cache-dir",
|
||||||
|
"--no-index",
|
||||||
|
local_pkg,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Now --upgrade should still pick the local version because it's "better".
|
||||||
|
script.pip(
|
||||||
|
"install",
|
||||||
|
"--use-feature=2020-resolver",
|
||||||
|
"--no-cache-dir",
|
||||||
|
"--no-index",
|
||||||
|
"--find-links", script.scratch_path,
|
||||||
|
"--upgrade",
|
||||||
|
"pkg",
|
||||||
|
)
|
||||||
|
assert_installed(script, pkg="2")
|
||||||
|
|
|
@ -58,8 +58,10 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory):
|
||||||
"""Requirements should return the correct number of candidates"""
|
"""Requirements should return the correct number of candidates"""
|
||||||
for spec, _, match_count in test_cases:
|
for spec, _, match_count in test_cases:
|
||||||
req = factory.make_requirement_from_spec(spec, comes_from=None)
|
req = factory.make_requirement_from_spec(spec, comes_from=None)
|
||||||
matches = factory.find_candidates([req], Constraint.empty())
|
matches = factory.find_candidates(
|
||||||
assert len(list(matches)) == match_count
|
[req], Constraint.empty(), prefers_installed=False,
|
||||||
|
)
|
||||||
|
assert sum(1 for _ in matches) == match_count
|
||||||
|
|
||||||
|
|
||||||
def test_new_resolver_candidates_match_requirement(test_cases, factory):
|
def test_new_resolver_candidates_match_requirement(test_cases, factory):
|
||||||
|
@ -67,7 +69,10 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory):
|
||||||
"""
|
"""
|
||||||
for spec, _, _ in test_cases:
|
for spec, _, _ in test_cases:
|
||||||
req = factory.make_requirement_from_spec(spec, comes_from=None)
|
req = factory.make_requirement_from_spec(spec, comes_from=None)
|
||||||
for c in factory.find_candidates([req], Constraint.empty()):
|
candidates = factory.find_candidates(
|
||||||
|
[req], Constraint.empty(), prefers_installed=False,
|
||||||
|
)
|
||||||
|
for c in candidates:
|
||||||
assert isinstance(c, Candidate)
|
assert isinstance(c, Candidate)
|
||||||
assert req.is_satisfied_by(c)
|
assert req.is_satisfied_by(c)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue