diff --git a/news/8792.bugfix b/news/8792.bugfix new file mode 100644 index 000000000..e83bdb09c --- /dev/null +++ b/news/8792.bugfix @@ -0,0 +1,2 @@ +New resolver: Pick up hash declarations in constraints files and use them to +filter available distributions. diff --git a/news/8839.bugfix b/news/8839.bugfix new file mode 100644 index 000000000..987b801e9 --- /dev/null +++ b/news/8839.bugfix @@ -0,0 +1,3 @@ +New resolver: If a package appears multiple times in user specification with +different ``--hash`` options, only hashes that present in all specifications +should be allowed. diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index 9245747bf..7c09cd70b 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -1,5 +1,8 @@ +from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.hashes import Hashes from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -8,7 +11,6 @@ if MYPY_CHECK_RUNNING: from pip._vendor.packaging.version import _BaseVersion from pip._internal.models.link import Link - from pip._internal.req.req_install import InstallRequirement CandidateLookup = Tuple[ Optional["Candidate"], @@ -24,6 +26,39 @@ def format_name(project, extras): return "{}[{}]".format(project, ",".join(canonical_extras)) +class Constraint(object): + def __init__(self, specifier, hashes): + # type: (SpecifierSet, Hashes) -> None + self.specifier = specifier + self.hashes = hashes + + @classmethod + def empty(cls): + # type: () -> Constraint + return Constraint(SpecifierSet(), Hashes()) + + @classmethod + def from_ireq(cls, ireq): + # type: (InstallRequirement) -> Constraint + return Constraint(ireq.specifier, ireq.hashes(trust_internet=False)) + + def __nonzero__(self): + # type: () -> bool + return bool(self.specifier) or bool(self.hashes) + + def __bool__(self): + # type: () -> bool + return self.__nonzero__() + + def __and__(self, other): + # type: (InstallRequirement) -> Constraint + if not isinstance(other, InstallRequirement): + return NotImplemented + specifier = self.specifier & other.specifier + hashes = self.hashes & other.hashes(trust_internet=False) + return Constraint(specifier, hashes) + + class Requirement(object): @property def name(self): diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index dab23aa09..172f054fa 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -22,6 +22,7 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import running_under_virtualenv +from .base import Constraint from .candidates import ( AlreadyInstalledCandidate, EditableCandidate, @@ -154,6 +155,7 @@ class Factory(object): self, ireqs, # type: Sequence[InstallRequirement] specifier, # type: SpecifierSet + hashes, # type: Hashes ): # type: (...) -> Iterable[Candidate] if not ireqs: @@ -166,11 +168,10 @@ class Factory(object): template = ireqs[0] name = canonicalize_name(template.req.name) - hashes = Hashes() extras = frozenset() # type: FrozenSet[str] for ireq in ireqs: specifier &= ireq.req.specifier - hashes |= ireq.hashes(trust_internet=False) + hashes &= ireq.hashes(trust_internet=False) extras |= frozenset(ireq.extras) # We use this to ensure that we only yield a single candidate for @@ -220,7 +221,7 @@ class Factory(object): return six.itervalues(candidates) def find_candidates(self, requirements, constraint): - # type: (Sequence[Requirement], SpecifierSet) -> Iterable[Candidate] + # type: (Sequence[Requirement], Constraint) -> Iterable[Candidate] explicit_candidates = set() # type: Set[Candidate] ireqs = [] # type: List[InstallRequirement] for req in requirements: @@ -233,7 +234,11 @@ class Factory(object): # If none of the requirements want an explicit candidate, we can ask # the finder for candidates. if not explicit_candidates: - return self._iter_found_candidates(ireqs, constraint) + return self._iter_found_candidates( + ireqs, + constraint.specifier, + constraint.hashes, + ) if constraint: name = explicit_candidates.pop().name diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index b2eb9d06e..80577a61c 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -1,8 +1,9 @@ -from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib.providers import AbstractProvider from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from .base import Constraint + if MYPY_CHECK_RUNNING: from typing import ( Any, @@ -41,7 +42,7 @@ class PipProvider(AbstractProvider): def __init__( self, factory, # type: Factory - constraints, # type: Dict[str, SpecifierSet] + constraints, # type: Dict[str, Constraint] ignore_dependencies, # type: bool upgrade_strategy, # type: str user_requested, # type: Set[str] @@ -134,7 +135,7 @@ class PipProvider(AbstractProvider): if not requirements: return [] constraint = self._constraints.get( - requirements[0].name, SpecifierSet(), + requirements[0].name, Constraint.empty(), ) candidates = self._factory.find_candidates(requirements, constraint) return reversed(self._sort_matches(candidates)) diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index aecddb113..449cfea28 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -14,12 +14,12 @@ from pip._internal.resolution.resolvelib.provider import PipProvider from pip._internal.utils.misc import dist_is_editable from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from .base import Constraint from .factory import Factory if MYPY_CHECK_RUNNING: from typing import Dict, List, Optional, Set, Tuple - from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib.resolvers import Result from pip._vendor.resolvelib.structs import Graph @@ -81,7 +81,7 @@ class Resolver(BaseResolver): def resolve(self, root_reqs, check_supported_wheels): # type: (List[InstallRequirement], bool) -> RequirementSet - constraints = {} # type: Dict[str, SpecifierSet] + constraints = {} # type: Dict[str, Constraint] user_requested = set() # type: Set[str] requirements = [] for req in root_reqs: @@ -94,9 +94,9 @@ class Resolver(BaseResolver): continue name = canonicalize_name(req.name) if name in constraints: - constraints[name] = constraints[name] & req.specifier + constraints[name] &= req else: - constraints[name] = req.specifier + constraints[name] = Constraint.from_ireq(req) else: if req.user_supplied and req.name: user_requested.add(canonicalize_name(req.name)) diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py index d1b062fed..d9f74a640 100644 --- a/src/pip/_internal/utils/hashes.py +++ b/src/pip/_internal/utils/hashes.py @@ -46,16 +46,24 @@ class Hashes(object): """ self._allowed = {} if hashes is None else hashes - def __or__(self, other): + def __and__(self, other): # type: (Hashes) -> Hashes if not isinstance(other, Hashes): return NotImplemented - new = self._allowed.copy() + + # If either of the Hashes object is entirely empty (i.e. no hash + # specified at all), all hashes from the other object are allowed. + if not other: + return self + if not self: + return other + + # Otherwise only hashes that present in both objects are allowed. + new = {} for alg, values in iteritems(other._allowed): - try: - new[alg] += values - except KeyError: - new[alg] = values + if alg not in self._allowed: + continue + new[alg] = [v for v in values if v in self._allowed[alg]] return Hashes(new) @property diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py new file mode 100644 index 000000000..4b13ebc30 --- /dev/null +++ b/tests/functional/test_new_resolver_hashes.py @@ -0,0 +1,213 @@ +import collections +import hashlib + +import pytest + +from pip._internal.utils.urls import path_to_url +from tests.lib import ( + create_basic_sdist_for_package, + create_basic_wheel_for_package, +) + +_FindLinks = collections.namedtuple( + "_FindLinks", "index_html sdist_hash wheel_hash", +) + + +def _create_find_links(script): + sdist_path = create_basic_sdist_for_package(script, "base", "0.1.0") + wheel_path = create_basic_wheel_for_package(script, "base", "0.1.0") + + sdist_hash = hashlib.sha256(sdist_path.read_bytes()).hexdigest() + wheel_hash = hashlib.sha256(wheel_path.read_bytes()).hexdigest() + + index_html = script.scratch_path / "index.html" + index_html.write_text( + """ + {sdist_path.stem} + {wheel_path.stem} + """.format( + sdist_url=path_to_url(sdist_path), + sdist_hash=sdist_hash, + sdist_path=sdist_path, + wheel_url=path_to_url(wheel_path), + wheel_hash=wheel_hash, + wheel_path=wheel_path, + ) + ) + + return _FindLinks(index_html, sdist_hash, wheel_hash) + + +@pytest.mark.parametrize( + "requirements_template, message", + [ + ( + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + """, + "Checked 2 links for project {name!r} against 2 hashes " + "(2 matches, 0 no digest): discarding no candidates", + ), + ( + # Different hash lists are intersected. + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + base==0.1.0 --hash=sha256:{sdist_hash} + """, + "Checked 2 links for project {name!r} against 1 hashes " + "(1 matches, 0 no digest): discarding 1 non-matches", + ), + ], + ids=["identical", "intersect"], +) +def test_new_resolver_hash_intersect(script, requirements_template, message): + find_links = _create_find_links(script) + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + requirements_template.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--verbose", + "--requirement", requirements_txt, + ) + + assert message.format(name=u"base") in result.stdout, str(result) + + +def test_new_resolver_hash_intersect_from_constraint(script): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + "base==0.1.0 --hash=sha256:{sdist_hash}".format( + sdist_hash=find_links.sdist_hash, + ), + ) + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + """.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--verbose", + "--constraint", constraints_txt, + "--requirement", requirements_txt, + ) + + message = ( + "Checked 2 links for project {name!r} against 1 hashes " + "(1 matches, 0 no digest): discarding 1 non-matches" + ).format(name=u"base") + assert message in result.stdout, str(result) + + +@pytest.mark.parametrize( + "requirements_template, constraints_template", + [ + ( + """ + base==0.1.0 --hash=sha256:{sdist_hash} + base==0.1.0 --hash=sha256:{wheel_hash} + """, + "", + ), + ( + "base==0.1.0 --hash=sha256:{sdist_hash}", + "base==0.1.0 --hash=sha256:{wheel_hash}", + ), + ], + ids=["both-requirements", "one-each"], +) +def test_new_resolver_hash_intersect_empty( + script, requirements_template, constraints_template, +): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + constraints_template.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + requirements_template.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--constraint", constraints_txt, + "--requirement", requirements_txt, + expect_error=True, + ) + + assert ( + "THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE." + ) in result.stderr, str(result) + + +def test_new_resolver_hash_intersect_empty_from_constraint(script): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + """ + base==0.1.0 --hash=sha256:{sdist_hash} + base==0.1.0 --hash=sha256:{wheel_hash} + """.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--constraint", constraints_txt, + "base==0.1.0", + expect_error=True, + ) + + message = ( + "Hashes are required in --require-hashes mode, but they are missing " + "from some requirements." + ) + assert message in result.stderr, str(result) diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 21de3df4a..a03edb6f7 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -1,8 +1,7 @@ import pytest -from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib import BaseReporter, Resolver -from pip._internal.resolution.resolvelib.base import Candidate +from pip._internal.resolution.resolvelib.base import Candidate, Constraint from pip._internal.utils.urls import path_to_url # NOTE: All tests are prefixed `test_rlr` (for "test resolvelib resolver"). @@ -59,7 +58,7 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory): """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - matches = factory.find_candidates([req], SpecifierSet()) + matches = factory.find_candidates([req], Constraint.empty()) assert len(list(matches)) == match_count @@ -68,7 +67,7 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory): """ for spec, _, _ in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - for c in factory.find_candidates([req], SpecifierSet()): + for c in factory.find_candidates([req], Constraint.empty()): assert isinstance(c, Candidate) assert req.is_satisfied_by(c)